diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6935beb..20572ef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,22 +88,35 @@ jobs: run: cargo run -- exec -m walltime --skip-upload --warmup-time 0s --max-rounds 5 -- ls -la bpf-tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, codspeed-macro] # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + include: + # The folio CO-RE relocations in the rmap hooks depend on kernel + # struct layouts that changed across versions (folio.flags in 6.18, + # pageflags values in 6.10/6.12); keep older-kernel images in the + # matrix so the legacy layouts stay covered once the latest images + # move past them. + - os: ubuntu-22.04 + test: rss_tests + - os: ubuntu-22.04-arm + test: rss_tests steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: lfs: true submodules: true + - name: Show kernel version + run: uname -rm - uses: ./.github/actions/install-rust with: - cache-key: ${{ matrix.test }} + cache-key: ${{ matrix.os }}-${{ matrix.test }} - uses: ./.github/actions/install-bpf-deps - name: Install additional allocators @@ -115,8 +128,11 @@ jobs: run: sudo -E $(which cargo) test --lib --test ${{ matrix.test }} -- --test-threads 1 --nocapture working-directory: crates/memtrack - # Since we ran the tests with sudo, the build artifacts will have root ownership + # Since we ran the tests with sudo, the build artifacts will have root + # ownership; always restore it so a failed run cannot poison the + # workspace of later jobs on persistent self-hosted runners. - name: Clean up + if: always() run: sudo chown -R $USER:$USER . ~/.cargo benchmarks: diff --git a/crates/memtrack/.clang-format b/crates/memtrack/.clang-format index b687ea40..4755a340 100644 --- a/crates/memtrack/.clang-format +++ b/crates/memtrack/.clang-format @@ -4,3 +4,4 @@ BasedOnStyle: Google PointerAlignment: Left ColumnLimit: 100 IndentWidth: 4 +AllowShortFunctionsOnASingleLine: None diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index c4311d9e..b3bd218e 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -46,7 +46,7 @@ bindgen = "0.72" tempfile = { workspace = true } rstest = { workspace = true } test-log = { workspace = true } -insta = { workspace = true } +insta = { workspace = true, features = ["json", "redactions"] } test-with = { workspace = true } [package.metadata.dist] diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 440d7a87..88566b36 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -5,22 +5,24 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ - BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ - SEC("uprobe") \ - int uprobe_##name(struct pt_regs* ctx) { return store_param(&name##_arg, arg_expr); } \ - SEC("uretprobe") \ - int uretprobe_##name(struct pt_regs* ctx) { \ - __u64* arg_ptr = take_param(&name##_arg); \ - if (!arg_ptr) { \ - return 0; \ - } \ - __u64 ret_val = PT_REGS_RC(ctx); \ - if (ret_val == 0) { \ - return 0; \ - } \ - __u64 arg0 = *arg_ptr; \ - submit_block; \ +#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ + BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ + SEC("uprobe") \ + int uprobe_##name(struct pt_regs* ctx) { \ + return store_param(&name##_arg, arg_expr); \ + } \ + SEC("uretprobe") \ + int uretprobe_##name(struct pt_regs* ctx) { \ + __u64* arg_ptr = take_param(&name##_arg); \ + if (!arg_ptr) { \ + return 0; \ + } \ + __u64 ret_val = PT_REGS_RC(ctx); \ + if (ret_val == 0) { \ + return 0; \ + } \ + __u64 arg0 = *arg_ptr; \ + submit_block; \ } #define UPROBE_RET(name, arg_expr, submit_block) \ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index ff41f0be..5dabffbe 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -9,6 +9,11 @@ #define EVENT_TYPE_MMAP 6 #define EVENT_TYPE_MUNMAP 7 #define EVENT_TYPE_BRK 8 +#define EVENT_TYPE_RSS 9 +#define EVENT_TYPE_RMAP 10 +#define EVENT_TYPE_FORK 11 +#define EVENT_TYPE_EXEC 12 +#define EVENT_TYPE_EXIT 13 /* Common header shared by all event types */ struct event_header { @@ -45,6 +50,22 @@ struct event { uint64_t addr; /* address of mapping */ uint64_t size; /* size of mapping */ } mmap; + + struct { + int32_t member; + uint64_t size; + } rss; + + struct { + int32_t member; /* MM_* counter index, same values as rss.member */ + int64_t delta; + uint64_t addr; + } rmap; + + /* Process lifecycle events (fork carries the parent; exec/exit have no payload) */ + struct { + uint32_t parent_pid; + } fork; } data; }; diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 1c5ad8d1..e46a144b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "rss.bpf.h" #include "utils/event_helpers.h" #include "utils/map_helpers.h" #include "utils/process_tracking.h" diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h new file mode 100644 index 00000000..2850a634 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -0,0 +1,388 @@ +#ifndef __RSS_BPF_H__ +#define __RSS_BPF_H__ + +#include "event.h" +#include "utils/event_helpers.h" +#include "utils/process_tracking.h" + +/* (rss_stat mm_id << 32 | member) -> {owning tgid, last in-context size}. Keyed per + * counter so an external (curr==0) update is attributed only once that mm/member was + * established in-context. An external event may only lower a counter: any size above + * the last in-context value is dropped, so neither a stale/racing reclaim read nor an + * mm_id hash collision with another task can invent a peak. LRU eviction + re-seeding + * on the owner's next in-context event covers hash reuse, so no teardown hook needed. */ +struct rss_owner { + __u32 pid; + __u64 size; +}; +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 40960); + __type(key, __u64); + __type(value, struct rss_owner); +} mm_to_pid SEC(".maps"); + +/* Foreign-actor rmap attribution: rmap events run by a task other than the mm's + * owner (kswapd reclaim, another process's process_madvise, khugepaged, KSM, + * uffd) carry no owning-pid context, so mm_owner recovers it from the mm_struct + * pointer. pid_mm is the inverse, letting the exec and exit hooks remove an entry + * by value. + * + * Lifecycle invariant: every mm_owner entry is removed when its process execs + * (the old mm is freed mid-life) or when its thread group dies, whichever comes + * first; LRU eviction is only a backstop. A stale entry surviving mm-pointer + * reuse would misattribute another process's events, so ownership is only ever + * registered from an in-context (task->mm == mm) event. + * + * pid_mm is a plain hash on purpose: an LRU inverse could be evicted while its + * forward twin stays lookup-hot, leaving exec/exit unable to remove the live + * mm_owner entry. Like tracked_pids, its entries are bound to the process + * lifecycle and removed at group death. */ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 10240); + __type(key, __u64); + __type(value, __u32); +} mm_owner SEC(".maps"); +BPF_HASH_MAP(pid_mm, __u32, __u64, 10240); + +#define FOLIO_MAPPING_ANON 0x1UL + +const volatile __u32 page_shift = 12; + +static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { + SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { + e->data.rss.member = member; + e->data.rss.size = size; + }); +} + +SEC("tracepoint/kmem/rss_stat") +int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { + if (ctx->member == MM_SWAPENTS) { + return 0; + } + + __u32 cur = bpf_get_current_pid_tgid() >> 32; + __u64 key = ((__u64)ctx->mm_id << 32) | (__u32)ctx->member; + __u64 size = ctx->size; + __u32 owner; + + if (ctx->curr) { + if (!is_tracked(cur)) { + return 0; + } + owner = cur; + struct rss_owner state = {.pid = cur, .size = size}; + bpf_map_update_elem(&mm_to_pid, &key, &state, BPF_ANY); + } else { + struct rss_owner* found = bpf_map_lookup_elem(&mm_to_pid, &key); + if (!found) { + return 0; + } + owner = found->pid; + /* The owner's own teardown also presents as curr==0 (current->mm is cleared + * on exit), so drop it. Genuine external actors (reclaim, another process's + * madvise) run in a different task, so cur != owner. */ + if (cur == owner) { + return 0; + } + /* An external actor may only lower a counter. A larger value is a stale + * reclaim read or an mm_id hash collision with another task; dropping it + * keeps the reconstructed peak identical to the in-context timeline. */ + if (size > found->size) { + return 0; + } + found->size = size; + } + + return submit_rss_event(owner, ctx->member, size); +} + +/* Kernels < 6.18 store folio->flags as a bare unsigned long instead of + * memdesc_flags_t; probe which layout the running kernel has. */ +struct folio___legacy { + unsigned long flags; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_read_flags(struct folio* folio) { + if (bpf_core_field_exists(folio->flags.f)) { + return BPF_CORE_READ(folio, flags).f; + } + return BPF_CORE_READ((struct folio___legacy*)folio, flags); +} + +/* Kernels < 6.6 store the large-folio order in a dedicated byte instead of + * the low byte of _flags_1. */ +struct folio___order_byte { + unsigned char _folio_order; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_order(struct folio* folio) { + if (bpf_core_field_exists(((struct folio___order_byte*)folio)->_folio_order)) { + return BPF_CORE_READ((struct folio___order_byte*)folio, _folio_order); + } + return BPF_CORE_READ(folio, _flags_1) & 0xff; +} + +static __always_inline __u64 folio_nr_pages_est(struct folio* folio) { + unsigned long flags = folio_read_flags(folio); + if (!(flags & (1UL << bpf_core_enum_value(enum pageflags, PG_head)))) { + return 1; + } + unsigned long order = folio_order(folio); + return 1UL << order; +} + +static __always_inline int folio_is_anon(struct folio* folio) { + unsigned long mapping = (unsigned long)BPF_CORE_READ(folio, mapping); + return (mapping & FOLIO_MAPPING_ANON) != 0; +} + +/* Mirrors the kernel's mm_counter(): anon folios are also swapbacked, so the + * anon check must come first. */ +static __always_inline __s32 folio_mm_counter(struct folio* folio) { + if (folio_is_anon(folio)) { + return MM_ANONPAGES; + } + unsigned long flags = folio_read_flags(folio); + if (flags & (1UL << bpf_core_enum_value(enum pageflags, PG_swapbacked))) { + return MM_SHMEMPAGES; + } + return MM_FILEPAGES; +} + +static __always_inline __u64 folio_page_address(struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + __u64 page_idx = ((__u64)page - (__u64)folio) / bpf_core_type_size(struct page); + __u64 pgoff = BPF_CORE_READ(folio, index) + page_idx; + __u64 vm_pgoff = BPF_CORE_READ(vma, vm_pgoff); + __u64 vm_start = BPF_CORE_READ(vma, vm_start); + if (pgoff < vm_pgoff) { + return vm_start; + } + return vm_start + ((pgoff - vm_pgoff) << page_shift); +} + +static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __s64 delta, + __u64 addr) { + __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); + struct task_struct* task = bpf_get_current_task_btf(); + __u64 tid = bpf_get_current_pid_tgid(); + __u32 pid = tid >> 32; + + if ((__u64)BPF_CORE_READ(task, mm) == mm) { + if (!is_tracked(pid)) { + return 0; + } + + /* Register ownership so foreign actors can later attribute to this pid. + * Both maps are validated (not just written) on every in-context event: + * the lookups keep the hot path cheap AND keep both entries LRU-fresh, + * since pid_mm is otherwise never read until exec/exit and could be + * evicted independently of its still-hot mm_owner twin. */ + __u32* owner = bpf_map_lookup_elem(&mm_owner, &mm); + if (!owner || *owner != pid) { + bpf_map_update_elem(&mm_owner, &mm, &pid, BPF_ANY); + } + __u64* cur_mm = bpf_map_lookup_elem(&pid_mm, &pid); + if (!cur_mm || *cur_mm != mm) { + bpf_map_update_elem(&pid_mm, &pid, &mm, BPF_ANY); + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); + } + + /* Foreign actor (task->mm != mm, including kthreads whose task->mm is NULL): + * recover the owner from the in-context registration. Fail toward dropping + * the event on any uncertainty about ownership. */ + __u32* found = bpf_map_lookup_elem(&mm_owner, &mm); + if (!found) { + return 0; + } + __u32 owner = *found; + if (!is_tracked(owner)) { + return 0; + } + + /* SUBMIT_EVENT_AS stamps header.tid from the current task, identifying the + * foreign actor that performed the rmap change. */ + SUBMIT_EVENT_AS(owner, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); +} + +SEC("fentry/folio_add_new_anon_rmap") +int BPF_PROG(fentry_folio_add_new_anon_rmap, struct folio* folio, struct vm_area_struct* vma, + unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_anon_rmap_ptes") +int BPF_PROG(fentry_folio_add_anon_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)nr_pages, address); +} + +SEC("fentry/folio_add_anon_rmap_pmd") +int BPF_PROG(fentry_folio_add_anon_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_file_rmap_ptes") +int BPF_PROG(fentry_folio_add_file_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)nr_pages, + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_pmd") +int BPF_PROG(fentry_folio_add_file_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_pud") +int BPF_PROG(fentry_folio_add_file_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_ptes") +int BPF_PROG(fentry_folio_remove_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)nr_pages, + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_pmd") +int BPF_PROG(fentry_folio_remove_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_pud") +int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +/* == Process lifecycle events == + * + * FORK lets userland seed a child's RSS from its parent at fork time: the + * kernel copies the mm counters during dup_mmap, but those updates fire + * rss_stat out of the child's context and anon COW faults are + * counter-neutral, so a child that only touches inherited memory never + * reports its RSS on its own. EXEC and EXIT mark the points where the + * address space is replaced or torn down, so userland resets to zero. + */ + +#define CLONE_THREAD 0x00010000 + +SEC("tracepoint/task/task_newtask") +int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { + if (ctx->clone_flags & CLONE_THREAD) { + return 0; + } + + __u64 tid = bpf_get_current_pid_tgid(); + __u32 parent_pid = tid >> 32; + if (!is_tracked(parent_pid)) { + return 0; + } + + /* Register the child here rather than on sched_process_fork: that + * tracepoint fires for CLONE_THREAD too and carries only raw task pids, + * which would fill the tracking maps with thread tids that no exit path + * removes (group death deletes only the tgid). task_newtask fires before + * wake_up_new_task, so registration precedes any event from the child. */ + __u32 child_pid = ctx->pid; + track_child(child_pid, parent_pid); + + SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); +} + +/* Remove pid's ownership registration. The mm_owner value is verified against + * pid before deleting: a stale pid_mm entry (LRU eviction skew) could otherwise + * point at an mm since re-registered by another process, and deleting that + * would silence a live owner's foreign attribution. */ +static __always_inline void drop_mm_ownership(__u32 pid) { + __u64* mm = bpf_map_lookup_elem(&pid_mm, &pid); + if (mm) { + __u32* owner = bpf_map_lookup_elem(&mm_owner, mm); + if (owner && *owner == pid) { + bpf_map_delete_elem(&mm_owner, mm); + } + } + bpf_map_delete_elem(&pid_mm, &pid); +} + +SEC("tracepoint/sched/sched_process_exec") +int tracepoint_sched_process_exec(void* ctx) { + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { + return 0; + } + + /* Maintain ownership before submitting (SUBMIT_EVENT_AS returns from the + * function). Exec frees the old mm long before group death, so the stale + * pointer must be dropped here or a reused mm_struct would be misattributed. */ + drop_mm_ownership(pid); + + struct task_struct* task = bpf_get_current_task_btf(); + __u64 new_mm = (__u64)BPF_CORE_READ(task, mm); + if (new_mm) { + bpf_map_update_elem(&mm_owner, &new_mm, &pid, BPF_ANY); + bpf_map_update_elem(&pid_mm, &pid, &new_mm, BPF_ANY); + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXEC, {}); +} + +SEC("tracepoint/sched/sched_process_exit") +int tracepoint_sched_process_exit(void* ctx) { + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { + return 0; + } + + /* EXIT marks the death of the whole thread group, not of one thread: the + * leader can pthread_exit while workers keep running, and the last thread + * to exit need not be the leader. do_exit decrements signal->live before + * this tracepoint fires, so live == 0 identifies the dying thread group's + * final exit — but concurrently exiting threads can BOTH read 0, so the + * tracked_pids delete below arbitrates: only the task that wins it emits. */ + struct task_struct* task = bpf_get_current_task_btf(); + if (BPF_CORE_READ(task, signal, live.counter) != 0) { + return 0; + } + + /* Untrack the pid before submitting: lifetime events are gated only on + * is_tracked, so a stale entry would keep streaming events if the kernel + * reuses the pid for an unrelated process. Untracking here also keeps the + * fixed-size tracking maps from filling up over long sessions. The delete + * doubles as the exactly-once claim on EXIT. */ + if (bpf_map_delete_elem(&tracked_pids, &pid) != 0) { + return 0; + } + bpf_map_delete_elem(&pids_ppid, &pid); + + /* Drop the ownership mapping so foreign actors stop attributing to a pid + * the kernel may reuse. */ + drop_mm_ownership(pid); + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXIT, {}); +} + +#endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index b5d79b37..fbbc0536 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -38,14 +38,18 @@ static __always_inline __u64* take_param(void* map) { return value; } -#define SUBMIT_EVENT(evt_type, fill_data) \ +/* Submission is split into two classes: + * - lifetime events (rss_stat, rmap, fork/exec/exit): emitted whenever the + * pid is tracked, ignoring the enable toggle. The parser reconstructs + * absolute per-process state from these, so it needs every event from + * process birth — a delta stream that starts mid-life can never recover + * the resident baseline faulted before enable. + * - allocator events (malloc/free/mmap/...): high-volume and only meaningful + * inside a measurement window, so they stay behind is_enabled(). + */ +#define SUBMIT_EVENT_AS(owner_pid, evt_type, fill_data) \ { \ __u64 tid = bpf_get_current_pid_tgid(); \ - __u32 pid = tid >> 32; \ - \ - if (!is_tracked(pid) || !is_enabled()) { \ - return 0; \ - } \ \ struct event* e = bpf_ringbuf_reserve(&events, sizeof(*e), 0); \ if (!e) { \ @@ -58,7 +62,7 @@ static __always_inline __u64* take_param(void* map) { } \ \ e->header.timestamp = bpf_ktime_get_ns(); \ - e->header.pid = pid; \ + e->header.pid = owner_pid; \ e->header.tid = tid & 0xFFFFFFFF; \ e->header.event_type = evt_type; \ \ @@ -68,33 +72,54 @@ static __always_inline __u64* take_param(void* map) { return 0; \ } +#define SUBMIT_EVENT(evt_type, fill_data) \ + { \ + __u64 tid = bpf_get_current_pid_tgid(); \ + __u32 pid = tid >> 32; \ + \ + if (!is_tracked(pid)) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT_AS(pid, evt_type, fill_data); \ + } + +#define SUBMIT_GATED_EVENT(evt_type, fill_data) \ + { \ + if (!is_enabled()) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT(evt_type, fill_data); \ + } + static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_MALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_CALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); } static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { - SUBMIT_EVENT(EVENT_TYPE_REALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; @@ -102,7 +127,7 @@ static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, } static __always_inline int submit_mmap_event(__u64 addr, __u64 size, __u8 event_type) { - SUBMIT_EVENT(event_type, { + SUBMIT_GATED_EVENT(event_type, { e->data.mmap.addr = addr; e->data.mmap.size = size; }); diff --git a/crates/memtrack/src/ebpf/c/utils/process_tracking.h b/crates/memtrack/src/ebpf/c/utils/process_tracking.h index 19cff338..69dd2a73 100644 --- a/crates/memtrack/src/ebpf/c/utils/process_tracking.h +++ b/crates/memtrack/src/ebpf/c/utils/process_tracking.h @@ -43,16 +43,4 @@ static __always_inline void track_child(__u32 child_pid, __u32 parent_pid) { bpf_map_update_elem(&pids_ppid, &child_pid, &parent_pid, BPF_ANY); } -SEC("tracepoint/sched/sched_process_fork") -int tracepoint_sched_fork(struct trace_event_raw_sched_process_fork* ctx) { - __u32 parent_pid = ctx->parent_pid; - __u32 child_pid = ctx->child_pid; - - if (is_tracked(parent_pid)) { - track_child(child_pid, parent_pid); - } - - return 0; -} - #endif /* __PROCESS_TRACKING_H__ */ diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index f18216c6..ccab7f5f 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -74,6 +74,28 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.mmap.size, }, ), + EVENT_TYPE_RSS => ( + 0, + MemtrackEventKind::Rss { + member: event.data.rss.member, + size: event.data.rss.size, + }, + ), + EVENT_TYPE_RMAP => ( + event.data.rmap.addr, + MemtrackEventKind::Rmap { + member: event.data.rmap.member, + delta: event.data.rmap.delta, + }, + ), + EVENT_TYPE_FORK => ( + 0, + MemtrackEventKind::Fork { + parent_pid: event.data.fork.parent_pid as i32, + }, + ), + EVENT_TYPE_EXEC => (0, MemtrackEventKind::Exec), + EVENT_TYPE_EXIT => (0, MemtrackEventKind::Exit), unknown => { panic!("Unknown event type: {unknown}"); } @@ -185,4 +207,92 @@ mod tests { _ => panic!("Expected Malloc event kind"), } } + + #[test] + fn test_parse_rss_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RSS as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rss.member = 1; + event.data.rss.size = 4096 * 10; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0); + + match parsed.kind { + MemtrackEventKind::Rss { member, size } => { + assert_eq!(member, 1); + assert_eq!(size, 4096 * 10); + } + _ => panic!("Expected Rss event kind"), + } + } + + #[test] + fn test_parse_rmap_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RMAP as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rmap.member = 3; + event.data.rmap.delta = 8; + event.data.rmap.addr = 0x7f00; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0x7f00); + + match parsed.kind { + MemtrackEventKind::Rmap { member, delta } => { + assert_eq!(member, 3); + assert_eq!(delta, 8); + } + _ => panic!("Expected Rmap event kind"), + } + } + + #[test] + fn test_parse_fork_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_FORK as u8; + event.header.timestamp = 12345678; + event.header.pid = 1001; + event.header.tid = 2000; + event.data.fork.parent_pid = 1000; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1001); + + match parsed.kind { + MemtrackEventKind::Fork { parent_pid } => { + assert_eq!(parent_pid, 1000); + } + _ => panic!("Expected Fork event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index 88da64a6..f4eca485 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -134,3 +134,18 @@ macro_rules! attach_tracepoint { } }; } + +macro_rules! attach_fentry { + ($func:ident, $prog:ident) => { + fn $func(&mut self) -> Result<()> { + let link = self + .skel + .progs + .$prog + .attach() + .context(format!("Failed to attach {} fentry", stringify!($prog)))?; + self.probes.push(link); + Ok(()) + } + }; +} diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 2faba0ec..f40f4c1b 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -1,8 +1,9 @@ use crate::prelude::*; -use libbpf_rs::Link; use libbpf_rs::skel::OpenSkel; use libbpf_rs::skel::SkelBuilder; +use libbpf_rs::{AsRawLibbpf, Link}; use std::collections::HashMap; +use std::ffi::CString; use std::mem::MaybeUninit; use std::path::Path; @@ -62,6 +63,27 @@ fn symbol_file_offset<'a>( Some((address - section.address() + sh_offset) as usize) } +fn page_shift() -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "Failed to read system page size"); + Ok((page_size as u32).trailing_zeros()) +} + +fn kernel_func_exists(btf: &libbpf_rs::btf::Btf, name: &str) -> bool { + let Ok(name) = CString::new(name) else { + return false; + }; + + let id = unsafe { + libbpf_rs::libbpf_sys::btf__find_by_name_kind( + btf.as_libbpf_object().as_ptr(), + name.as_ptr(), + libbpf_rs::libbpf_sys::BTF_KIND_FUNC, + ) + }; + id >= 0 +} + /// Attach targets resolved from a library's symbol tables. pub struct ResolvedSymbols { offsets: HashMap, @@ -76,16 +98,100 @@ impl ResolvedSymbols { pub struct MemtrackBpf { skel: Box>, probes: Vec, + track_rmap: bool, + btf_disabled_rmap_targets: Vec<&'static str>, } impl MemtrackBpf { pub fn new() -> Result { + let track_rmap = std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1"); + Self::new_with_rmap(track_rmap) + } + + pub fn new_with_rmap(track_rmap: bool) -> Result { let builder = MainSkelBuilder::default(); let open_object = Box::leak(Box::new(MaybeUninit::uninit())); - let open_skel = builder + let mut open_skel = builder .open(open_object) .context("Failed to open syscalls BPF skeleton")?; + open_skel + .maps + .rodata_data + .as_deref_mut() + .context("rodata map missing")? + .page_shift = page_shift()?; + let mut btf_disabled_rmap_targets = Vec::new(); + + if !track_rmap { + open_skel + .progs + .fentry_folio_add_new_anon_rmap + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_anon_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_anon_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_pud + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_pud + .set_autoload(false); + } else { + let btf = libbpf_rs::btf::Btf::from_vmlinux().context("Failed to load vmlinux BTF")?; + macro_rules! disable_missing_kernel_func { + ($prog:ident, $target:literal) => { + if !kernel_func_exists(&btf, $target) { + open_skel.progs.$prog.set_autoload(false); + btf_disabled_rmap_targets.push($target); + warn!( + "Kernel function {} not present in BTF, disabling rmap fentry", + $target + ); + } + }; + } + + disable_missing_kernel_func!(fentry_folio_add_new_anon_rmap, "folio_add_new_anon_rmap"); + disable_missing_kernel_func!( + fentry_folio_add_anon_rmap_ptes, + "folio_add_anon_rmap_ptes" + ); + disable_missing_kernel_func!(fentry_folio_add_anon_rmap_pmd, "folio_add_anon_rmap_pmd"); + disable_missing_kernel_func!( + fentry_folio_add_file_rmap_ptes, + "folio_add_file_rmap_ptes" + ); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_pmd, "folio_add_file_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_pud, "folio_add_file_rmap_pud"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_ptes, "folio_remove_rmap_ptes"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_pmd, "folio_remove_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_pud, "folio_remove_rmap_pud"); + } + let skel = Box::new( open_skel .load() @@ -95,6 +201,8 @@ impl MemtrackBpf { Ok(Self { skel, probes: Vec::new(), + track_rmap, + btf_disabled_rmap_targets, }) } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index dd2a5604..38e11a93 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -3,10 +3,87 @@ use crate::prelude::*; use paste::paste; impl MemtrackBpf { - attach_tracepoint!(sched_fork); + attach_tracepoint!(rss_stat); + attach_tracepoint!(task_newtask); + attach_tracepoint!(sched_process_exec); + attach_tracepoint!(sched_process_exit); + attach_fentry!( + attach_fentry_folio_add_new_anon_rmap, + fentry_folio_add_new_anon_rmap + ); + attach_fentry!( + attach_fentry_folio_add_anon_rmap_ptes, + fentry_folio_add_anon_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_add_anon_rmap_pmd, + fentry_folio_add_anon_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_ptes, + fentry_folio_add_file_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_pmd, + fentry_folio_add_file_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_pud, + fentry_folio_add_file_rmap_pud + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_ptes, + fentry_folio_remove_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_pmd, + fentry_folio_remove_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_pud, + fentry_folio_remove_rmap_pud + ); + + fn rmap_target_enabled(&self, target: &str) -> bool { + !self.btf_disabled_rmap_targets.contains(&target) + } pub fn attach_tracepoints(&mut self) -> Result<()> { - self.attach_sched_fork()?; + self.attach_task_newtask()?; + self.attach_sched_process_exec()?; + self.attach_sched_process_exit()?; + if let Err(e) = self.attach_rss_stat() { + warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); + } + if self.track_rmap { + if self.rmap_target_enabled("folio_add_new_anon_rmap") { + self.attach_fentry_folio_add_new_anon_rmap()?; + } + if self.rmap_target_enabled("folio_add_anon_rmap_ptes") { + self.attach_fentry_folio_add_anon_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_add_anon_rmap_pmd") { + self.attach_fentry_folio_add_anon_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_ptes") { + self.attach_fentry_folio_add_file_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_pmd") { + self.attach_fentry_folio_add_file_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_pud") { + self.attach_fentry_folio_add_file_rmap_pud()?; + } + if self.rmap_target_enabled("folio_remove_rmap_ptes") { + self.attach_fentry_folio_remove_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_remove_rmap_pmd") { + self.attach_fentry_folio_remove_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_remove_rmap_pud") { + self.attach_fentry_folio_remove_rmap_pud()?; + } + } Ok(()) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 11ecaf04..83b34853 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -33,10 +33,26 @@ impl Tracker { }) } + /// Track per-process RSS via the rss_stat tracepoint and folio rmap fentry + /// hooks without attaching any allocator probes. No exec-mapping watcher + /// runs, so `spawn` arms no on-demand allocator attachment — the tracker + /// observes only tracepoints and (when `track_rmap`) the rmap fentries. + pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { + Self::bump_memlock_rlimit()?; + + let mut bpf = MemtrackBpf::new_with_rmap(track_rmap)?; + bpf.attach_tracepoints()?; + + Ok(Self { + bpf: Arc::new(Mutex::new(bpf)), + worker: Mutex::new(None), + }) + } + /// Spawn `cmd` under tracking: the target is wrapped so it stops itself - /// before exec'ing, its pid is armed while stopped, then it is resumed. - /// The watcher observes the target's own `execve` mappings — no allocation - /// escapes untracked. + /// before exec'ing, its pid is armed while stopped, then it is resumed. When + /// the tracker runs an exec-mapping watcher, arming the pid before resume + /// ensures no allocation mapping escapes untracked. /// /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). @@ -48,11 +64,9 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - self.worker - .lock() - .as_ref() - .context("tracker already finished")? - .set_root_pid(pid); + if let Some(worker) = self.worker.lock().as_ref() { + worker.set_root_pid(pid); + } let (tx, rx) = mpsc::channel(); let poller = { @@ -65,12 +79,14 @@ impl Tracker { Ok(Session::new(child, rx, poller)) } - /// Enable event tracking in the BPF program + /// Enable allocator-event tracking in the BPF program. Lifetime events + /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids + /// regardless of this toggle. pub fn enable_tracking(&self) -> Result<()> { self.bpf.lock().enable_tracking() } - /// Disable event tracking in the BPF program + /// Disable allocator-event tracking in the BPF program pub fn disable_tracking(&self) -> Result<()> { self.bpf.lock().disable_tracking() } @@ -81,15 +97,14 @@ impl Tracker { self.bpf.lock().dropped_events_count() } - /// Stop the attach worker and surface any fatal error it recorded, - /// including missed exec mappings (incomplete allocator coverage). + /// Stop the attach worker, if any, and surface any fatal error it recorded, + /// including missed exec mappings (incomplete allocator coverage). A tracker + /// without an allocator watcher has no worker, so this is a no-op. pub fn finish(&self) -> Result<()> { - let worker = self - .worker - .lock() - .take() - .context("tracker already finished")?; - worker.finish() + match self.worker.lock().take() { + Some(worker) => worker.finish(), + None => Ok(()), + } } /// Detach all attached probes. Called explicitly at teardown because the diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 8ad48791..283cff19 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -95,8 +95,9 @@ fn track_command( } })) } else { - // Without IPC, nothing toggles the tracking_enabled map, so events would - // be dropped by the eBPF is_enabled() check. Enable it up front. + // Without IPC, nothing toggles the tracking_enabled map, so allocator + // events would be dropped by the eBPF is_enabled() check. Enable it up + // front. tracker.enable_tracking()?; None }; @@ -135,8 +136,8 @@ fn track_command( let status = session.wait().context("Failed to wait for command")?; debug!("Command exited with status: {status}"); - // Stop event production before draining: the child has exited, so anything - // still arriving is already in the ring buffer. + // Stop allocator-event production before draining: the child has exited, + // so anything still arriving is already in the ring buffer. if let Err(e) = tracker.disable_tracking() { warn!("Failed to disable tracking: {e:#}"); } diff --git a/crates/memtrack/testdata/rss/anon.c b/crates/memtrack/testdata/rss/anon.c new file mode 100644 index 00000000..8aa0974e --- /dev/null +++ b/crates/memtrack/testdata/rss/anon.c @@ -0,0 +1,17 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/file.c b/crates/memtrack/testdata/rss/file.c new file mode 100644 index 00000000..5d45c293 --- /dev/null +++ b/crates/memtrack/testdata/rss/file.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + char template[] = "/tmp/memtrack-rss-file-XXXXXX"; + int fd = mkstemp(template); + if (fd < 0) return 1; + unlink(template); + char chunk[65536]; + memset(chunk, 0x42, sizeof(chunk)); + for (size_t off = 0; off < len; off += sizeof(chunk)) { + if (write(fd, chunk, sizeof(chunk)) != (ssize_t)sizeof(chunk)) return 1; + } + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret + (sink & 0); +} diff --git a/crates/memtrack/testdata/rss/fork.c b/crates/memtrack/testdata/rss/fork.c new file mode 100644 index 00000000..f6fcfac0 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +static int append_rss(const char* path, const char* label) { + long kb = rss_status_kb("RssAnon:"); + if (kb < 0) return 1; + FILE* report = fopen(path, "a"); + if (!report) return 1; + fprintf(report, "%s: %ld\n", label, kb); + fclose(report); + return 0; +} + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* parent = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (parent == MAP_FAILED) return 1; + memset(parent, 0x42, len); + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + void* child = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (child == MAP_FAILED) _exit(1); + memset(child, 0x24, len); + _exit(append_rss(argv[1], "ChildRssAnonKb")); + } + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + int ret = append_rss(argv[1], "ParentRssAnonKb"); + munmap(parent, len); + return ret || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/fork_idle.c b/crates/memtrack/testdata/rss/fork_idle.c new file mode 100644 index 00000000..0c5e5643 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork_idle.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The child only touches memory inherited from the parent: COW faults of anon + * pages are counter-neutral, so the child never reports its own RSS. Its + * footprint is only observable through fork-event seeding. The child must not + * allocate (no stdio/malloc), so the parent samples /proc//status while + * the child blocks on a pipe. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + + int ready[2]; + int release[2]; + if (pipe(ready) || pipe(release)) return 1; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + memset(mem, 0x24, len); + char b = 1; + if (write(ready[1], &b, 1) != 1) _exit(1); + if (read(release[0], &b, 1) != 1) _exit(1); + _exit(0); + } + + char b; + if (read(ready[0], &b, 1) != 1) return 1; + long child_anon = rss_status_kb_pid(pid, "RssAnon:"); + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ChildRssAnon: %ld\n", child_anon); + fclose(report); + if (write(release[1], &b, 1) != 1) return 1; + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + return ret || child_anon < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/madvise.c b/crates/memtrack/testdata/rss/madvise.c new file mode 100644 index 00000000..e5f77972 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise.c @@ -0,0 +1,64 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* AnonHugePages for the whole process, from smaps_rollup (not in /proc/status). */ +static long anon_huge_pages_kb(void) { + FILE* rollup = fopen("/proc/self/smaps_rollup", "r"); + if (!rollup) return 0; + char line[256]; + long kb = 0; + while (fgets(line, sizeof(line), rollup)) { + if (sscanf(line, "AnonHugePages: %ld", &kb) == 1) break; + } + fclose(rollup); + return kb; +} + +/* Two 32 MiB anon regions: one advised MADV_HUGEPAGE (2 MiB-aligned so the + * kernel *may* fault it as PMD folios), one MADV_NOHUGEPAGE (guaranteed pte + * path). MADV_HUGEPAGE is only advisory, so nothing here depends on THP + * actually materializing: the accounted totals are identical either way. + * + * Both regions are dropped with MADV_DONTNEED and faulted a second time. If + * the in-context removes were missed, the reconstructed running total reaches + * 128 MiB instead of 64 MiB, so the snapshot peak is the assertion. The report + * also carries the observed AnonHugePages (hex, so it stays out of the + * snapshot) letting the test require PMD-sized rmap deltas exactly when THP + * actually materialized. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + const size_t align = 2UL * 1024 * 1024; + + void* raw = mmap(NULL, len + align, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) return 1; + char* huge = (char*)(((uintptr_t)raw + align - 1) & ~(uintptr_t)(align - 1)); + if (madvise(huge, len, MADV_HUGEPAGE) != 0) return 1; + + char* base = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) return 1; + if (madvise(base, len, MADV_NOHUGEPAGE) != 0) return 1; + + memset(huge, 0x42, len); + memset(base, 0x42, len); + + if (madvise(huge, len, MADV_DONTNEED) != 0) return 1; + if (madvise(base, len, MADV_DONTNEED) != 0) return 1; + + memset(huge, 0x43, len); + memset(base, 0x43, len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ThpKb: 0x%lx\n", anon_huge_pages_kb()); + fclose(report); + munmap(raw, len + align); + munmap(base, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/madvise_extern.c b/crates/memtrack/testdata/rss/madvise_extern.c new file mode 100644 index 00000000..4be4c6b2 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise_extern.c @@ -0,0 +1,47 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + (void)argc; (void)argv; + sleep(1); /* let the tracker attach + enable + add root pid */ + + size_t len = 64UL * 1024 * 1024; + + char path[] = "/tmp/memtrack_madv_XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) return 1; + unlink(path); + if (ftruncate(fd, len) != 0) return 1; + + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + + /* Fault the file pages into A's RSS (MM_FILEPAGES), in-context -> seeds ownership. */ + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + (void)sink; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + /* B: page out A's (parent's) region from B's context. */ + int pidfd = syscall(SYS_pidfd_open, getppid(), 0); + if (pidfd < 0) _exit(1); + struct iovec iov = {.iov_base = mem, .iov_len = len}; + syscall(SYS_process_madvise, pidfd, &iov, 1UL, MADV_PAGEOUT, 0UL); + _exit(0); + } + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + sleep(1); /* let the external decrement flush to the ring buffer */ + /* No munmap: an in-context decrement would mask the external-path signal. */ + return 0; +} diff --git a/crates/memtrack/testdata/rss/mremap_move.c b/crates/memtrack/testdata/rss/mremap_move.c new file mode 100644 index 00000000..3b86fc03 --- /dev/null +++ b/crates/memtrack/testdata/rss/mremap_move.c @@ -0,0 +1,31 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +/* Fault 32 MiB, then force mremap to move it to a reserved destination. The + * move relocates page tables without rmap remove/add, so no events should + * fire and the second memset must not refault: a peak of 64 MiB instead of + * 32 MiB means the move was double-counted or the pages were dropped. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + + char* src = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (src == MAP_FAILED) return 1; + memset(src, 0x42, len); + + void* reserved = mmap(NULL, len, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) return 1; + + char* dst = mremap(src, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, reserved); + if (dst == MAP_FAILED) return 1; + memset(dst, 0x43, len); + + int ret = write_rss_report(argv[1]); + munmap(dst, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/munmap_hole.c b/crates/memtrack/testdata/rss/munmap_hole.c new file mode 100644 index 00000000..e769ff5b --- /dev/null +++ b/crates/memtrack/testdata/rss/munmap_hole.c @@ -0,0 +1,40 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Fault 64 MiB, punch a 16 MiB hole with munmap, then map and fault the hole + * again. If the partial-unmap removes were missed (or covered the wrong + * range), the reconstructed running total peaks at 80 MiB instead of 64 MiB. + * MADV_NOHUGEPAGE keeps every folio a single page: the region is only + * page-aligned, so a straddling PMD folio would blur the hole boundaries. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 64UL * 1024 * 1024; + const size_t hole_off = 24UL * 1024 * 1024; + const size_t hole_len = 16UL * 1024 * 1024; + + char* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + if (madvise(mem, len, MADV_NOHUGEPAGE) != 0) return 1; + memset(mem, 0x42, len); + + if (munmap(mem + hole_off, hole_len) != 0) return 1; + + void* refill = mmap(mem + hole_off, hole_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (refill == MAP_FAILED) return 1; + if (madvise(refill, hole_len, MADV_NOHUGEPAGE) != 0) return 1; + memset(refill, 0x43, hole_len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "Layout: 0x%lx 0x%lx 0x%lx 0x%lx\n", (unsigned long)mem, hole_off, hole_len, + len); + fclose(report); + return ret; +} diff --git a/crates/memtrack/testdata/rss/rmap_late_enable.c b/crates/memtrack/testdata/rss/rmap_late_enable.c new file mode 100644 index 00000000..0bdae37b --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_late_enable.c @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Two-phase fixture that faults anonymous memory before and after a handshake: + * + * 1. Fault a baseline region, then signal `ready` and block on `go`. + * 2. After `go` appears, fault a second region of the same (anon) member. + * + * The handshake lets the caller act between the two phases (e.g. start + * observing only phase 2). Both regions touch MM_ANONPAGES, so an absolute + * counter read after phase 2 covers baseline + growth, while a delta observed + * only from phase 2 covers growth alone. + * + * argv: [1]=report path, [2]=ready path, [3]=go path. */ +static void touch(const char* path) { + FILE* f = fopen(path, "w"); + if (f) { + fclose(f); + } +} + +int main(int argc, char** argv) { + if (argc != 4) { + return 1; + } + const char* report_path = argv[1]; + const char* ready_path = argv[2]; + const char* go_path = argv[3]; + + size_t region = 64UL * 1024 * 1024; + + void* baseline = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (baseline == MAP_FAILED) { + return 1; + } + memset(baseline, 0x42, region); + + touch(ready_path); + while (access(go_path, F_OK) != 0) { + usleep(1000); + } + + void* growth = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (growth == MAP_FAILED) { + return 1; + } + memset(growth, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(baseline, region); + munmap(growth, region); + return ret; +} diff --git a/crates/memtrack/testdata/rss/rmap_leader_exit.c b/crates/memtrack/testdata/rss/rmap_leader_exit.c new file mode 100644 index 00000000..51e7b23c --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_leader_exit.c @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The main thread publishes its tid and leaves via pthread_exit while a worker + * thread keeps the process alive. The worker waits until the leader is a + * zombie — its exit path, including the sched_process_exit tracepoint, has + * fully run — then faults an anon region, writes the report, and exits the + * whole group. A tracker keyed on leader exit instead of thread-group death + * would stop watching before the worker's region is faulted. + * + * argv: [1]=report path. */ + +static const char* report_path; +static pid_t leader_tid; + +static int leader_is_zombie(void) { + char path[64]; + char buf[256]; + snprintf(path, sizeof(path), "/proc/self/task/%d/stat", leader_tid); + FILE* f = fopen(path, "r"); + if (!f) { + return 1; + } + size_t n = fread(buf, 1, sizeof(buf) - 1, f); + fclose(f); + buf[n] = '\0'; + /* The state field follows the parenthesized comm. */ + const char* p = strrchr(buf, ')'); + if (!p || p[1] == '\0' || p[2] == '\0') { + return 0; + } + return p[2] == 'Z'; +} + +static void* worker(void* arg) { + (void)arg; + while (!leader_is_zombie()) { + usleep(1000); + } + + size_t region = 64UL * 1024 * 1024; + void* mem = + mmap(NULL, region, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + /* The leader is a zombie by now, so /proc//status has no Rss lines; + * report through this worker's own task instead. */ + int ret = write_rss_report_pid((int)syscall(SYS_gettid), report_path); + munmap(mem, region); + _exit(ret); +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + leader_tid = (pid_t)syscall(SYS_gettid); + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + pthread_exit(NULL); +} diff --git a/crates/memtrack/testdata/rss/rmap_thread_fork.c b/crates/memtrack/testdata/rss/rmap_thread_fork.c new file mode 100644 index 00000000..831fd110 --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_thread_fork.c @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* A worker thread (not the group leader) calls fork(); the child faults an + * anon region and writes the report. Child tracking must key on the parent's + * TGID: a scheme keyed on the raw creator tid would only cover forks issued + * by the leader. + * + * argv: [1]=report path. */ + +static const char* report_path; + +static void* worker(void* arg) { + (void)arg; + + pid_t pid = fork(); + if (pid < 0) { + _exit(1); + } + if (pid == 0) { + size_t region = 64UL * 1024 * 1024; + void* mem = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(mem, region); + _exit(ret); + } + + int status; + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + _exit(1); + } + return NULL; +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + if (pthread_join(thread, NULL) != 0) { + return 1; + } + return 0; +} diff --git a/crates/memtrack/testdata/rss/rss_report.h b/crates/memtrack/testdata/rss/rss_report.h new file mode 100644 index 00000000..169bdc07 --- /dev/null +++ b/crates/memtrack/testdata/rss/rss_report.h @@ -0,0 +1,58 @@ +#ifndef RSS_REPORT_H +#define RSS_REPORT_H + +#include +#include +#include + +static long rss_status_kb_pid(int pid, const char* key) { + char status_path[64]; + snprintf(status_path, sizeof(status_path), "/proc/%d/status", pid); + FILE* status = fopen(status_path, "r"); + if (!status) return -1; + char line[256]; + long kb = -1; + size_t key_len = strlen(key); + while (fgets(line, sizeof(line), status)) { + if (strncmp(line, key, key_len) == 0 && sscanf(line + key_len, " %ld", &kb) == 1) { + break; + } + } + fclose(status); + return kb; +} + +static long rss_status_kb(const char* key) { + return rss_status_kb_pid(getpid(), key); +} + +/* Reads Rss* through /proc//status of the given task. For the + * thread-group dir this is the leader's task: once the leader is a zombie its + * mm pointer is gone and the Rss lines disappear, so callers whose leader has + * exited must pass a live thread's tid instead. */ +static int write_rss_report_pid(int pid, const char* path) { + long anon = rss_status_kb_pid(pid, "RssAnon:"); + long file = rss_status_kb_pid(pid, "RssFile:"); + long shmem = rss_status_kb_pid(pid, "RssShmem:"); + /* VmHWM instead of getrusage(): ru_maxrss includes signal->maxrss, which + * survives execve and so reports the peak of the pre-exec parent image. + * VmHWM belongs to the mm and starts fresh at exec. */ + long max_rss = rss_status_kb_pid(pid, "VmHWM:"); + if (anon < 0 || file < 0 || shmem < 0 || max_rss < 0) { + return 1; + } + FILE* report = fopen(path, "w"); + if (!report) { + return 1; + } + fprintf(report, "RssAnon: %ld\nRssFile: %ld\nRssShmem: %ld\nMaxRssKb: %ld\n", anon, file, + shmem, max_rss); + fclose(report); + return 0; +} + +static int write_rss_report(const char* path) { + return write_rss_report_pid(getpid(), path); +} + +#endif diff --git a/crates/memtrack/testdata/rss/shmem.c b/crates/memtrack/testdata/rss/shmem.c new file mode 100644 index 00000000..63b53e01 --- /dev/null +++ b/crates/memtrack/testdata/rss/shmem.c @@ -0,0 +1,22 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + int fd = memfd_create("memtrack-rss-shmem", 0); + if (fd < 0) return 1; + if (ftruncate(fd, len) != 0) return 1; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret; +} diff --git a/crates/memtrack/testdata/rss/triangle.c b/crates/memtrack/testdata/rss/triangle.c new file mode 100644 index 00000000..69bf7bf6 --- /dev/null +++ b/crates/memtrack/testdata/rss/triangle.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t chunk = 16UL * 1024 * 1024; + void* bufs[4]; + for (int i = 0; i < 4; i++) { + bufs[i] = mmap(NULL, chunk, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bufs[i] == MAP_FAILED) return 1; + memset(bufs[i], 0x42, chunk); + usleep(50 * 1000); + } + int ret = write_rss_report(argv[1]); + for (int i = 0; i < 4; i++) { + munmap(bufs[i], chunk); + usleep(50 * 1000); + } + return ret; +} diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs new file mode 100644 index 00000000..be834fd2 --- /dev/null +++ b/crates/memtrack/tests/rss_tests.rs @@ -0,0 +1,669 @@ +#[macro_use] +mod shared; + +use itertools::Itertools; +use rstest::rstest; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::process::Command; +use tempfile::TempDir; + +const MIB: u64 = 1024 * 1024; + +fn mib_16(bytes: u64) -> u64 { + (bytes + 8 * MIB) / (16 * MIB) * 16 +} + +fn parse_report(report: &str) -> BTreeMap { + report + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let key = parts.next()?.to_string(); + let kb: u64 = parts.next()?.parse().ok()?; + Some((key, mib_16(kb * 1024))) + }) + .collect() +} + +#[derive(Debug, Serialize)] +struct PidRss { + pid: i32, + file_mib: u64, + anon_mib: u64, + shmem_mib: u64, + max_rss_mib: u64, +} + +#[derive(Serialize)] +struct RssSummary { + report: BTreeMap, + rss_stat: Vec, + rmap: Vec, +} + +#[derive(Default)] +struct RssAccum { + latest: [u64; 4], + peaks: [u64; 4], + max_rss: u64, +} + +impl RssAccum { + fn update_peaks(&mut self) { + for (peak, latest) in self.peaks.iter_mut().zip(self.latest) { + *peak = (*peak).max(latest); + } + self.max_rss = self + .max_rss + .max(self.latest[0] + self.latest[1] + self.latest[3]); + } +} + +#[derive(Default)] +struct RmapAccum { + totals: [i64; 4], + peaks: [i64; 4], + max_rss: i64, +} + +impl RmapAccum { + fn update_peaks(&mut self) { + for (peak, total) in self.peaks.iter_mut().zip(self.totals) { + *peak = (*peak).max(total); + } + self.max_rss = self + .max_rss + .max(self.totals[0] + self.totals[1] + self.totals[3]); + } +} + +fn per_pid_peaks(events: &[MemtrackEvent]) -> (Vec, Vec) { + // Pid values can wrap, so numeric order is not stable; both views emit + // their rows in one shared first-activity order, keeping the relative + // order of processes consistent between `rss_stat` and `rmap` after + // the pids are redacted. + let mut order: Vec = Vec::new(); + let mut rss: BTreeMap = BTreeMap::new(); + let mut rmap: BTreeMap = BTreeMap::new(); + + fn seen(order: &mut Vec, pid: i32) { + if !order.contains(&pid) { + order.push(pid); + } + } + + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + let acc = rss.entry(event.pid).or_default(); + acc.latest[index] = size; + acc.update_peaks(); + } + MemtrackEventKind::Rmap { member, delta } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + let acc = rmap.entry(event.pid).or_default(); + acc.totals[index] += delta * 4096; + acc.update_peaks(); + } + MemtrackEventKind::Fork { parent_pid } => { + seen(&mut order, event.pid); + + let seed = rss.get(&parent_pid).map(|p| p.latest).unwrap_or_default(); + let acc = rss.entry(event.pid).or_default(); + acc.latest = seed; + acc.update_peaks(); + + let seed = rmap.get(&parent_pid).map(|p| p.totals).unwrap_or_default(); + let acc = rmap.entry(event.pid).or_default(); + acc.totals = seed; + acc.update_peaks(); + } + MemtrackEventKind::Exec | MemtrackEventKind::Exit => { + if let Some(acc) = rss.get_mut(&event.pid) { + acc.latest = [0; 4]; + } + if let Some(acc) = rmap.get_mut(&event.pid) { + acc.totals = [0; 4]; + } + } + _ => {} + } + } + + let rss_stat = order + .iter() + .filter_map(|pid| { + let acc = rss.remove(pid)?; + Some(PidRss { + pid: *pid, + file_mib: mib_16(acc.peaks[0]), + anon_mib: mib_16(acc.peaks[1]), + shmem_mib: mib_16(acc.peaks[3]), + max_rss_mib: mib_16(acc.max_rss), + }) + }) + .collect(); + let rmap = order + .iter() + .filter_map(|pid| { + let acc = rmap.remove(pid)?; + Some(PidRss { + pid: *pid, + file_mib: mib_16(acc.peaks[0].max(0) as u64), + anon_mib: mib_16(acc.peaks[1].max(0) as u64), + shmem_mib: mib_16(acc.peaks[3].max(0) as u64), + max_rss_mib: mib_16(acc.max_rss.max(0) as u64), + }) + }) + .collect(); + (rss_stat, rmap) +} + +/// Compile a fixture that writes a `/proc` RSS report to its argv[1], run it under +/// `track`, and return the raw report text alongside the collected events. +fn track_fixture( + source: &str, + name: &str, + track: impl FnOnce(Command) -> shared::TrackResult, +) -> Result<(String, Vec), Box> { + let temp_dir = TempDir::new()?; + std::fs::write( + temp_dir.path().join("rss_report.h"), + include_str!("../testdata/rss/rss_report.h"), + )?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let report_path = temp_dir.path().join(format!("{name}.report")); + let mut command = Command::new(&binary); + command.arg(&report_path); + + let (events, thread_handle) = track(command)?; + let raw_report = std::fs::read_to_string(&report_path)?; + thread_handle.join().unwrap(); + Ok((raw_report, events)) +} + +/// Pins reconstructed rmap addresses to the exact punched range: hole pages are +/// the only ones removed and later re-added; every other page in the region is +/// added first and only removed afterwards. +/// +/// Only own-context events (tid == pid; the fixture is single-threaded) are +/// considered: foreign actors like kcompactd migrating a page produce a +/// remove-then-add on arbitrary pages, which the attribution of foreign rmap +/// events makes visible here. +fn assert_rmap_hole_addresses( + events: &[MemtrackEvent], + base: u64, + hole_off: u64, + hole_len: u64, + len: u64, +) { + const PAGE: u64 = 4096; + let n_pages = (len / PAGE) as usize; + let mut first_remove = vec![u64::MAX; n_pages]; + let mut added = vec![false; n_pages]; + let mut readded = vec![false; n_pages]; + + for event in events.iter().sorted_by_key(|e| e.timestamp) { + let MemtrackEventKind::Rmap { delta, .. } = event.kind else { + continue; + }; + if event.tid != event.pid { + continue; + } + if event.addr < base || event.addr >= base + len { + continue; + } + let first = ((event.addr - base) / PAGE) as usize; + let last = (first + delta.unsigned_abs() as usize).min(n_pages); + for page in first..last { + if delta > 0 { + added[page] = true; + if event.timestamp > first_remove[page] { + readded[page] = true; + } + } else { + first_remove[page] = first_remove[page].min(event.timestamp); + } + } + } + + let hole = (hole_off / PAGE) as usize..((hole_off + hole_len) / PAGE) as usize; + for page in 0..n_pages { + assert!(added[page], "page {page} never saw an rmap add"); + assert_eq!( + readded[page], + hole.contains(&page), + "page {page}: remove-then-add pattern does not match the hole range" + ); + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::anon(include_str!("../testdata/rss/anon.c"), "anon")] +#[case::file(include_str!("../testdata/rss/file.c"), "file")] +#[case::shmem(include_str!("../testdata/rss/shmem.c"), "shmem")] +#[case::fork(include_str!("../testdata/rss/fork.c"), "fork")] +#[case::fork_idle(include_str!("../testdata/rss/fork_idle.c"), "fork_idle")] +#[case::triangle(include_str!("../testdata/rss/triangle.c"), "triangle")] +#[case::madvise(include_str!("../testdata/rss/madvise.c"), "madvise")] +#[case::munmap_hole(include_str!("../testdata/rss/munmap_hole.c"), "munmap_hole")] +#[case::mremap_move(include_str!("../testdata/rss/mremap_move.c"), "mremap_move")] +fn test_rss_rmap_tracking( + #[case] source: &str, + #[case] name: &str, +) -> Result<(), Box> { + let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (rss_stat, rmap) = per_pid_peaks(&events); + let summary = RssSummary { + report: parse_report(&raw_report), + rss_stat, + rmap, + }; + insta::assert_json_snapshot!(format!("rss_{name}"), summary, { + ".rss_stat[].pid" => "[pid]", + ".rmap[].pid" => "[pid]", + }); + + if let Some(layout) = raw_report + .lines() + .find_map(|line| line.strip_prefix("Layout:")) + { + let values: Vec = layout + .split_whitespace() + .map(|token| u64::from_str_radix(token.trim_start_matches("0x"), 16)) + .collect::>()?; + let [base, hole_off, hole_len, len] = values[..] else { + panic!("malformed Layout line: {layout}"); + }; + assert_rmap_hole_addresses(&events, base, hole_off, hole_len, len); + } + + // ThpKb > 0 means the MADV_HUGEPAGE region really faulted as PMD folios, so + // huge-folio accounting must be visible: a +512-page delta from the new-anon + // fault path and a -512-page delta that can only come from the + // folio_remove_rmap_pmd hook (MADV_DONTNEED / munmap of a pmd-mapped THP). + if let Some(thp) = raw_report + .lines() + .find_map(|line| line.strip_prefix("ThpKb:")) + { + let thp_kb = u64::from_str_radix(thp.trim().trim_start_matches("0x"), 16)?; + if thp_kb > 0 { + let deltas = events.iter().filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { delta, .. } => Some(delta), + _ => None, + }); + let (mut huge_add, mut huge_remove) = (false, false); + for delta in deltas { + huge_add |= delta >= 512; + huge_remove |= delta <= -512; + } + assert!( + huge_add, + "THP present ({thp_kb} kB) but no huge-folio rmap add" + ); + assert!( + huge_remove, + "THP present ({thp_kb} kB) but no pmd-sized rmap remove" + ); + } + } + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_external_reclaim() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + temp_dir.path(), + )?; + let (events, handle) = shared::track_command(Command::new(&binary))?; + handle.join().unwrap(); + + // Single fork: parent_pid == A (owner), event.pid == B (external caller, single-threaded + // so its tid == its pid). + let (a, b) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .expect("expected a fork event"); + + let peak = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rss { member: 0, size } if e.pid == a => Some(size), + _ => None, + }) + .max() + .unwrap_or(0); + assert!(peak >= 32 * MIB, "peak file RSS too small: {peak}"); + + // A file decrement owned by A but emitted from B's context: only present when + // out-of-context rss_stat updates are attributed to the owning process. + let external_decrement = events.iter().any(|e| { + e.pid == a + && e.tid == b + && matches!(e.kind, MemtrackEventKind::Rss { member: 0, size } if size < peak) + }); + assert!( + external_decrement, + "external file-RSS decrement not attributed to A (tid=B)" + ); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_external_reclaim() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + temp_dir.path(), + )?; + let (events, handle) = shared::track_command_with_rmap(Command::new(&binary))?; + handle.join().unwrap(); + + // Single fork: parent_pid == A (owner that faulted the file region), event.pid == B + // (external caller of process_madvise(MADV_PAGEOUT) against A from its own context). + let (a, b) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .expect("expected a fork event"); + + // Sanity: A's own in-context file-page faults are reconstructed by rmap. + let in_context_add: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } if e.pid == a && delta > 0 => Some(delta), + _ => None, + }) + .sum(); + let in_context_bytes = in_context_add as u64 * 4096; + assert!( + in_context_bytes >= 32 * MIB, + "in-context file rmap adds too small: {in_context_bytes}" + ); + + // The point of the test: a file-page remove owned by A but emitted from B's + // context (tid == B), only present when the foreign reclaim's rmap events are + // attributed to the owning process via the mm_owner map. + let external_removed: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } + if e.pid == a && e.tid == b && delta < 0 => + { + Some(-delta) + } + _ => None, + }) + .sum(); + assert!( + external_removed > 0, + "external MADV_PAGEOUT remove not attributed to the owner (pid=A, tid=B)" + ); + let external_bytes = external_removed as u64 * 4096; + assert!( + external_bytes >= 8 * MIB, + "external MADV_PAGEOUT remove not attributed to the owner: only {external_bytes} bytes" + ); + Ok(()) +} + +/// TEMPORARY diagnostic: track a real-world workload (`ls /nix/store`, ~50 MiB +/// peak on a populated store) and cross-check the reconstructed rss_stat and +/// rmap peaks against the kernel's own accounting (`wait4` ru_maxrss) from an +/// identical untracked run. `ls` is single-process, so raw byte peaks are +/// accumulated directly without per-pid splitting. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_ls_nix_store() -> Result<(), Box> { + if !std::path::Path::new("/nix/store").is_dir() { + eprintln!("skipping: /nix/store not available"); + return Ok(()); + } + + let ls_command = || { + let mut cmd = Command::new("ls"); + cmd.arg("/nix/store").stdout(std::process::Stdio::null()); + cmd + }; + + // Ground truth: identical untracked run, reaped via wait4 for ru_maxrss. + let child = ls_command().spawn()?; + let pid = child.id() as i32; + let mut status = 0i32; + let mut rusage: libc::rusage = unsafe { std::mem::zeroed() }; + let reaped = unsafe { libc::wait4(pid, &mut status, 0, &mut rusage) }; + assert_eq!(reaped, pid, "wait4 failed"); + assert!( + libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, + "untracked ls failed: status {status}" + ); + let truth_bytes = rusage.ru_maxrss as u64 * 1024; + + let (events, handle) = shared::track_command_with_rmap(ls_command())?; + handle.join().unwrap(); + + let mut rss = RssAccum::default(); + let mut rmap = RmapAccum::default(); + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + if let Ok(index @ 0..4) = usize::try_from(member) { + rss.latest[index] = size; + rss.update_peaks(); + } + } + MemtrackEventKind::Rmap { member, delta } => { + if let Ok(index @ 0..4) = usize::try_from(member) { + rmap.totals[index] += delta * 4096; + rmap.update_peaks(); + } + } + _ => {} + } + } + + let rss_bytes = rss.max_rss; + let rmap_bytes = rmap.max_rss.max(0) as u64; + eprintln!( + "ls /nix/store max RSS: wait4={:.1} MiB rss_stat={:.1} MiB rmap={:.1} MiB", + truth_bytes as f64 / MIB as f64, + rss_bytes as f64 / MIB as f64, + rmap_bytes as f64 / MIB as f64, + ); + + let within = |measured: u64| { + (truth_bytes as f64 * 0.8..=truth_bytes as f64 * 1.2).contains(&(measured as f64)) + }; + assert!( + within(rss_bytes), + "rss_stat peak {rss_bytes} outside 20% of wait4 {truth_bytes}" + ); + assert!( + within(rmap_bytes), + "rmap peak {rmap_bytes} outside 20% of wait4 {truth_bytes}" + ); + Ok(()) +} + +/// rss_stat is an absolute kernel counter; the rmap estimate is reconstructed +/// from zero by summing folio add/remove deltas. Both are emitted for the +/// whole lifetime of a tracked pid, independent of the enable toggle (which +/// only gates allocator events): a delta stream that starts mid-life could +/// never recover the resident baseline faulted before enable. +/// +/// The fixture faults a 64 MiB anon baseline before `enable_tracking`, then a +/// 64 MiB anon region after. Reduced with the same Exec-reset lifecycle the +/// production parser uses, both series peak at ~128 MiB: the pre-enable +/// baseline is visible to rmap because the pid is tracked from spawn. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_late_enable_covers_baseline() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let temp_dir = TempDir::new()?; + std::fs::write( + temp_dir.path().join("rss_report.h"), + include_str!("../testdata/rss/rss_report.h"), + )?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/rmap_late_enable.c"), + "rmap_late_enable", + temp_dir.path(), + )?; + let report_path = temp_dir.path().join("rmap_late_enable.report"); + let ready_path = temp_dir.path().join("ready"); + let go_path = temp_dir.path().join("go"); + + let mut command = Command::new(&binary); + command.arg(&report_path).arg(&ready_path).arg(&go_path); + + let (events, handle) = + shared::track_command_with_rmap_late_enable(command, &ready_path, &go_path)?; + handle.join().unwrap(); + + let (rss_stat, rmap) = per_pid_peaks(&events); + let rss = rss_stat.first().ok_or("no rss_stat pid observed")?; + let rmap = rmap.first().ok_or("no rmap pid observed")?; + eprintln!( + "late-enable anon peaks: rss_stat={} MiB rmap={} MiB (region={} MiB each)", + rss.anon_mib, rmap.anon_mib, REGION_MIB + ); + + // rss_stat's absolute counter covers baseline + growth. + assert!( + rss.anon_mib >= 2 * REGION_MIB - 16, + "rss_stat anon peak {} MiB below the expected ~{} MiB baseline+growth", + rss.anon_mib, + 2 * REGION_MIB + ); + // The rmap accumulator covers the pre-enable baseline too: lifetime + // events are gated on is_tracked, not is_enabled. + assert!( + rmap.anon_mib >= 2 * REGION_MIB - 16, + "rmap anon peak {} MiB misses the pre-enable baseline; expected ~{} MiB", + rmap.anon_mib, + 2 * REGION_MIB + ); + let gap = rss.anon_mib.abs_diff(rmap.anon_mib); + assert!( + gap <= 16, + "rss_stat ({} MiB) and rmap ({} MiB) diverged by {} MiB despite lifetime tracking", + rss.anon_mib, + rmap.anon_mib, + gap + ); + Ok(()) +} + +/// The leader thread can pthread_exit while worker threads keep the process +/// alive; EXIT must mark thread-group death, not leader exit. The fixture's +/// worker faults a 64 MiB anon region only after the leader is a zombie, so +/// an exit path keyed on the leader would untrack the pid before the region +/// is faulted and emit EXIT ahead of the worker's rmap events. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_leader_exit_keeps_tracking() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_leader_exit.c"), + "rmap_leader_exit", + shared::track_command_with_rmap, + )?; + + // The fixture process is the pid with the largest anon rmap peak. + let (_rss_stat, rmap) = per_pid_peaks(&events); + let rmap_peak = rmap + .iter() + .max_by_key(|p| p.anon_mib) + .ok_or("no rmap pid observed")?; + assert!( + rmap_peak.anon_mib >= REGION_MIB - 16, + "rmap anon peak {} MiB misses the worker's post-leader-exit region (~{} MiB)", + rmap_peak.anon_mib, + REGION_MIB + ); + + let pid = rmap_peak.pid; + let exits: Vec<&MemtrackEvent> = events + .iter() + .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Exit)) + .collect(); + assert_eq!(exits.len(), 1, "expected exactly one EXIT for pid {pid}"); + + let last_rmap_ts = events + .iter() + .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Rmap { .. })) + .map(|e| e.timestamp) + .max() + .ok_or("no rmap events for fixture pid")?; + assert!( + exits[0].timestamp > last_rmap_ts, + "EXIT fired before the worker's rmap events: leader exit was treated as process death" + ); + Ok(()) +} + +/// A fork issued by a worker thread must still track the child: registration +/// keys on the parent's tgid (task_newtask fires in the cloning task, whose +/// pid_tgid upper half is the tgid), not on the raw creator tid. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_thread_fork.c"), + "rmap_thread_fork", + shared::track_command_with_rmap, + )?; + + // Single fork in the fixture: parent = the fixture process (tgid), child = + // the region-faulting process. + let (parent, child) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .ok_or("no fork event: worker-thread fork was not tracked")?; + assert_ne!(parent, child); + + let child_anon: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 1, delta } if e.pid == child && delta > 0 => { + Some(delta) + } + _ => None, + }) + .sum(); + assert!( + child_anon * 4096 >= ((REGION_MIB - 16) * MIB) as i64, + "child of a worker-thread fork missed rmap tracking: anon adds {} bytes, expected ~{} MiB", + child_anon * 4096, + REGION_MIB + ); + Ok(()) +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index d31bce09..9d5a040a 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -6,7 +6,7 @@ use runner_shared::artifacts::{MemtrackEvent as Event, MemtrackEventKind}; use std::path::Path; use std::process::Command; -type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; +pub type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; /// Asserts memory events using snapshot testing without marker filtering. /// @@ -27,6 +27,17 @@ macro_rules! assert_events_snapshot { // Dedup events by address and type to remove duplicates let events = $events .iter() + .filter(|e| { + // Allocation snapshots track only allocator events; RSS and + // process-lifecycle events are asserted by dedicated tests. + !matches!( + e.kind, + MemtrackEventKind::Rss { .. } + | MemtrackEventKind::Fork { .. } + | MemtrackEventKind::Exec + | MemtrackEventKind::Exit + ) + }) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .collect::>(); @@ -76,6 +87,7 @@ macro_rules! assert_events_with_marker { // Remove events outside our 0xC0D59EED marker allocations let filtered_events = $events .iter() + .filter(|e| !matches!(e.kind, MemtrackEventKind::Rss { .. })) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .skip_while(|e| { @@ -141,12 +153,46 @@ pub fn track_binary(binary: &Path) -> TrackResult { track_command(Command::new(binary)) } +pub fn compile_c_source( + source_code: &str, + name: &str, + output_dir: &Path, +) -> Result> { + let source_path = output_dir.join(format!("{name}.c")); + let binary_path = output_dir.join(name); + std::fs::write(&source_path, source_code)?; + + let output = Command::new("gcc") + .args(["-O0", "-o", binary_path.to_str().unwrap()]) + .arg(&source_path) + .output()?; + if !output.status.success() { + error!("gcc stderr: {}", String::from_utf8_lossy(&output.stderr)); + return Err("Failed to compile C fixture".into()); + } + + Ok(binary_path) +} + /// Track a command, collecting all memory events. /// /// No allocators are pre-attached: the exec-mapping watcher discovers and /// attaches them as the tracked tree maps executable files. pub fn track_command(command: Command) -> TrackResult { - let tracker = Tracker::new()?; + track_command_impl(command, false) +} + +/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. +pub fn track_command_with_rmap(command: Command) -> TrackResult { + track_command_impl(command, true) +} + +fn track_command_impl(command: Command, track_rmap: bool) -> TrackResult { + let tracker = if track_rmap { + Tracker::new_without_allocators_with_rmap(true)? + } else { + Tracker::new()? + }; tracker.enable_tracking()?; let mut session = tracker.spawn(&command, None)?; @@ -168,3 +214,50 @@ pub fn track_command(command: Command) -> TrackResult { Ok((events, thread_handle)) } + +/// Track a command with rmap, enabling tracking only after the target creates +/// `ready_path`. The target is spawned and resumed first, so any memory it +/// faults before signalling `ready` is already resident when tracking turns on. +/// The caller enables tracking, then creates `go_path` to release the target. +pub fn track_command_with_rmap_late_enable( + command: Command, + ready_path: &Path, + go_path: &Path, +) -> TrackResult { + let tracker = Tracker::new_without_allocators_with_rmap(true)?; + + let mut session = tracker.spawn(&command, None)?; + let rx = session.take_events()?; + + let handshake = (|| -> anyhow::Result<()> { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !ready_path.exists() { + if std::time::Instant::now() > deadline { + anyhow::bail!("target never signalled baseline-ready"); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + tracker.enable_tracking()?; + std::fs::write(go_path, b"go")?; + Ok(()) + })(); + + // A failed handshake leaves the target blocked on `go_path`; Session has no + // Drop kill, so reap it explicitly before propagating or the test hangs. + if let Err(e) = handshake { + unsafe { libc::kill(session.pid(), libc::SIGKILL) }; + let _ = session.wait(); + let _ = tracker.finish(); + return Err(e); + } + + session.wait()?; + drop(session); + let events: Vec = rx.iter().collect(); + + tracker.finish()?; + let thread_handle = std::thread::spawn(move || drop(tracker)); + + info!("Tracked {} events (late enable)", events.len()); + Ok((events, thread_handle)) +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap new file mode 100644 index 00000000..2745b74d --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 64, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap new file mode 100644 index 00000000..bc24de5b --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap @@ -0,0 +1,42 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnonKb:": 64, + "ParentRssAnonKb:": 32 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap new file mode 100644 index 00000000..3a97ec7d --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap @@ -0,0 +1,45 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnon:": 32, + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap new file mode 100644 index 00000000..6e2a2202 --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap new file mode 100644 index 00000000..19bb441e --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 0, + "RssShmem:": 64 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 62aa2451..a6c610e8 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..8) { + let kind = match rng.gen_range(0..10) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -25,6 +25,14 @@ fn generate_events(n: usize) -> Vec { 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, + 8 => MemtrackEventKind::Rss { + member: rng.gen_range(0..4), + size, + }, + 9 => MemtrackEventKind::Rmap { + member: rng.gen_range(0..4), + delta: rng.gen_range(-1024..1024), + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index ad843b28..94c911c4 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -78,6 +78,19 @@ pub enum MemtrackEventKind { Brk { size: u64, }, + Rss { + member: i32, + size: u64, + }, + Rmap { + member: i32, + delta: i64, + }, + Fork { + parent_pid: pid_t, + }, + Exec, + Exit, } pub struct MemtrackEventStream { @@ -116,6 +129,16 @@ mod tests { addr: 0x20, kind: MemtrackEventKind::Free, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 300, + addr: 0, + kind: MemtrackEventKind::Rss { + member: 1, + size: 40960, + }, + }, ]; let artifact = MemtrackArtifact { diff --git a/justfile b/justfile new file mode 100644 index 00000000..0b6c28f0 --- /dev/null +++ b/justfile @@ -0,0 +1,39 @@ +# memtrack's eBPF tests need the host kernel and root, so run them in a privileged +# container. --pid=host is required: eBPF filters on host-namespace PIDs, and without +# it the PID filter matches nothing (valid artifact, 0 events). The tracefs/debugfs +# mounts let libbpf read tracepoint IDs. GITHUB_ACTIONS gates the tests at BUILD time. + +image := "codspeed-memtrack-test" +root := justfile_directory() + +# Build the memtrack test container image. +memtrack-image: + docker build -t {{image}} {{root}}/.agents/docker + +# Run the memtrack RSS tests in docker. Pass any value for `update` to rewrite snapshots, +# e.g. `just memtrack-rss 1`. +memtrack-rss update="": memtrack-image + docker run --rm --privileged --pid=host \ + -v {{root}}:/work \ + -v codspeed-memtrack-target:/tmp/target \ + -v /sys/kernel/tracing:/sys/kernel/tracing:ro \ + -v /sys/kernel/debug:/sys/kernel/debug:ro \ + -e GITHUB_ACTIONS=1 \ + -e CARGO_TARGET_DIR=/tmp/target \ + {{ if update != "" { "-e INSTA_UPDATE=always" } else { "" } }} \ + -w /work {{image}} \ + cargo test -p memtrack --test rss_tests + +# TEMPORARY: cross-check tracked RSS of a real-world `ls /nix/store` run against +# wait4 rusage. Needs the host nix store mounted into the container. +memtrack-rss-ls: memtrack-image + docker run --rm --privileged --pid=host \ + -v {{root}}:/work \ + -v codspeed-memtrack-target:/tmp/target \ + -v /nix/store:/nix/store:ro \ + -v /sys/kernel/tracing:/sys/kernel/tracing:ro \ + -v /sys/kernel/debug:/sys/kernel/debug:ro \ + -e GITHUB_ACTIONS=1 \ + -e CARGO_TARGET_DIR=/tmp/target \ + -w /work {{image}} \ + cargo test -p memtrack --test rss_tests test_rss_ls_nix_store -- --nocapture