Add opt-in MI_SINGLE_THREADED specialization (idiomatic + safe) with analysis

CMakeLists.txt

@@ -37,6 +37,7 @@ option(MI_BUILD_OBJECT      "Build object library" ON)
 option(MI_BUILD_TESTS       "Build test executables" ON)
 
 option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF)
+option(MI_SINGLE_THREADED   "Specialize for single-threaded use (removes the per-free thread-ownership check; unsafe with multiple threads)" OFF)
 option(MI_NO_PADDING        "Force no use of padding even in DEBUG mode etc." OFF)
 option(MI_INSTALL_TOPLEVEL  "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF)
 option(MI_NO_THP            "Disable transparent huge pages support on Linux/Android for the mimalloc process only" OFF)
@@ -301,6 +302,11 @@ if (MI_SKIP_COLLECT_ON_EXIT)
   list(APPEND mi_defines MI_SKIP_COLLECT_ON_EXIT=1)
 endif()
 
+if (MI_SINGLE_THREADED)
+  message(STATUS "Specialize for single-threaded use (MI_SINGLE_THREADED=ON)")
+  list(APPEND mi_defines MI_SINGLE_THREADED=1)
+endif()
+
 if(MI_DEBUG_FULL)
   message(STATUS "Set debug level to full assertion and internal invariant checking (MI_DEBUG_FULL=ON, expensive)")
   set(MI_DEBUG ON)

bench/single-threaded/README.md

@@ -0,0 +1,64 @@
+# `MI_SINGLE_THREADED` — analysis & reproduction
+
+This directory contains the full investigation behind the opt-in
+`MI_SINGLE_THREADED` specialization of the free fast path, plus everything
+needed to reproduce it. It documents **two** analyses:
+
+1. **Delivered change** — an idiomatic, safe `MI_SINGLE_THREADED` switch that
+   removes the per-free thread-ownership check. It gives a **statistically
+   significant 3.5–4.6 % cycle reduction** on allocator-bound single-threaded
+   workloads, and is a no-op for the default (multi-threaded) build.
+2. **Rejected experiment** — an ExGen-Malloc-style *single free list* (push
+   frees straight onto `page->free` for immediate hot-block reuse). It is
+   correct and reduces L1 misses, but **regresses 4–4.5 %** because it destroys
+   the instruction-level parallelism that mimalloc's `free`/`local_free` split
+   provides. Kept on the `single-threaded-sota` branch as a documented negative
+   result.
+
+See [`REPORT.md`](REPORT.md) for the full write-up with numbers, counters and
+methodology.
+
+## Layout
+
+```
+bench/single-threaded/
+  REPORT.md                 full analysis (both parts)
+  microbench/
+    churn.cpp               allocator-bound micro: pure-LIFO + windowed reuse
+    reuse.c                 shows whether freed blocks are reused immediately
+  scripts/
+    build_variants.sh       build baseline + MI_SINGLE_THREADED=ON
+    run_micro.sh            interleaved, pinned, perf-counter micro sweep
+    analyze_micro.py        p50/p90/p99 + Mann-Whitney U + bootstrap CI
+  results/                  reference data from an AMD EPYC 7742 run
+```
+
+## Quick reproduce
+
+```bash
+# 1. build baseline + single-threaded variants (Release)
+bench/single-threaded/scripts/build_variants.sh /tmp/mi-st
+
+# 2. pick the produced .so names (version suffix varies)
+BASE=$(ls /tmp/mi-st/baseline/libmimalloc.so.*.* | grep -E '\.[0-9]+$' | head -1)
+SINGLE=$(ls /tmp/mi-st/single/libmimalloc.so.*.* | grep -E '\.[0-9]+$' | head -1)
+
+# 3. interleaved, pinned sweep (set CORE to an isolated core; performance governor recommended)
+CORE=4 REPEATS=21 OUT=micro.csv \
+  bench/single-threaded/scripts/run_micro.sh "$BASE" baseline "$SINGLE" single
+
+# 4. stats
+bench/single-threaded/scripts/analyze_micro.py micro.csv --baseline baseline
+```
+
+To reproduce the **rejected** single-list experiment, `git checkout
+single-threaded-sota`, rebuild `single`, and add it as a third variant.
+
+## Notes on methodology
+
+* Pin to one isolated core (`taskset -c <CORE>`); set the governor to
+  `performance` and warm the core before measuring (the runner does a 2 s warm-up).
+* Measurements are **interleaved** (baseline/variant alternate every run) so
+  slow frequency/thermal drift cancels out.
+* We report **cycles** (and IPC) from `perf`, not wall-clock alone, and never
+  rely on instruction count as a proxy for speed (see `REPORT.md`, §3).

bench/single-threaded/REPORT.md

@@ -0,0 +1,167 @@
+# Single-threaded specialization of mimalloc — full analysis
+
+**Hardware:** AMD EPYC 7742 (Zen 2, 64C/128T, 1 NUMA node), `performance`
+governor, single pinned core, interleaved runs.
+**Toolchain:** gcc 13.3, `-O3` Release, mimalloc `dev` @ `92854277`.
+**Stats:** medians with p90/p99, Mann-Whitney U, 95 % bootstrap CI on the
+median delta. All raw samples are in [`results/`](results/).
+
+---
+
+## 0. The change
+
+In `mi_free_ex` (`src/free.c`) the free fast path performs a thread-ownership
+test on every free:
+
+```c
+const bool is_local = (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));
+```
+
+If the program only ever calls mimalloc from one thread, every block is
+thread-local and that test is redundant. `MI_SINGLE_THREADED` forces
+`is_local = true` and (in debug builds) asserts the invariant instead:
+
+```c
+#if MI_SINGLE_THREADED
+  mi_assert(_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));
+  const bool is_local = true;
+#else
+  const bool is_local = (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id));
+#endif
+```
+
+This is wired as a documented default in `include/mimalloc/types.h`
+(`#if !defined(MI_SINGLE_THREADED) #define MI_SINGLE_THREADED 0`) and a CMake
+option (`-DMI_SINGLE_THREADED=ON`), matching the existing `MI_*` switches. The
+default build is byte-identical to baseline.
+
+### Safety
+
+The flag is a single-thread-only contract. With it on, calling mimalloc from
+more than one thread is undefined. Unlike a bare branch removal, the debug build
+keeps the ownership check as `mi_assert` (active from `MI_DEBUG>=1`, i.e. any
+debug build including `-DMI_DEBUG=ON`), so multithreaded misuse aborts
+immediately (verified: `test-stress` trips it at `free.c` with SIGABRT) instead
+of silently corrupting the heap. Zero cost in release.
+
+---
+
+## 1. Codegen verification
+
+`objdump` of `mi_ufree` (which inlines `mi_free_ex`):
+
+| build | instructions | `%fs:0x0` reads | mt branch |
+|---|---|---|---|
+| baseline | 48 | yes | `jne mi_free_generic_mt` |
+| `MI_SINGLE_THREADED` | 41 | **none** | gone |
+
+The TLS read, the relaxed atomic load of `segment->thread_id`, the compare and
+the `mt` branch are all removed. Output of `cfrac`/`espresso` is byte-identical.
+
+---
+
+## 2. Why the gain depends on the workload
+
+The eliminated branch is **loop-invariant and always true**, so the hardware
+predicts it perfectly: removing it does **not** change branch-miss counts. Its
+only value is fewer *instructions*, which helps only when the allocator fast
+path is actually on the critical path.
+
+`perf` branch-miss attribution on `alloc-test 1` (a common allocator benchmark):
+
+| where | share of branch-misses |
+|---|---|
+| **the benchmark's own RNG** (`randomPos_RandomSize<…>`) | **78.5 %** |
+| libmimalloc (spread across alloc/free thunks) | 17.0 % |
+| libc | 4.2 % |
+
+So on `alloc-test`/`sh6bench` most cycles are *not* in the allocator, and the
+few saved instructions are hidden. That is why the original PR's wall-clock
+claims did not reproduce here — and on `sh6bench` the change even regressed
++3.7 % from a **code-layout** artifact (instructions −4.4 %, but cycles +3.4 %,
+IPC −7.5 %; the regression vanishes under PGO). Instruction count is not a
+proxy for speed on an out-of-order core.
+
+(Real-benchmark sweep, n=51, is in `results/real_benchmarks_n51.csv`.)
+
+---
+
+## 3. Where the gain is real — allocator-bound microbenchmarks
+
+`microbench/churn.cpp` keeps the harness work minimal so the allocator fast
+path *is* the hot loop (pure LIFO, and windowed reuse of N live objects).
+
+**Cycles, baseline vs `MI_SINGLE_THREADED` (`single`), n=21, interleaved:**
+
+| workload | Δ cycles | IPC | Mann-Whitney p | 95 % CI |
+|---|---|---|---|---|
+| pure LIFO churn | **−4.61 %** | 2.98 → 3.04 | 2.4e-3 | excludes 0 |
+| windowed, 4 K live | **−4.15 %** | flat | 4.4e-6 | excludes 0 |
+| windowed, 64 K live | **−3.45 %** | flat | 2.9e-8 | excludes 0 |
+
+A genuine, significant **3.5–4.6 %** when the allocator is the bottleneck, with
+IPC neutral-to-better (the saving converts cleanly). Reproduce with
+`scripts/run_micro.sh` + `scripts/analyze_micro.py`; reference data in
+`results/microbench_base_pron_sota_n21.csv`.
+
+---
+
+## 4. Rejected experiment — ExGen-style single free list
+
+Motivated by *"Old is Gold: Optimizing Single-threaded Applications with
+ExGen-Malloc"* (Li, John, Yadwadkar, IEEE CAL 2025, arXiv:2510.10219), which
+uses a single free-block list. mimalloc instead splits `free` (malloc pops) from
+`local_free` (free pushes), reconciling them in `_mi_page_free_collect`. A
+consequence (`microbench/reuse.c`): a freed block is **not** reused on the next
+allocation — it stays in `local_free` until a migration, so it goes cold.
+
+On the `single-threaded-sota` branch, under `MI_SINGLE_THREADED` the free path
+pushes directly onto `page->free` (clearing `free_is_zero` for calloc
+correctness):
+
+```c
+#if MI_SINGLE_THREADED
+  mi_block_set_next(page, block, page->free);
+  page->free = block;
+  page->free_is_zero = false;
+#else
+  mi_block_set_next(page, block, page->local_free);
+  page->local_free = block;
+#endif
+```
+
+* **Correct:** 43/43 api tests pass and heavy churn runs clean under
+  `MI_DEBUG_FULL`; `reuse.c` confirms immediate LIFO reuse.
+* **Helps the metrics it targets:** L1-dcache-load-misses −6.7 %, instructions
+  −3.5 % on large-window churn.
+* **But it regresses (cycles):** pure LIFO −3.7 % (worse than `single`'s
+  −4.6 %), windowed 4 K **+4.34 %**, windowed 64 K **+4.55 %** — all significant.
+  IPC drops 2.57 → 2.36.
+
+**Root cause.** mimalloc's `free`/`local_free` split is not only for
+thread-safety: it decouples the pointer malloc reads (`page->free`) from the
+pointer free writes (`page->local_free`), letting the out-of-order core overlap
+alloc and free. Merging them serializes the chain (free's store to `page->free`
+must retire before the next malloc's load), so IPC collapses and the change
+loses despite fewer instructions and fewer cache misses. ExGen is a from-scratch
+design without this property; porting just the list-merge into mimalloc
+backfires. **Rejected.**
+
+Reference data: `results/microbench_base_pron_sota_n21.csv` (the `sota` column).
+
+---
+
+## 5. Conclusions
+
+* `MI_SINGLE_THREADED` (idiomatic + debug assert) is a safe, no-op-by-default
+  switch that delivers a real, significant **3.5–4.6 %** on allocator-bound
+  single-threaded workloads. Recommended to ship as such — **not** as a general
+  speedup, since workloads where the allocator isn't the bottleneck see ~0.
+* On x86 the release fast path is already minimal (`MI_STAT=0`, security/debug
+  checks compile out), so the predicted-branch elimination is the realistic
+  ceiling for a *safe* single-thread fast-path specialization.
+* The single-list idea, though SOTA-motivated, is the wrong port for mimalloc
+  and is rejected on evidence (ILP loss).
+* Larger single-thread wins, if pursued, lie in ExGen's *other* lever —
+  memory-footprint / metadata compaction — validated against RSS and LLC/dTLB,
+  not instruction count.

bench/single-threaded/microbench/churn.cpp

@@ -0,0 +1,39 @@
+// Allocator-bound churn: tight alloc/free with minimal harness work.
+// Mode A: pure LIFO reuse (alloc then immediately free) -> tests fast-path cycles.
+// Mode B: windowed churn (keep N live, free oldest) -> tests temporal reuse / L1.
+#include <cstdlib>
+#include <cstdint>
+#include <cstring>
+#include <vector>
+#include <cstdio>
+int main(int argc, char** argv){
+  int mode = argc>1? atoi(argv[1]) : 1;     // 1=pure, 2=windowed
+  long iters = argc>2? atol(argv[2]) : 200000000L;
+  size_t win = argc>3? atol(argv[3]) : 4096; // live objects in windowed mode
+  size_t sz  = argc>4? atol(argv[4]) : 32;
+  if((mode!=1 && mode!=2) || iters<=0 || (mode==2 && win==0) || sz==0){
+    fprintf(stderr,"usage: churn <mode:1|2> <iters(>0)> [window(>0)] [size(>0)]\n");
+    return 2;
+  }
+  volatile uint64_t sink=0;   // prevent the touch/read loop from being optimized away
+  if(mode==1){
+    for(long i=0;i<iters;i++){
+      char* p=(char*)malloc(sz);
+      p[0]=(char)i; p[sz-1]=(char)i;        // touch block (realistic)
+      sink+=p[0];
+      free(p);
+    }
+  } else {
+    std::vector<char*> live(win,nullptr);
+    for(long i=0;i<iters;i++){
+      size_t idx=i%win;
+      if(live[idx]){ sink+=live[idx][0]; free(live[idx]); }
+      char* p=(char*)malloc(sz);
+      p[0]=(char)i; p[sz-1]=(char)i;
+      live[idx]=p;
+    }
+    for(auto p:live) if(p) free(p);
+  }
+  fprintf(stderr,"sink=%llu\n",(unsigned long long)sink);
+  return 0;
+}

bench/single-threaded/microbench/reuse.c

@@ -0,0 +1,15 @@
+#include <stdio.h>
+#include <stdlib.h>
+int main(){
+  for(int i=0;i<5;i++){
+    void* a=malloc(32); free(a);
+    void* b=malloc(32); free(b);
+    printf("a=%p b=%p same=%d\n",a,b,a==b);
+  }
+  // sequence: alloc two, free both, realloc two
+  void* x=malloc(32); void* y=malloc(32);
+  printf("x=%p y=%p\n",x,y); free(x); free(y);
+  void* z=malloc(32); void* w=malloc(32);
+  printf("z=%p w=%p  z==y?%d z==x?%d\n",z,w,z==y,z==x);
+  return 0;
+}

bench/single-threaded/results/ipc_sh6bench_n11.csv

@@ -0,0 +1,23 @@
+bench,variant,cycles,instructions,task_ms
+sh6bench1,base,5061863111,12601890942,1502.04
+sh6bench1,pron,5175116799,12051422378,1528.01
+sh6bench1,base,5007903932,12601463011,1478.61
+sh6bench1,pron,5173371947,12050799968,1542.45
+sh6bench1,base,4928619778,12602232778,1457.63
+sh6bench1,pron,5164915301,12050440414,1529.49
+sh6bench1,base,4966924959,12600941126,1477.39
+sh6bench1,pron,5208098391,12049742882,1539.60
+sh6bench1,base,5041147742,12601901927,1497.06
+sh6bench1,pron,5151798803,12051041328,1526.98
+sh6bench1,base,5088767504,12601758941,1503.23
+sh6bench1,pron,5157653849,12050574951,1524.38
+sh6bench1,base,5005914095,12601691389,1480.56
+sh6bench1,pron,5102136564,12049790174,1508.75
+sh6bench1,base,4995136342,12602410178,1476.02
+sh6bench1,pron,5198407042,12050575976,1535.47
+sh6bench1,base,5017657877,12600643466,1481.73
+sh6bench1,pron,5208681176,12050992263,1539.90
+sh6bench1,base,4969227458,12600994052,1476.35
+sh6bench1,pron,5184571231,12052025416,1546.24
+sh6bench1,base,4956722350,12603472176,1467.03
+sh6bench1,pron,5195855075,12050150961,1539.24