Unify benchmark timing contracts and add calibrated CI gates

.github/benchmark_thresholds.json

@@ -0,0 +1,53 @@
+{
+  "version": 1,
+  "calibration": "Initial allowlist uses a 20% relative threshold above the documented ~14% gfx950 clock-variation band, plus a 10 us absolute floor.",
+  "architectures": {
+    "gfx942": [
+      {
+        "op": "softmax",
+        "shape": "32768x8192",
+        "dtype": "bf16",
+        "relative_pct": 20.0,
+        "absolute_us": 10.0
+      },
+      {
+        "op": "layernorm",
+        "shape": "32768x8192",
+        "dtype": "bf16",
+        "relative_pct": 20.0,
+        "absolute_us": 10.0
+      },
+      {
+        "op": "rmsnorm",
+        "shape": "32768x8192",
+        "dtype": "bf16",
+        "relative_pct": 20.0,
+        "absolute_us": 10.0
+      }
+    ],
+    "gfx950": [
+      {
+        "op": "softmax",
+        "shape": "32768x8192",
+        "dtype": "bf16",
+        "relative_pct": 20.0,
+        "absolute_us": 10.0
+      },
+      {
+        "op": "layernorm",
+        "shape": "32768x8192",
+        "dtype": "bf16",
+        "relative_pct": 20.0,
+        "absolute_us": 10.0
+      },
+      {
+        "op": "rmsnorm",
+        "shape": "32768x8192",
+        "dtype": "bf16",
+        "relative_pct": 20.0,
+        "absolute_us": 10.0
+      }
+    ],
+    "gfx1201": []
+  }
+}

.github/dashboard/ingest/ingest.py

@@ -147,8 +147,11 @@ def ingest_run(repo: str, run: dict, regression_pct: float) -> tuple[list[dict],
             "url": job.get("html_url"),
         }
         job_status.append(js)
-        if job.get("status") != "completed" or job.get("conclusion") != "success":
-            continue  # only completed-successful jobs have parseable benchmark output
+        if job.get("status") != "completed" or job.get("conclusion") not in {"success", "failure"}:
+            continue
+        # Performance-gate failures still contain the benchmark table and
+        # comparison block that explain the regression. Parse those logs so a
+        # red CI result also remains visible in the dashboard.
         try:
             text = gh_text(f"repos/{repo}/actions/jobs/{job['id']}/logs")
         except RuntimeError as e:

.github/dashboard/ingest/test_ingest.py

@@ -194,6 +194,46 @@ def test_runner_of_matches_known_box_and_rejects_unknown():
     assert ingest.runner_of("") is None
 
 
+def test_ingest_run_parses_completed_failure_logs(monkeypatch):
+    job = {
+        "id": 9,
+        "name": "test (linux-flydsl-mi355-1)",
+        "status": "completed",
+        "conclusion": "failure",
+        "started_at": "2026-01-01T00:00:00Z",
+        "completed_at": "2026-01-01T00:01:00Z",
+        "html_url": "https://example.test/job/9",
+    }
+    monkeypatch.setattr(ingest, "run_jobs", lambda repo, run_id: [job])
+    monkeypatch.setattr(ingest, "resolve_pr", lambda repo, run: 123)
+    monkeypatch.setattr(
+        ingest,
+        "gh_text",
+        lambda path: (
+            "op shape dtype TB/s TFLOPS\n"
+            "softmax 32768x8192 bf16 4.000 -\n"
+        ),
+    )
+    run = {
+        "id": 1,
+        "head_sha": "abc",
+        "head_branch": "feature",
+        "event": "pull_request",
+        "display_title": "timing",
+        "status": "completed",
+        "conclusion": "failure",
+        "html_url": "https://example.test/run/1",
+        "created_at": "2026-01-01T00:00:00Z",
+        "updated_at": "2026-01-01T00:01:00Z",
+        "actor": {"login": "user"},
+    }
+
+    records, summary = ingest.ingest_run("ROCm/FlyDSL", run, regression_pct=-3.0)
+
+    assert records and records[0]["op"] == "softmax"
+    assert summary["jobs"][0]["conclusion"] == "failure"
+
+
 # --------------------------------------------------------------------------- #
 # list_runs — default scans all branches (so PR runs are included)
 # --------------------------------------------------------------------------- #

.github/workflows/flydsl.yaml

@@ -233,6 +233,7 @@ jobs:
       GIT_CONFIG_NOSYSTEM: "1"
       # Temporary quarantine for linux-flydsl-navi-2 GPU 1; see #858.
       CI_HIP_VISIBLE_DEVICES: ${{ contains(matrix.runners, 'navi') && '3' || '' }}
+      BENCHMARK_ARCH: ${{ contains(matrix.runners, 'mi355') && 'gfx950' || (contains(matrix.runners, 'mi325') && 'gfx942' || 'gfx1201') }}
     strategy:
       matrix:
         runners: [
@@ -342,9 +343,16 @@ jobs:
           set -e -o pipefail
           export PYTHONPATH=/tmp/aiter:${PYTHONPATH:-}
           export AITER_REPO=/tmp/aiter
+          export FLYDSL_PERFTEST_USE_EVENTS=0
           cd /flydsl-test
-          BENCH_LOG_DIR=/tmp/flydsl_bench_current bash scripts/run_benchmark.sh 2>&1 | tee /tmp/bench_current.out
-          python3 scripts/benchmark_output_to_csv.py /tmp/bench_current.out /tmp/bench_current.csv
+          python3 -m pytest --collect-only -q -m benchmark \
+            tests/kernels/test_rmsnorm.py \
+            tests/kernels/test_allreduce.py \
+            tests/kernels/test_moe_a8w4_mxscale_gfx1250.py \
+            tests/unit/test_tdm_mcast_add_gfx1250.py
+          BENCH_LOG_DIR=/tmp/flydsl_bench_current \
+            BENCH_OUTPUT_CSV=/tmp/bench_current.csv \
+            bash scripts/run_benchmark.sh 2>&1 | tee /tmp/bench_current.out
           BASH
 
       - name: Run benchmark baselines
@@ -378,12 +386,15 @@ jobs:
               (
                 set -e -o pipefail
                 cd "${worktree}"
+                cp /flydsl-test/scripts/run_benchmark.sh scripts/run_benchmark.sh
+                cp /flydsl-test/scripts/benchmark_log_parser.py scripts/benchmark_log_parser.py
                 export MLIR_PATH=/llvm-project/mlir_install
                 python3 -m pip install -e . --use-pep517 2>&1 | tail -5
                 export PYTHONPATH=/tmp/aiter:${PYTHONPATH:-}
                 export AITER_REPO=/tmp/aiter
-                BENCH_LOG_DIR="${log_dir}" bash scripts/run_benchmark.sh 2>&1 | tee "${output}"
-                python3 /flydsl-test/scripts/benchmark_output_to_csv.py "${output}" "${csv}"
+                export FLYDSL_PERFTEST_USE_EVENTS=0
+                BENCH_LOG_DIR="${log_dir}" BENCH_OUTPUT_CSV="${csv}" \
+                  bash scripts/run_benchmark.sh 2>&1 | tee "${output}"
               )
               status=$?
               if [ "${status}" -eq 0 ] && [ -s "${csv}" ]; then
@@ -418,8 +429,12 @@ jobs:
                   python3 -m pip install --only-binary=:all: "flydsl==${package_version}" 2>&1 | tail -5
                   export PYTHONPATH=/tmp/aiter:${PYTHONPATH:-}
                   export AITER_REPO=/tmp/aiter
-                  BENCH_LOG_DIR="${log_dir}" bash scripts/run_benchmark.sh 2>&1 | tee "${output}"
-                  python3 /flydsl-test/scripts/benchmark_output_to_csv.py "${output}" "${csv}"
+                  export FLYDSL_PERFTEST_USE_EVENTS=0
+                  BENCH_LOG_DIR="${log_dir}" BENCH_OUTPUT_CSV="${csv}" \
+                    bash scripts/run_benchmark.sh 2>&1 | tee "${output}"
+                  if [ ! -s "${csv}" ]; then
+                    python3 /flydsl-test/scripts/benchmark_output_to_csv.py "${output}" "${csv}"
+                  fi
                 )
                 status=$?
                 if [ "${status}" -eq 0 ] && [ -s "${csv}" ]; then
@@ -451,11 +466,14 @@ jobs:
             cd /flydsl-test
             main_label=\$(cat /tmp/bench_main_label 2>/dev/null || echo main)
             python3 scripts/compare_benchmark.py /tmp/bench_main.csv /tmp/bench_current.csv \
-              --baseline-label \"\${main_label}\" --current-label current
+              --baseline-label \"\${main_label}\" --current-label current \
+              --arch '${{ env.BENCHMARK_ARCH }}' \
+              --threshold-config .github/benchmark_thresholds.json \
+              --fail-on-regression
           "
 
       - name: Check benchmark performance (current vs latest tag)
-        if: steps.bench-baselines.outcome != 'skipped'
+        if: always() && steps.bench-baselines.outcome != 'skipped'
         timeout-minutes: 5
         run: |
           docker exec flydsl_test bash -c "

docs/testing_benchmarking_guide.md

@@ -207,7 +207,8 @@ def my_kernel_test(Input, Output):
 
 Features:
 - Device memory profiling to determine rotation count
-- Torch CUDA event timing
+- `torch.profiler` device-time attribution by default
+- Pipelined CUDA/HIP event fallback with `FLYDSL_PERFTEST_USE_EVENTS=1`
 - HIPGraph capture mode (`testGraph=True`)
 - Cache-aware iteration calculation
 
@@ -225,14 +226,44 @@ High-level validation wrapper around `checkAllclose`.
 
 ### 4.2 `tests/kernels/benchmark_common.py`
 
-Shared benchmark harness for performance comparison.
+Compatibility wrappers and performance-comparison formatting. Event timing
+delegates to the canonical `flydsl.do_bench` implementation.
 
 **Key functions:**
 ```python
 # Measure device time (torch CUDA events)
 gpu_us = bench_gpu_us_torch(fn, warmup=20, iters=200)
 ```
 
+### 4.3 Canonical event timer
+
+`flydsl.do_bench` is the single implementation for eager CUDA/HIP event
+measurement. Its explicit schedule keeps physically different experiments from
+being hidden behind ambiguous helper names:
+
+```python
+from flydsl import do_bench
+
+result = do_bench(
+    fn,
+    warmup=20,
+    rep=200,
+    schedule="pipelined",  # or "per_iter" / "isolated"
+    statistic="mean",
+    return_result=True,
+)
+print(result.value_us, result.samples_us)
+```
+
+`BenchResult` always stores microseconds and records the schedule, statistic,
+cache policy, warmup, and iteration count. The historical
+`flydsl.autotune.do_bench(fn, warmup, rep, quantiles)` scalar interface remains
+available and returns milliseconds for backward compatibility.
+
+Do not replace `run_perftest` with this helper when profiler attribution is the
+question: profiler dwell, event elapsed time, host wall clock, and graph replay
+are separate instruments.
+
 ---
 
 ## 5. Test Utilities (`tests/utils.py`)
@@ -308,7 +339,7 @@ def test_my_kernel():
 ### 6.3 Benchmark Test Pattern
 
 ```python
-from tests.kernels.benchmark_common import bench_gpu_us_torch
+from flydsl import do_bench
 
 def benchmark_my_kernel():
     # Setup
@@ -318,14 +349,39 @@ def benchmark_my_kernel():
         launch_fn(input_tensor, output_tensor)
 
     # Measure
-    gpu_us = bench_gpu_us_torch(run, warmup=20, iters=200)
+    result = do_bench(
+        run,
+        warmup=20,
+        rep=200,
+        schedule="pipelined",
+        statistic="mean",
+        return_result=True,
+    )
+    gpu_us = result.value_us
 
     # Compute metrics
     total_bytes = 2 * M * N * elem_size
     bandwidth_tbs = total_bytes / (gpu_us * 1e-6) / 1e12
     print(f"Time: {gpu_us:.1f} us, Bandwidth: {bandwidth_tbs:.2f} TB/s")
 ```
 
+### 6.4 Benchmark CI records and gates
+
+`run_benchmark.sh --output_csv PATH` preserves the existing throughput columns
+and appends normalized `avg_us`, sample metadata, measurement semantics, and
+GPU architecture. Human-readable stdout intentionally keeps its original
+five-column format for dashboard compatibility.
+
+The general comparator uses the same relative-AND-absolute rule as the
+allreduce gate. Hard failures are limited to the stable per-architecture
+allowlist in `.github/benchmark_thresholds.json`; uncalibrated rows remain
+report-only. Missing baselines skip the comparison rather than failing a PR.
+
+Initial gfx942/gfx950 rows use a 20% relative threshold plus a 10 us absolute
+floor. The relative threshold is intentionally wider than the documented
+approximately 14% gfx950 clock-variation band and must be recalibrated from CI
+history before adding more rows.
+
 ---
 
 ## 7. GEMM Test CLI Arguments
@@ -358,6 +414,7 @@ python tests/kernels/test_preshuffle_gemm.py \
 | `FLYDSL_RUNTIME_CACHE_DIR` | Compiler | Cache directory (default: `~/.flydsl/cache`) |
 | `RUN_TESTS_FULL` | `run_tests.sh` | Set to `1` to run all parametrized cases |
 | `BENCH_LOG_DIR` | `run_benchmark.sh` | Benchmark log directory (default: `/tmp/flydsl_bench`) |
+| `BENCH_OUTPUT_CSV` | `run_benchmark.sh` | Write enriched CSV with raw us and measurement metadata |
 
 ---
 
@@ -385,11 +442,15 @@ bash scripts/dumpir.sh
 |---|---|
 | `scripts/run_tests.sh` | Full test runner (pytest + examples + FileCheck) |
 | `scripts/run_benchmark.sh` | Benchmark harness with configurable shapes |
+| `scripts/benchmark_log_parser.py` | Normalize benchmark logs, including raw latency in us |
+| `scripts/compare_benchmark.py` | Throughput report plus calibrated raw-us regression gate |
+| `.github/benchmark_thresholds.json` | Per-architecture hard-gate allowlist and thresholds |
 | `scripts/dumpir.sh` | IR dump helper script |
 | `tests/conftest.py` | Pytest fixtures (MLIR context, module, insert point) |
 | `tests/test_common.py` | `perftest()`, `checkAllclose()`, `verify_output()` |
 | `tests/utils.py` | `pertoken_quant()`, `shuffle_weight()` |
-| `tests/kernels/benchmark_common.py` | `bench_gpu_us_torch()`, benchmark harness |
+| `python/flydsl/autotune.py` | Canonical `do_bench()` event timer and `BenchResult` |
+| `tests/kernels/benchmark_common.py` | Compatibility wrappers and benchmark formatting |
 | `tests/mlir/{LayoutAlgebra,Conversion,Transforms}/` | MLIR lit tests (18 files) |
 | `tests/python/examples/` | Python AOT examples |
 | `tests/kernels/test_*.py` | GPU kernel tests (12 files) |

python/flydsl/__init__.py

@@ -4,4 +4,9 @@
 
 __version__ = "0.3.0"
 
-from .autotune import Config as Config, autotune as autotune  # noqa: E402
+from .autotune import (  # noqa: E402
+    BenchResult as BenchResult,
+    Config as Config,
+    autotune as autotune,
+    do_bench as do_bench,
+)