From 99181466cdaa51339e3e777f8fccc4b38f305336 Mon Sep 17 00:00:00 2001 From: Cemberk Date: Thu, 13 Aug 2026 08:15:25 -0500 Subject: [PATCH 1/2] fix(profiling): stop losing TraceLens reports to two avoidable outcomes Both come from running the tools against real traces on an AMD GPU node. - `--short_kernel_study` makes TraceLens die while writing the workbook on some real traces: one of its short-kernel sheets has MultiIndex columns, and TraceLens writes every sheet with `index=False`, which pandas refuses. Every CSV is written before that, so the analysis itself was fine and only the .xlsx was lost, but the non-zero exit turned the trace into a FAILURE. madengine no longer requests the flag; it stays available through the analyzer's trailing extra args. Reported upstream as AMD-AGI/TraceLens#938. - A trace that holds no GPU activity is now SKIPPED rather than FAILURE, with a reason. dynolog configures every process that registered with it, so a torchrun job hands us the launcher's trace alongside the ranks': it only supervises children and runs no kernels. Every distributed run therefore ended with a failure row sitting next to its real report. The docs gain the two things only the GPU runs could show: an iteration-based request that lands before the workload's first optimizer step captures nothing, and an N-rank torchrun job produces N+1 traces. Co-authored-by: Cursor --- docs/profiling.md | 8 ++- .../scripts/common/tools/tracelens_analyze.py | 37 +++++++++++-- tests/e2e/test_tracelens_dummy_pipeline.py | 52 +++++++++++++++++++ .../TraceLens/Reporting/_dummy.py | 34 +++++++++++- tests/unit/test_tracelens_analyze.py | 42 +++++++++++++++ 5 files changed, 166 insertions(+), 7 deletions(-) diff --git a/docs/profiling.md b/docs/profiling.md index 39352804..a4a9eaa9 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -394,7 +394,11 @@ For a short-lived workload, shorten the warmup so the request lands while the mo } ``` -**No trace produced?** The teardown script reports how many traces it found. `no PyTorch process registered` means `dyno gputrace` never matched the workload: confirm the model is PyTorch, and raise `TORCH_PROFILE_WARMUP_S` and `TORCH_PROFILE_MAX_ATTEMPTS` for slow-starting jobs. +**No trace produced?** The teardown script reports how many traces it found. `no PyTorch process registered` means `dyno gputrace` never matched the workload: confirm the model is PyTorch, and raise `TORCH_PROFILE_WARMUP_S` and `TORCH_PROFILE_MAX_ATTEMPTS` for slow-starting jobs. `dyno rejected the trace request` means the request itself was refused, which points at a dynolog version mismatch rather than at the workload. + +**Trace produced but empty?** Iteration-based capture waits for the workload's next `optimizer.step()`, so the request has to land after training has actually started. A request that arrives while the model is still being built (or while MIOpen is autotuning the first convolution) yields a trace with no GPU activity. The warmup must cover startup, not just process launch. + +Every process that registers with dynolog is traced, including the `torchrun` launcher, which supervises its children and runs no kernels itself. An `N`-rank job therefore produces `N + 1` traces, and the launcher's holds nothing to report. ### tracelens - TraceLens Trace Analysis @@ -411,6 +415,8 @@ Each trace format is routed to the matching TraceLens report: **Unreadable formats:** TraceLens cannot read rocprofv3's default SQLite (`*_results.db`) or RPD (`.rpd`) databases. Those are listed in the summary as `SKIPPED` with a pointer at a preset that works — use `rocprofv3_lightweight` for JSON or `rocprofv3_perfetto` for `.pftrace`. For `rpd`, point TraceLens at the `trace.json` its post-script writes alongside the database. +**Traces with nothing to analyze:** a trace that holds no GPU activity is also reported as `SKIPPED` rather than as a failure. The usual source is the `torchrun` launcher process, which dynolog traces along with the ranks that do the work. + #### Analyzing on the Host (Recommended) Running analysis on the host keeps TraceLens' pinned `protobuf` and `xprof` out of your workload image: diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index cd23dc9b..b033f33c 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -97,6 +97,14 @@ # Read size used when checking a trace for undecodable bytes and rewriting it. _SANITIZE_CHUNK_BYTES = 1 << 20 +# What TraceLens says when a trace holds no GPU activity for it to report on. +_NO_GPU_EVENTS_ERROR = "No GPU events found in the trace" +_NO_GPU_EVENTS_REASON = ( + "the trace holds no GPU activity, so there is nothing to report. dynolog " + "configures every process that registered with it, which for a torchrun job " + "includes the launcher: it only supervises its children and runs no kernels." +) + SUMMARY_CSV_FIELDS = ( "trace_file", "kind", @@ -329,6 +337,11 @@ def _failure_detail(returncode: int, output: str) -> str: return lines[-1] if lines else f"exit code {returncode}" +def _has_no_gpu_events(output: str) -> bool: + """Return True when TraceLens refused a trace for carrying no GPU activity.""" + return _NO_GPU_EVENTS_ERROR in output + + def _report_stem(path: str, root: str) -> str: """Return a filesystem-safe, collision-resistant name for a trace's reports.""" relative = os.path.relpath(path, root) @@ -342,6 +355,13 @@ def _report_stem(path: str, root: str) -> str: def _pytorch_args( trace: str, out_base: str, gpu_arch: Optional[str], extra: Sequence[str] ) -> List[str]: + # --short_kernel_study is deliberately not requested. On some real traces its + # sheets have MultiIndex columns, and TraceLens writes every sheet with + # `index=False`, which pandas refuses: "Writing to Excel with MultiIndex + # columns and no index ('index'=False) is not yet implemented". That loses the + # whole workbook after the CSVs are already written. Pass it back through the + # analyzer's trailing extra args if you want those sheets. + # https://github.com/AMD-AGI/TraceLens/issues/938 args = [ "--profile_json_path", trace, @@ -350,7 +370,6 @@ def _pytorch_args( "--output_csvs_dir", f"{out_base}_csv", "--enable_kernel_summary", - "--short_kernel_study", ] if gpu_arch: args += ["--gpu_arch_platform", gpu_arch] @@ -567,6 +586,16 @@ def analyze( for trace, kind, tool, args in jobs: print(f"[tracelens] {tool}: {trace}", flush=True) code, output = _run(_build_command(interpreter, tool, args)) + if code == 0: + status, detail, produced = "SUCCESS", "", os.path.relpath(output_dir, root) + elif _has_no_gpu_events(output): + # Nothing was wrong with the analysis, and nothing was produced. + print(f"[tracelens] skipping {trace}: {_NO_GPU_EVENTS_REASON}", flush=True) + status, detail, produced = "SKIPPED", _NO_GPU_EVENTS_REASON, "" + else: + status = "FAILURE" + detail = _failure_detail(code, output) + produced = os.path.relpath(output_dir, root) results.append( { "trace_file": ( @@ -574,9 +603,9 @@ def analyze( ), "kind": kind, "tracelens_tool": tool, - "status": "SUCCESS" if code == 0 else "FAILURE", - "output": os.path.relpath(output_dir, root), - "detail": "" if code == 0 else _failure_detail(code, output), + "status": status, + "output": produced, + "detail": detail, } ) diff --git a/tests/e2e/test_tracelens_dummy_pipeline.py b/tests/e2e/test_tracelens_dummy_pipeline.py index b3da5679..27024b93 100644 --- a/tests/e2e/test_tracelens_dummy_pipeline.py +++ b/tests/e2e/test_tracelens_dummy_pipeline.py @@ -188,6 +188,32 @@ def write_chrome_trace(path: Path) -> Path: return path +def write_chrome_trace_without_gpu_events(path: Path) -> Path: + """Write the kind of trace dynolog collects from a torchrun launcher. + + The launcher registers with dynolog like any other PyTorch process, so it is + configured and traced alongside the ranks, but it only supervises children: + its trace carries Python frames and no GPU work at all. + """ + payload = { + "schemaVersion": 1, + "traceEvents": [ + { + "ph": "X", + "cat": "python_function", + "name": "torch/distributed/run.py(892): main", + "pid": 724, + "tid": 724, + "ts": 100, + "dur": 9000, + } + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def write_rocprof_json(path: Path) -> Path: """Write a rocprofv3 JSON result document, as ``rocprofv3_lightweight`` does. @@ -391,6 +417,32 @@ def test_a_valid_trace_is_analyzed_where_it_lies(self, profiled_run, dummy_trace assert analyzed == str(profiled_run / "rocprof_output" / "1234_results.json") assert "sanitized copy" not in result.stdout + def test_a_launcher_trace_is_skipped_rather_than_failed( + self, tmp_path, dummy_tracelens + ): + """A torchrun job hands madengine one trace with no GPU work every run. + + dynolog configures every process that registered with it, so the ranks' + traces arrive next to the launcher's. Failing on the launcher would mean + every distributed run ends with a failure row beside its real report. + """ + work = tmp_path / "torchrun" + write_chrome_trace(work / "torch_profiler_output" / "libkineto_trace_892.json") + write_chrome_trace_without_gpu_events( + work / "torch_profiler_output" / "libkineto_trace_724.json" + ) + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + rows = {Path(row["trace_file"]).name: row for row in summary_rows(work)} + assert rows["libkineto_trace_892.json"]["status"] == "SUCCESS" + launcher = rows["libkineto_trace_724.json"] + assert launcher["status"] == "SKIPPED", launcher + assert "no GPU activity" in launcher["detail"] + # No report was written for it, so naming an output would mislead. + assert launcher["output"] == "" + def test_gzipped_kineto_trace_is_analyzed(self, tmp_path, dummy_tracelens): """tensorboard_trace_handler's gzipped traces are picked up too.""" work = tmp_path / "gz" diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py index 0f81bb00..feb423a7 100644 --- a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py @@ -18,6 +18,7 @@ import argparse import glob +import gzip import json import os import sys @@ -29,9 +30,10 @@ _SPECS: Dict[str, Dict[str, object]] = { "TraceLens_generate_perf_report_pytorch": { "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), - "switches": ("--enable_kernel_summary", "--short_kernel_study"), + "switches": ("--enable_kernel_summary",), "optional": ("--gpu_arch_platform",), "input_file": "profile_json_path", + "needs_gpu_events": True, }, "TraceLens_generate_perf_report_rocprof": { "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), @@ -65,6 +67,14 @@ _FORCED_FAILURE_EXIT_CODE = 3 +# Kineto categories that carry GPU work. A trace without any of them gives +# TraceLens nothing to report on, and it says so rather than writing an empty +# report. dynolog traces every process that registered with it, so a torchrun +# job hands madengine one such trace per run: the launcher's. +_GPU_ACTIVITY_CATEGORIES = frozenset( + {"kernel", "gpu_memcpy", "gpu_memset", "gpu_user_annotation"} +) + def _parser(entry_point: str) -> argparse.ArgumentParser: spec = _SPECS[entry_point] @@ -108,6 +118,21 @@ def _check_input_file(path: str) -> Optional[int]: return None +def _check_gpu_events(path: str) -> Optional[int]: + opener = gzip.open if path.endswith(".gz") else open + try: + with opener(path, "rt", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, ValueError): + # Unreadable input is what the other checks are for. + return None + events = payload.get("traceEvents", []) if isinstance(payload, dict) else [] + for event in events: + if isinstance(event, dict) and event.get("cat") in _GPU_ACTIVITY_CATEGORIES: + return None + return _fail("ValueError: No GPU events found in the trace") + + def _check_input_glob(pattern: str, world_size: str) -> Optional[int]: matches = [p for p in glob.glob(pattern, recursive=True) if os.path.isfile(p)] if len(matches) < 2: @@ -172,9 +197,14 @@ def _report(entry_point: str, argv: Optional[Sequence[str]]) -> int: spec = _SPECS[entry_point] if "input_file" in spec: - failure = _check_input_file(getattr(parsed, str(spec["input_file"]))) + trace = getattr(parsed, str(spec["input_file"])) + failure = _check_input_file(trace) if failure is not None: return failure + if spec.get("needs_gpu_events"): + failure = _check_gpu_events(trace) + if failure is not None: + return failure if "input_glob" in spec: failure = _check_input_glob( getattr(parsed, str(spec["input_glob"])), parsed.world_size diff --git a/tests/unit/test_tracelens_analyze.py b/tests/unit/test_tracelens_analyze.py index 6a6312ab..9e3e1354 100644 --- a/tests/unit/test_tracelens_analyze.py +++ b/tests/unit/test_tracelens_analyze.py @@ -153,6 +153,18 @@ def test_pytorch_args_omit_roofline_without_arch(self, analyzer): args = analyzer._pytorch_args("t.json", "/out/t", None, []) assert "--gpu_arch_platform" not in args + def test_pytorch_args_do_not_request_the_short_kernel_study(self, analyzer): + # TraceLens writes that sheet with index=False, and pandas refuses when + # its columns are a MultiIndex, which loses the whole workbook. + args = analyzer._pytorch_args("t.json", "/out/t", "MI300X", []) + assert "--short_kernel_study" not in args + + def test_the_short_kernel_study_can_be_asked_for_explicitly(self, analyzer): + args = analyzer._pytorch_args( + "t.json", "/out/t", "MI300X", ["--short_kernel_study"] + ) + assert "--short_kernel_study" in args + def test_pftrace_produces_three_complementary_reports(self, analyzer): jobs = analyzer._pftrace_jobs("t.pftrace", "/out/t", []) assert [tool for tool, _ in jobs] == [ @@ -227,6 +239,36 @@ def fake_run(command, cwd=None): assert len(rows) == 11 assert set(rows[0]) == set(analyzer.SUMMARY_CSV_FIELDS) + def test_a_trace_without_gpu_activity_is_skipped_not_failed( + self, analyzer, tmp_path, monkeypatch + ): + """dynolog traces the torchrun launcher too, and it runs no kernels. + + Reporting that as a failure means every multi-process run profiled + through dynolog ends with a failure row next to its real report. + """ + _write(tmp_path, "torch_profiler_output/libkineto_trace_724.json", CHROME_TRACE) + monkeypatch.setattr( + analyzer, + "_run", + lambda command, cwd=None: ( + 1, + "Traceback (most recent call last):\n" + "ValueError: No GPU events found in the trace", + ), + ) + + summary = analyzer.analyze( + root=str(tmp_path), output_dir=str(tmp_path / "out"), python=sys.executable + ) + assert summary["failed"] == 0 + assert summary["skipped"] == 1 + row = summary["results"][0] + assert row["status"] == "SKIPPED" + assert "no GPU activity" in row["detail"] + # Nothing was written, so pointing at a report directory would mislead. + assert row["output"] == "" + def test_records_failure_detail(self, analyzer, tmp_path, monkeypatch): _write(tmp_path, "rocprof_output/1_results.json", b"{}") monkeypatch.setattr( From 3a980948715e4cf1dcbe3677c3cc085c65ed18fc Mon Sep 17 00:00:00 2001 From: Cemberk Date: Thu, 13 Aug 2026 10:01:23 -0500 Subject: [PATCH 2/2] docs(profiling): record why rocprof reports have no workbook TraceLens' rocprof report writes either the CSVs or the workbook, never both, and madengine asks for the CSVs. That also makes the flag we had to drop for the PyTorch report harmless here, which is worth saying next to the arguments so nobody drops it twice. Co-authored-by: Cursor --- docs/profiling.md | 2 ++ src/madengine/scripts/common/tools/tracelens_analyze.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/docs/profiling.md b/docs/profiling.md index a4a9eaa9..76c688b4 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -473,6 +473,8 @@ Use a mode-specific variant to restrict analysis to one trace kind: | `tracelens_pftrace` | Perfetto traces only | | `tracelens_collective` | Multi-rank collective report only | +Kineto traces yield both an `.xlsx` workbook and a CSV directory per trace. rocprofv3 traces yield the CSV directory only: TraceLens' rocprof report writes either format but not both, and madengine asks for the CSVs. + **Full PyTorch pipeline** — capture and analyze in one run: ```bash diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index b033f33c..75c74458 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -377,6 +377,10 @@ def _pytorch_args( def _rocprof_args(trace: str, out_base: str, extra: Sequence[str]) -> List[str]: + # --short_kernel_study is safe to keep here, unlike for the PyTorch report: + # this generator writes either the CSVs or the workbook, never both, so with + # a CSV directory requested it never reaches the Excel writer that the flag's + # MultiIndex sheets break. That also means --output_xlsx_path is ignored. return [ "--profile_json_path", trace,