Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,18 @@ on:
type: string
default: testnet-455087014-snapload
description: 'load-only testnet ledger (genesis.bin + full snapshot) for the snapshot row'
epoch-ledger:
type: string
default: mainnet-epoch-boundary-461376000
description: 'minimized mainnet ledger whose first replayed slots cross an epoch boundary, for the epoch row'
workflow_dispatch:
inputs:
snapshot-ledger:
type: string
default: testnet-455087014-snapload
epoch-ledger:
type: string
default: mainnet-epoch-boundary-461376000

permissions:
contents: read
Expand Down Expand Up @@ -158,6 +165,7 @@ jobs:
BENCH_DIR: ${{ github.workspace }}/../bench
DUMP_DIR: ${{ github.workspace }}/../dump
BENCH_SNAP_LEDGER: ${{ inputs.snapshot-ledger }}
BENCH_EPOCH_LEDGER: ${{ inputs.epoch-ledger }}

steps:
- uses: actions/checkout@v7
Expand Down Expand Up @@ -223,6 +231,9 @@ jobs:
- run: contrib/ci/bench/bench.sh snapshot base
continue-on-error: true # a baseline that cannot run is not the PR's fault; its rows stay pending
- run: contrib/ci/bench/bench.sh snapshot new
- run: contrib/ci/bench/bench.sh epoch base
continue-on-error: true
- run: contrib/ci/bench/bench.sh epoch new
- name: Free RAM disk # brd keeps its pages until rmmod
if: always()
continue-on-error: true
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/on_pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
with:
github-token: ${{ github.token }}
snapshot-ledger: testnet-455087014-snapload
epoch-ledger: mainnet-epoch-boundary-461376000
secrets: inherit
build_checks_1:
if: github.event.pull_request.draft == false
Expand Down
5 changes: 3 additions & 2 deletions contrib/ci/bench/bench.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/bin/bash
# PR perf rows. bench.sh build|replay|snapshot|bench base|new
# PR perf rows. bench.sh build|replay|snapshot|epoch|bench base|new
# Raw results land in $BENCH_DIR/<side>.<what>.*; benchmark_comment.py turns them into the comment.
set -euo pipefail
what=${1:?usage: bench.sh build|replay|snapshot|bench base|new} side=${2:?side}
what=${1:?usage: bench.sh build|replay|snapshot|epoch|bench base|new} side=${2:?side}
BENCH_DIR=${BENCH_DIR:-$(realpath ..)/bench}
DUMP_DIR=${DUMP_DIR:-$(realpath ..)/dump}
SHREDB=/dev/shm/fd-bench-shreds.db # tmpfs: on disk, writeback holds its inode lock and stalls shred/rserve pwrite() for whole slots
Expand Down Expand Up @@ -76,6 +76,7 @@ case $what in
ACCOUNTS_DB=$RAMDB/accounts.db out=$out.warmup backtest "${BENCH_SNAP_LEDGER:?}" -m 100000000
fi
ACCOUNTS_DB=$RAMDB/accounts.db backtest "${BENCH_SNAP_LEDGER:?}" -m 100000000 ;;
epoch) backtest "${BENCH_EPOCH_LEDGER:-mainnet-epoch-boundary-461376000}" -m 2000000 ;; # whole ledger: crosses one epoch boundary
bench)
{ cat "$BENCH_DIR/$side/bench.toml"; printf '[paths]\n accounts = "%s"\n shredb = "%s"\n' "$DUMP_DIR/accounts.db" "$SHREDB"; } > "$out.toml"
quiesce
Expand Down
9 changes: 8 additions & 1 deletion contrib/ci/bench/benchmark_comment.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ def snapshot_load(d, s):
log = read(d, s, "snapshot.log")
return (stamp(log, "replay ready at slot") - stamp(log, "reading full snapshot from file")) % 86400 # midnight

def epoch_boundary(d, s): # ms fd_runtime_process_new_epoch() took at the boundary the epoch ledger crosses
log = re.sub(r"\x1b\[[0-9;]*m", "", read(d, s, "epoch.log")) # the runtime styles this line when colorized
took = re.findall(r"starting epoch \d+ at slot \d+ \(took ([0-9.]+) seconds\)", log)
if not took: raise LookupError("epoch.log: no epoch boundary")
return float(took[-1]) * 1e3

def mem(d, s, cluster):
return json.loads(read(d, s, f"build.mem.{cluster}.json"))["summary"]["total_memory_locked_bytes"] / 2**30

Expand All @@ -36,12 +42,13 @@ def mem(d, s, cluster):
("replay_tps", "replay tps, mainnet", "replay", lambda d, s: summary(d, s, "replay")["tps"], ",.0f tps", 1.25, 2.5, True),
("bench_tps", "bench tps, localnet", "bench", lambda d, s: summary(d, s, "bench")["tps"], ",.0f tps", 1.75, 3.5, True),
("snapshot_load", "snapshot load, testnet", "snapshot", snapshot_load, ".2f s", 1.5, 3.0, False),
("epoch_boundary","epoch boundary, mainnet", "snapshot", epoch_boundary, ".1f ms", 5.0, 10.0, False),
("mem_mainnet", "mem total, mainnet", "replay", lambda d, s: mem(d, s, "mainnet"), ".2f GiB", 0.0, 1.0, False),
("mem_testnet", "mem total, testnet", "replay", lambda d, s: mem(d, s, "testnet"), ".2f GiB", 0.0, 1.0, False),
("compile", "clean compile, firedancer", "*", lambda d, s: float(read(d, s, "build.time").split()[0]), ".2f s", 1.5, 3.0, False),
("binsize", "binary size, firedancer", "replay", lambda d, s: os.path.getsize(f"{d}/{s}/bin/firedancer") / 1e6, ".2f MB", 0.5, 2.0, False),
)
HIST_HDR = ("TPS", "BENCH", "SNAP", "MEM·M", "MEM·T", "COMPILE", "BINARY")
HIST_HDR = ("TPS", "BENCH", "SNAP", "EPOCH", "MEM·M", "MEM·T", "COMPILE", "BINARY")

def tier(row, d):
_, _, _, _, _, warn, red, up = row
Expand Down
Loading