ci: update benchmark mage targets

.github/workflows/benchmark.yaml

@@ -15,7 +15,7 @@ jobs:
   benchmarks:
     name: "Run Full Go benchmarks on Main"
     runs-on: "depot-ubuntu-24.04-arm-8" # same as build-test.yaml
-    timeout-minutes: 15
+    timeout-minutes: 20
     steps:
       - uses: "actions/checkout@v7"
         with:

internal/datastore/benchmark/driver_bench_test.go

@@ -57,6 +57,10 @@ var sortOrders = map[string]options.SortOrder{
 }
 
 func BenchmarkDatastoreDriver(b *testing.B) {
+	if testing.Short() {
+		b.Skip("skipping datastore driver benchmarks in -short mode")
+	}
+
 	for _, driver := range drivers {
 		b.Run(driver.name+driver.suffix, func(b *testing.B) {
 			b.StopTimer()

internal/services/integrationtesting/benchmark_executors_test.go

@@ -55,7 +55,7 @@ const classicDispatchDepthRemaining = 50
 func BenchmarkExecutors(b *testing.B) {
 	for _, engineID := range enginesToBenchmark {
 		if testing.Short() && engineID != "memory" {
-			continue
+			b.Skip("skipping non-memory driver benchmarks in -short mode")
 		}
 		b.Run(engineID, func(b *testing.B) {
 			for _, scenarioName := range executorScenarios {

internal/services/integrationtesting/benchmark_test.go

@@ -299,10 +299,8 @@ var allScenarios = []benchmarkScenario{
 // Hierarchy: engine → scenario (test data) → operation → {dispatch, queryplan}
 func BenchmarkServices(b *testing.B) {
 	for _, engineID := range enginesToBenchmark {
-		if testing.Short() {
-			if engineID != "memory" {
-				continue
-			}
+		if testing.Short() && engineID != "memory" {
+			b.Skip("skipping non-memory driver benchmarks in -short mode")
 		}
 		b.Run(engineID, func(b *testing.B) {
 			for _, scenario := range allScenarios {

magefiles/benchmark.go

@@ -3,6 +3,8 @@
 package main
 
 import (
+	"fmt"
+
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 )
@@ -11,10 +13,29 @@ type Benchmark mg.Namespace
 
 // All Runs all benchmarks
 func (b Benchmark) All() error {
-	return sh.RunV("go", "test", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "5s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem")
+	if err := prebuildTestBinaries(); err != nil {
+		return err
+	}
+	fmt.Println("running all benchmarks")
+	// -p 1 runs one package's test binary at a time so no other package runs during a measurement window.
+	// -cpu 1 sets GOMAXPROCS=1 inside each benchmark, measuring single-threaded per-op latency
+	return sh.RunV("go", "test", "-p", "1", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "5s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem")
 }
 
 // Short runs the short benchmarks for individual PRs on CI -- less time, only the short ones
 func (b Benchmark) Short() error {
-	return sh.RunV("go", "test", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "2s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem", "-short")
+	if err := prebuildTestBinaries(); err != nil {
+		return err
+	}
+	fmt.Println("running short benchmarks")
+	// -p 1 runs one package's test binary at a time so no other package runs during a measurement window.
+	// -cpu 1 sets GOMAXPROCS=1 inside each benchmark, measuring single-threaded per-op latency
+	return sh.RunV("go", "test", "-p", "1", "./...", "-ldflags=-checklinkname=0", "-bench", ".", "-benchtime", "2s", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX", "-cpu", "1", "-benchmem", "-short")
+}
+
+// prebuildTestBinaries compiles and links every test binary in parallel (-run=XXX runs nothing),
+// with the same tags/ldflags as the benchmark runs so they hit the build cache.
+func prebuildTestBinaries() error {
+	fmt.Println("prebuilding all test binaries in parallel")
+	return sh.RunV("go", "test", "./...", "-ldflags=-checklinkname=0", "-tags", "memoryprotection", "-timeout", "0", "-run=XXX")
 }