Dump stacktraces from goroutines on timeouts for inspection

.github/workflows/tests.yml

@@ -52,6 +52,10 @@ jobs:
     env:
       CI: "true"
       MYSQL_VERSION: ${{ matrix.mysql }}
+      # Emit full goroutine stacks (all goroutines, not just the crashing one)
+      # when the test binary panics or times out.  This is the primary
+      # diagnostic for stuck / deadlocked tests.
+      GOTRACEBACK: all
 
     steps:
       - uses: actions/checkout@v6.0.2

Makefile

@@ -64,8 +64,8 @@ test-go:
 		curl -sL $(GOTESTSUM_URL) | tar -xz -C ./bin gotestsum; \
 	fi
 
-	ulimit -n 1024 && ./bin/gotestsum --format short-verbose ./test/go ./copydb/test ./sharding/test -count 1 -p 1 -failfast
-	go test -v
+	ulimit -n 1024 && ./bin/gotestsum --format short-verbose ./test/go ./copydb/test ./sharding/test -count 1 -p 1 -failfast -timeout 5m
+	go test -v -timeout 5m
 
 test-ruby:
 	bundle install

test/helpers/ghostferry_helper.rb

@@ -305,6 +305,14 @@ def start_ghostferry(resuming_state = nil)
                   if logline["level"] == "error"
                     @error_lines << logline
                   end
+                else
+                  # Non-JSON stderr lines are runtime diagnostics: goroutine
+                  # dumps (from SIGQUIT), panic traces, GOTRACEBACK output.
+                  # Write them immediately to Ruby's stderr so they appear in
+                  # CI logs in real time rather than only inside the
+                  # post-failure debug buffer printed by LogCapturer.
+                  $stderr.puts(line)
+                  $stderr.flush
                 end
                 @logger.debug("stderr: #{line}")
               end
@@ -329,6 +337,16 @@ def start_server_watchdog
       @server_watchdog_thread = Thread.new do
         while @subprocess_thread.alive? do
           if (now - @last_message_time) > @message_timeout
+            # Ask the Go process to dump all goroutine stacks to its stderr
+            # before we shut down.  SIGQUIT is the standard Go
+            # goroutine-dump signal (equivalent to pressing Ctrl-\ on the
+            # terminal).  With GOTRACEBACK=all set in the environment, the
+            # runtime will print every goroutine — not just the crashing
+            # one — which is essential for diagnosing stuck tests.
+            @logger.warn("ghostferry watchdog: sending SIGQUIT to #{@pid} for goroutine dump")
+            send_signal("QUIT")
+            sleep 2 # allow the runtime to finish writing the dump to stderr
+
             @server.shutdown
             raise TimeoutError, "ghostferry did not report to the integration test server for the last #{@message_timeout}s"
           end

testhelpers/integration_test_case.go

@@ -2,14 +2,23 @@ package testhelpers
 
 import (
 	"fmt"
+	"os"
+	"runtime"
 	"sync"
 	"testing"
+	"time"
 
 	sql "github.com/Shopify/ghostferry/sqlwrapper"
 
 	"github.com/Shopify/ghostferry"
 )
 
+// integrationTestTimeout is the per-test deadline enforced by startWatchdog.
+// It is intentionally shorter than the -timeout flag passed to go test so
+// that the goroutine dump produced here is visible in CI logs before the
+// test binary is killed by the Go test runner.
+const integrationTestTimeout = 4 * time.Minute
+
 type IntegrationTestCase struct {
 	T *testing.T
 
@@ -32,11 +41,55 @@ type IntegrationTestCase struct {
 }
 
 func (this *IntegrationTestCase) Run() {
+	watchdogDone := this.startWatchdog()
+	defer close(watchdogDone)
 	defer this.Teardown()
 	this.CopyData()
 	this.VerifyData()
 }
 
+// startWatchdog starts a background goroutine that fires after
+// integrationTestTimeout.  When it fires it writes a full goroutine dump to
+// stderr — the primary diagnostic for a stuck / deadlocked test — then
+// terminates the process with exit code 1 so that CI sees a failure instead
+// of silently waiting for the job-level timeout.
+//
+// The caller must close the returned channel when the test finishes normally
+// so the watchdog goroutine exits cleanly.
+func (this *IntegrationTestCase) startWatchdog() chan struct{} {
+	done := make(chan struct{})
+	go func() {
+		select {
+		case <-done:
+			return
+		case <-time.After(integrationTestTimeout):
+		}
+
+		// runtime.Stack with true collects stacks for all goroutines, not
+		// just the calling one.  1 MB is enough for hundreds of goroutines.
+		buf := make([]byte, 1<<20)
+		n := runtime.Stack(buf, true)
+
+		// Write directly to stderr so the output is not swallowed by the
+		// test runner's log-capture buffer.
+		fmt.Fprintf(os.Stderr,
+			"\n=== GHOSTFERRY INTEGRATION TEST WATCHDOG FIRED ===\n"+
+				"Test %q has been running for more than %s.\n"+
+				"All goroutines at the time of the hang:\n\n"+
+				"%s\n"+
+				"===================================================\n",
+			this.T.Name(), integrationTestTimeout, buf[:n])
+		os.Stderr.Sync()
+
+		// os.Exit terminates immediately, bypassing deferred teardown.
+		// This is intentional: the test is hung and we want fast CI
+		// failure with the diagnostic above rather than waiting for the
+		// GitHub Actions job timeout.
+		os.Exit(1)
+	}()
+	return done
+}
+
 func (this *IntegrationTestCase) CopyData() {
 	this.Setup()
 	this.StartFerryAndDataWriter()