Skip to content

Commit 5ea3685

Browse files
authored
Start the autobahn cluster from prebuilt images and bound setup (#4554)
The autobahn integration test still rebuilds the validator side of the cluster in CI even though the workflow hands it prebuilt images. #4143 moved the fullnode sidecar onto the prebuilt rpcnode image, but `setupCluster` still runs `docker-cluster-start`, which rebuilds `sei-chain/localnode` from apt and, because it does not set `SKIP_BUILD`, has node 0 run `make clean` and recompile seid over the `build/seid` the job downloaded. That costs about three minutes on a normal run and took 25 minutes on Oct 7 when the Ubuntu mirrors were slow. All of this happens in `TestMain` before `m.Run`, so a stalled build is only caught by the 40m `go test` alarm or the job timeout, with no indication of which step hung. When `AUTOBAHN_PREBUILT_IMAGES=true`, the cluster now starts with `docker-cluster-start-ci`, the same target the other integration jobs use, which reuses the pulled localnode image and the downloaded seid. Local runs without the variable keep the rebuilding target. Setup now runs under a 30 minute context: `runMake` uses `exec.CommandContext`, and the readiness polls stop when the context ends, so a hang fails with the step that stalled. Teardown stays outside that context. This only touches the integration test harness, with no app-hash or wire impact. The one behaviour change in CI is that the cluster now gets the CI EVM rate limits the `-ci` target sets. Checked with `go vet` and golangci-lint on the package; the Autobahn Basic job on this PR runs the prebuilt path end to end. Fixes PLT-947
1 parent 5a5f672 commit 5ea3685

1 file changed

Lines changed: 74 additions & 35 deletions

File tree

‎integration_test/autobahn/autobahn_test.go‎

Lines changed: 74 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"strconv"
2525
"strings"
2626
"sync/atomic"
27+
"syscall"
2728
"testing"
2829
"time"
2930

@@ -39,6 +40,9 @@ import (
3940
)
4041

4142
const (
43+
// setupTimeout bounds TestMain's setup, which runs outside the go test timeout.
44+
setupTimeout = 30 * time.Minute
45+
4246
// Cluster lifecycle (TestMain).
4347
clusterBootTimeout = 5 * time.Minute
4448
clusterBootPoll = 5 * time.Second
@@ -63,8 +67,8 @@ const (
6367
// progressTxAccountBase is past the 4_000 deterministic senders EVMOnlyLoad
6468
// uses, so a later progress tx is a nonce-0 transfer from a funded account.
6569
progressTxAccountBase = 1_000_000
66-
// prebuiltImagesEnv selects run-rpc-node-skipbuild-ci for the sidecar, which
67-
// runs the already present sei-chain/rpcnode image instead of rebuilding it.
70+
// prebuiltImagesEnv selects the -ci make targets, which reuse the prebuilt
71+
// sei-chain/localnode and sei-chain/rpcnode images and build/seid.
6872
prebuiltImagesEnv = "AUTOBAHN_PREBUILT_IMAGES"
6973

7074
// Fault-tolerance subtests. Execution height is read from each
@@ -159,13 +163,8 @@ func TestMain(m *testing.M) {
159163
fmt.Fprintf(os.Stderr, "chdir to %s: %v\n", root, err)
160164
os.Exit(1)
161165
}
162-
if err := setupCluster(); err != nil {
163-
fmt.Fprintf(os.Stderr, "cluster setup failed: %v\n", err)
164-
teardownCluster()
165-
os.Exit(1)
166-
}
167-
if err := setupFullnodeNode(); err != nil {
168-
fmt.Fprintf(os.Stderr, "fullnode sidecar setup failed: %v\n", err)
166+
if err := setup(); err != nil {
167+
fmt.Fprintln(os.Stderr, err)
169168
teardownCluster()
170169
os.Exit(1)
171170
}
@@ -193,54 +192,90 @@ func findRepoRoot() (string, error) {
193192
}
194193
}
195194

195+
// setup brings up the validator cluster and the fullnode sidecar, failing
196+
// if they are not both ready within setupTimeout.
197+
func setup() error {
198+
ctx, cancel := context.WithTimeout(context.Background(), setupTimeout)
199+
defer cancel()
200+
if err := setupCluster(ctx); err != nil {
201+
return fmt.Errorf("cluster setup failed: %w", err)
202+
}
203+
if err := setupFullnodeNode(ctx); err != nil {
204+
return fmt.Errorf("fullnode sidecar setup failed: %w", err)
205+
}
206+
return nil
207+
}
208+
196209
// runMake runs `make <target>` from the current directory, streaming output.
197-
func runMake(env []string, target string) error {
198-
cmd := exec.Command("make", target)
210+
// The process and its children are killed when ctx ends.
211+
func runMake(ctx context.Context, env []string, target string) error {
212+
cmd := exec.CommandContext(ctx, "make", target)
213+
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
214+
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
199215
cmd.Env = append(os.Environ(), env...)
200216
cmd.Stdout = os.Stdout
201217
cmd.Stderr = os.Stderr
202-
return cmd.Run()
218+
err := cmd.Run()
219+
if err != nil && ctx.Err() != nil {
220+
return fmt.Errorf("%w: %w", ctx.Err(), err)
221+
}
222+
return err
223+
}
224+
225+
// pause waits for d, returning ctx's error if ctx ends first.
226+
func pause(ctx context.Context, d time.Duration) error {
227+
select {
228+
case <-ctx.Done():
229+
return ctx.Err()
230+
case <-time.After(d):
231+
return nil
232+
}
203233
}
204234

205235
// setupCluster starts the autobahn docker cluster and waits until all nodes
206236
// have signalled readiness via build/generated/launch.complete.
207-
func setupCluster() error {
237+
func setupCluster(ctx context.Context) error {
208238
fmt.Println("=== Starting Autobahn Integration Tests ===")
209239
// Best-effort cleanup of any prior cluster, then wipe generated state.
210-
_ = runMake(nil, "docker-cluster-stop")
240+
_ = runMake(ctx, nil, "docker-cluster-stop")
211241
if err := os.RemoveAll("build/generated"); err != nil {
212242
return fmt.Errorf("rm -rf build/generated: %w", err)
213243
}
214-
if err := runMake([]string{"AUTOBAHN=true", "DOCKER_DETACH=true"}, "docker-cluster-start"); err != nil {
215-
return fmt.Errorf("docker-cluster-start: %w", err)
244+
target := "docker-cluster-start"
245+
if prebuiltImages() {
246+
target += "-ci"
247+
}
248+
if err := runMake(ctx, []string{"AUTOBAHN=true", "DOCKER_DETACH=true"}, target); err != nil {
249+
return fmt.Errorf("%s: %w", target, err)
216250
}
217251
// Count created containers (they exist immediately post-compose-up) to
218252
// determine how many launch.complete entries to wait for.
219-
expected, err := countSeiContainers()
253+
expected, err := countSeiContainers(ctx)
220254
if err != nil {
221255
return fmt.Errorf("count cluster containers: %w", err)
222256
}
223257
if expected == 0 {
224-
return fmt.Errorf("no sei-node-* containers found after docker-cluster-start")
258+
return fmt.Errorf("no sei-node-* containers found after %s", target)
225259
}
226260
fmt.Printf("Waiting for %d nodes to be ready...\n", expected)
227261
deadline := time.Now().Add(clusterBootTimeout)
228262
for time.Now().Before(deadline) {
229263
if n := countLaunchComplete("build/generated/launch.complete"); n >= expected {
230264
fmt.Printf("All %d nodes are ready\n", expected)
231265
fmt.Printf("Waiting %s for autobahn connections to establish...\n", autobahnSettleDelay)
232-
time.Sleep(autobahnSettleDelay)
233-
return nil
266+
return pause(ctx, autobahnSettleDelay)
267+
}
268+
if err := pause(ctx, clusterBootPoll); err != nil {
269+
return fmt.Errorf("waiting for nodes to be ready: %w", err)
234270
}
235-
time.Sleep(clusterBootPoll)
236271
}
237272
return fmt.Errorf("cluster failed to start within %s", clusterBootTimeout)
238273
}
239274

240275
// countSeiContainers returns the number of sei-node-* containers that exist
241276
// (running or not yet started).
242-
func countSeiContainers() (int, error) {
243-
out, err := exec.Command("docker", "ps", "-a",
277+
func countSeiContainers(ctx context.Context) (int, error) {
278+
out, err := exec.CommandContext(ctx, "docker", "ps", "-a",
244279
"--filter", "name=sei-node-",
245280
"--format", "{{.Names}}").Output()
246281
if err != nil {
@@ -263,14 +298,14 @@ func prebuiltImages() bool {
263298
// `go install` cycle. The autobahn role itself comes from mode = "full"
264299
// in docker/rpcnode/config/config.toml — setup.go picks the fullnode
265300
// constructor when there's no local validator key.
266-
func setupFullnodeNode() error {
301+
func setupFullnodeNode(ctx context.Context) error {
267302
fmt.Println("=== Starting fullnode sidecar ===")
268-
_ = runMake(nil, "kill-rpc-node") // best-effort cleanup
303+
_ = runMake(ctx, nil, "kill-rpc-node") // best-effort cleanup
269304

270305
// Discover the cluster size from docker so the rpc-node's autobahn config
271306
// covers exactly the validators that came up — non-four-node test runs
272307
// would otherwise produce a mismatched committee.
273-
clusterSize, err := countSeiContainers()
308+
clusterSize, err := countSeiContainers(ctx)
274309
if err != nil {
275310
return fmt.Errorf("count cluster containers: %w", err)
276311
}
@@ -297,7 +332,7 @@ func setupFullnodeNode() error {
297332
// Phase 1: wait for the container to exist and run. Anything `make` does
298333
// before `docker run` (image build, pull) lands here, not in the boot budget.
299334
startDeadline := time.Now().Add(fullnodeStartTimeout)
300-
for !fullnodeRunning() {
335+
for !fullnodeRunning(ctx) {
301336
select {
302337
case err := <-exited:
303338
return fmt.Errorf("make %s exited before %s was running: %v", target, fullnodeContainer, err)
@@ -306,7 +341,9 @@ func setupFullnodeNode() error {
306341
if !time.Now().Before(startDeadline) {
307342
return fmt.Errorf("fullnode sidecar container didn't start within %s", fullnodeStartTimeout)
308343
}
309-
time.Sleep(fullnodeBootPoll)
344+
if err := pause(ctx, fullnodeBootPoll); err != nil {
345+
return fmt.Errorf("waiting for %s to start: %w", fullnodeContainer, err)
346+
}
310347
}
311348

312349
// Phase 2: the node's own boot.
@@ -317,17 +354,19 @@ func setupFullnodeNode() error {
317354
return fmt.Errorf("make %s exited while %s was booting: %v", target, fullnodeContainer, err)
318355
default:
319356
}
320-
if fullnodeRunning() && fullnodeEVMReady() {
357+
if fullnodeRunning(ctx) && fullnodeEVMReady() {
321358
fmt.Println("fullnode sidecar is ready")
322359
return nil
323360
}
324-
time.Sleep(fullnodeBootPoll)
361+
if err := pause(ctx, fullnodeBootPoll); err != nil {
362+
return fmt.Errorf("waiting for %s to boot: %w", fullnodeContainer, err)
363+
}
325364
}
326365
return fmt.Errorf("fullnode sidecar didn't come up within %s of the container starting", fullnodeBootTimeout)
327366
}
328367

329-
func fullnodeRunning() bool {
330-
out, err := exec.Command("docker", "ps",
368+
func fullnodeRunning(ctx context.Context) bool {
369+
out, err := exec.CommandContext(ctx, "docker", "ps",
331370
"--filter", "name="+fullnodeContainer,
332371
"--filter", "status=running",
333372
"--format", "{{.Names}}").Output()
@@ -366,7 +405,7 @@ func evmRPCInContainer(container, method string, params any) (*evmRPCResponse, e
366405
return nil, err
367406
}
368407
out, err := exec.Command("docker", "exec", container,
369-
"curl", "-sf", "-X", "POST",
408+
"curl", "-sf", "--max-time", "10", "-X", "POST",
370409
"-H", "content-type: application/json",
371410
"--data", string(body),
372411
evmRPCURLOnContainerLocalhost).Output()
@@ -387,9 +426,9 @@ func evmRPCInContainer(container, method string, params any) (*evmRPCResponse, e
387426
// new sidecars later goes here too.
388427
func teardownCluster() {
389428
fmt.Println("=== Stopping fullnode sidecar ===")
390-
_ = runMake(nil, "kill-rpc-node")
429+
_ = runMake(context.Background(), nil, "kill-rpc-node")
391430
fmt.Println("=== Stopping cluster ===")
392-
_ = runMake(nil, "docker-cluster-stop")
431+
_ = runMake(context.Background(), nil, "docker-cluster-stop")
393432
}
394433

395434
// countLaunchComplete returns the number of non-empty lines in the launch

0 commit comments

Comments
 (0)