From 679c5bc91b72824042c99e4f83c96a249b447bf9 Mon Sep 17 00:00:00 2001 From: Satya Date: Wed, 30 Sep 2026 22:43:06 +0800 Subject: [PATCH 1/9] Catch a devnet up to wall clock after a long pause A devnet paused for longer than its stability window (laptop lid closed, DevKit stopped for days) could never produce blocks again: a Haskell producer only forges while its tip is less than 3k/f slots behind wall clock (5 minutes on the default devnet). The only way out was reset, which throws away contracts, wallets and stake. DevKit now catches the chain up with Yano and keeps it (see ADR-0017): - Companion: the Haskell node restarts as a relay, Yano follows its chain, backfills sparse empty blocks to wall clock (one per forecast window plus every epoch boundary), the relay adopts them, and the node restarts as the producer. A 7-minute pause took 18 s. - Yano-only: start runs Yano without forging first, backfills, then starts the live producer, so every skipped epoch gets its boundary instead of one block jumping over them. It runs on start, from a watchdog every 30 s while a companion devnet runs (the closed-lid case), with the new catch-up command, and through POST /devnet/catch-up; GET /devnet/chain-lag reports the lag. devnet.auto.catch.up=false turns the automatic part off. Supporting changes: - YanoService takes a run mode (LIVE, PAST_TIME_TRAVEL, FOLLOW, CATCH_UP) and passes every property as an env var as well - cardano-node is stopped with SIGTERM to the node process, not a forced kill, so restarts do not revalidate the whole ChainDB (also for stop) - RelaySyncWaiter can wait for a slot, not only an epoch - warn when the installed Yano is not the pinned version: download keeps an existing binary, so a DevKit upgrade left pre6 in place Verified: lid-closed freeze of 6.5 min healed by the watchdog; stop, wait 7 min, start; yano-only stop across 4 epochs (boundaries 3->4 to 6->7 processed one by one); SDK e2e suites 13/13 on the caught-up chain. Co-Authored-By: Claude Opus 5.5 --- .../cli/config/application.properties | 6 + .../yacicli/localcluster/ClusterCommands.java | 17 + .../yacicli/localcluster/ClusterService.java | 29 ++ .../localcluster/ClusterStartService.java | 167 ++++++- .../api/ClusterAdminController.java | 23 + .../api/model/ChainLagResponse.java | 23 + .../localcluster/catchup/ChainLag.java | 109 +++++ .../catchup/DevnetCatchUpService.java | 406 ++++++++++++++++++ .../catchup/DevnetStallWatchdog.java | 121 ++++++ .../localcluster/catchup/NodeControl.java | 16 + .../localcluster/yano/RelaySyncWaiter.java | 54 ++- .../yano/YanoBootstrapService.java | 68 +++ .../yano/YanoCompanionService.java | 2 +- .../localcluster/yano/YanoConfigBuilder.java | 62 ++- .../localcluster/yano/YanoRunMode.java | 25 ++ .../localcluster/yano/YanoService.java | 118 +++-- .../localcluster/catchup/ChainLagTest.java | 93 ++++ .../yano/RelaySyncWaiterTest.java | 41 ++ .../yano/YanoRunModeConfigTest.java | 80 ++++ docs/pages/commands.mdx | 16 + docs/pages/yano-node-modes.mdx | 61 +++ 21 files changed, 1459 insertions(+), 78 deletions(-) create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/model/ChainLagResponse.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/NodeControl.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java diff --git a/applications/cli/config/application.properties b/applications/cli/config/application.properties index cbb220f5..a544abe9 100644 --- a/applications/cli/config/application.properties +++ b/applications/cli/config/application.properties @@ -48,6 +48,12 @@ bp.create.enabled=true #yano.relay.sync.stall.timeout.seconds=30 #yano.relay.sync.max.wait.seconds=0 +## Catch-up after a long pause (companion and yano-only). A devnet paused for longer than its stability window +## (laptop lid closed, stopped for days) cannot produce blocks again on its own. With auto catch-up, DevKit +## backfills the gap with Yano (sparse empty blocks, one per epoch boundary) and keeps the same chain: +## on 'start', and every 30s while the devnet runs. false: only report it; run 'catch-up' by hand. +#devnet.auto.catch.up=true + ###################################################### #To configure an external database for Yaci Store (Indexer), diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java index d2cf3e9b..d3513c41 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java @@ -278,6 +278,23 @@ public void startLocalCluster() { } + @ShellMethod(value = "Catch the devnet up to wall clock after the machine slept or the devnet was paused longer than " + + "the stability window. Runs automatically on start and while the devnet runs (devnet.auto.catch.up).", key = "catch-up") + @ShellMethodAvailability("localClusterCmdAvailability") + public void catchUp(@ShellOption(value = {"--force"}, defaultValue = "false", + help = "Catch up even when the chain is not stalled") boolean force) { + String clusterName = CommandContext.INSTANCE.getProperty(ClusterConfig.CLUSTER_NAME); + try { + var result = localClusterService.catchUp(clusterName, force, msg -> writeLn(msg)); + if (!result.success()) + writeLn(error(result.message())); + else if (result.blocksProduced() == 0) + writeLn(info(result.message())); + } catch (IOException e) { + writeLn(error("Catch-up failed: " + e.getMessage())); + } + } + @ShellMethod(value = "Stop the running local devnet", key = "stop") @ShellMethodAvailability("localClusterCmdAvailability") public void stopLocalCluster() { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java index 6e5a1f24..451ce14f 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java @@ -3,6 +3,8 @@ import com.bloxbean.cardano.yaci.core.protocol.localstate.api.Era; import com.bloxbean.cardano.yaci.core.util.OSUtil; import com.bloxbean.cardano.yacicli.common.CommandContext; +import com.bloxbean.cardano.yacicli.localcluster.catchup.ChainLag; +import com.bloxbean.cardano.yacicli.localcluster.catchup.DevnetCatchUpService; import com.bloxbean.cardano.yacicli.localcluster.config.CustomGenesisConfig; import com.bloxbean.cardano.yacicli.localcluster.model.RunStatus; import com.bloxbean.cardano.yacicli.localcluster.config.ApplicationConfig; @@ -93,6 +95,9 @@ public class ClusterService { @Autowired private LocalPeerService localPeerService; + @Autowired + private DevnetCatchUpService devnetCatchUpService; + public ClusterService(ClusterConfig config, ClusterStartService clusterStartService, BlockStreamerService blockStreamerService, @@ -136,6 +141,30 @@ public void stopCluster(Consumer writer) { publisher.publishEvent(new ClusterStopped(clusterName)); } + /** + * How far the running devnet's tip is behind wall clock. + * + * @return the lag, or null when the devnet is not running or its tip cannot be read + */ + public ChainLag chainLag(String clusterName) throws IOException { + if (!clusterStartService.isClusterRunning()) + return null; + ClusterInfo clusterInfo = getClusterInfo(clusterName); + return clusterInfo != null ? devnetCatchUpService.probeLag(clusterInfo) : null; + } + + /** + * Catch the running devnet up to wall clock (see ADR-0017). + * + * @param force catch up even when the chain is not stalled + */ + public DevnetCatchUpService.Result catchUp(String clusterName, boolean force, Consumer writer) throws IOException { + ClusterInfo clusterInfo = getClusterInfo(clusterName); + if (clusterInfo == null) + return new DevnetCatchUpService.Result(false, "Devnet not found: " + clusterName, -1, -1, 0, 0); + return clusterStartService.catchUp(clusterInfo, getClusterFolder(clusterName), force, writer); + } + public void stopClusterNode(Consumer writer) { clusterStartService.stopCluster(writer); } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java index de4e6db6..a7d1aef2 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java @@ -2,6 +2,9 @@ import com.bloxbean.cardano.yaci.core.util.OSUtil; import com.bloxbean.cardano.yacicli.common.CommandContext; +import com.bloxbean.cardano.yacicli.localcluster.catchup.ChainLag; +import com.bloxbean.cardano.yacicli.localcluster.catchup.DevnetCatchUpService; +import com.bloxbean.cardano.yacicli.localcluster.catchup.NodeControl; import com.bloxbean.cardano.yacicli.localcluster.config.CustomGenesisConfig; import com.bloxbean.cardano.yacicli.localcluster.config.GenesisConfig; import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStarted; @@ -56,9 +59,13 @@ public class ClusterStartService { private final YanoService yanoService; private final YanoBootstrapService yanoBootstrapService; private final YanoTimeTravelBootstrap yanoTimeTravelBootstrap; + private final DevnetCatchUpService devnetCatchUpService; private ObjectMapper objectMapper = new ObjectMapper(); - private List processes = new ArrayList<>(); + // Copy-on-write: a catch-up (watchdog thread) replaces the node process while other threads iterate the list + private List processes = new CopyOnWriteArrayList<>(); + // The Haskell node process (the `sh node.sh` wrapper), replaced when a catch-up restarts the node + private volatile Process haskellNodeProcess; private Queue logs = EvictingQueue.create(200); private Queue submitApiLogs = EvictingQueue.create(100); @@ -106,8 +113,9 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu return new RunStatus(false, firstRun); } } else { - // Restart: resume the existing chain at wall-clock; no shift/bootstrap. - boolean yanoStarted = yanoService.start(clusterInfo, clusterFolder, false, writer); + // Restart: resume the existing chain. With auto catch-up, whatever time has passed is + // backfilled first, so the live producer does not jump over the skipped epochs. + boolean yanoStarted = devnetCatchUpService.startYanoOnly(clusterInfo, clusterFolder, writer); if (!yanoStarted) { writer.accept(error("Failed to start Yano.")); return new RunStatus(false, firstRun); @@ -209,6 +217,11 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu } } + // Companion restart after a pause longer than the stability window: the producer cannot forge until + // the chain is caught up to wall clock + if (companionMode && !firstRun) + catchUpOnRestart(clusterInfo, clusterFolder, writer); + Process submitApiProcess = startSubmitApi(clusterInfo, clusterFolder, writer); if (submitApiProcess == null) { writer.accept(error("Submit API process could not be started.")); @@ -269,25 +282,27 @@ private static boolean portAvailabilityCheck(ClusterInfo clusterInfo, Consumer writer) { try { + if (devnetCatchUpService.isInProgress()) { + writer.accept(info("Waiting for the running catch-up to finish before stopping ...")); + if (!devnetCatchUpService.awaitIdle(java.time.Duration.ofMinutes(3))) + writer.accept(warn("The catch-up is still running; stopping anyway")); + } if (processes != null && processes.size() > 0) writer.accept(info("Trying to stop the running cluster ...")); boolean error = false; for (Process process : processes) { if (process != null && process.isAlive()) { - process.descendants().forEach(processHandle -> { - writer.accept(infoLabel("Process", String.valueOf(processHandle.pid()))); - processHandle.destroyForcibly(); - }); - process.destroy(); + process.descendants().forEach(processHandle -> + writer.accept(infoLabel("Process", String.valueOf(processHandle.pid())))); writer.accept(info("Stopping node process : " + process)); - process.waitFor(15, TimeUnit.SECONDS); - if (!process.isAlive()) + if (stopProcessTree(process)) writer.accept(success("Killed : " + process)); else writer.accept(error("Process could not be killed : " + process)); } } + haskellNodeProcess = null; if (!error) { //clean pid files @@ -365,6 +380,7 @@ private Process startNode(Path clusterFolder, ClusterInfo clusterInfo, boolean a writer.accept(success("Starting node from directory : " + nodeStartDir.getAbsolutePath())); Process process = processUtil.startLongRunningProcess(NODE_PROCESS_NAME, builder, logs, writer); if (process == null) return null; + haskellNodeProcess = process; Path nodeSocketPath = clusterFolder.resolve(ClusterConfig.NODE_FOLDER_PREFIX).resolve("node.sock"); int counter = 0; @@ -378,6 +394,137 @@ private Process startNode(Path clusterFolder, ClusterInfo clusterInfo, boolean a return process; } + /** + * Stop a process and its children, gracefully first. The node runs as a child of {@code sh node.sh}, so the + * signal has to reach it directly: SIGTERM lets cardano-node close its ChainDB (0.1 s), while a forced kill + * leaves it unclean and the next start revalidates every chunk, which grows with chain length. + * + * @return true if the process tree is gone + */ + static boolean stopProcessTree(Process process) throws InterruptedException { + List children = process.descendants().toList(); + children.forEach(ProcessHandle::destroy); + process.destroy(); + + long deadline = System.currentTimeMillis() + 15_000; + for (ProcessHandle child : children) { + while (child.isAlive() && System.currentTimeMillis() < deadline) + Thread.sleep(100); + if (child.isAlive()) + child.destroyForcibly(); + } + if (!process.waitFor(Math.max(1, deadline - System.currentTimeMillis()), TimeUnit.MILLISECONDS)) { + process.destroyForcibly(); + process.waitFor(5, TimeUnit.SECONDS); + } + return !process.isAlive() && children.stream().noneMatch(ProcessHandle::isAlive); + } + + /** The Haskell node of this devnet, as a {@link NodeControl} a catch-up can restart as relay or producer. */ + private NodeControl nodeControl(ClusterInfo clusterInfo, Path clusterFolder) { + return new NodeControl() { + @Override + public boolean stopNode(Consumer writer) { + Process process = haskellNodeProcess; + try { + if (process != null && process.isAlive() && !stopProcessTree(process)) { + writer.accept(error("Could not stop the Haskell node")); + return false; + } + if (process != null) + processes.remove(process); + haskellNodeProcess = null; + Files.deleteIfExists(clusterFolder.resolve(ClusterConfig.NODE_FOLDER_PREFIX).resolve("node.sock")); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (IOException e) { + writer.accept(error("Could not remove the node socket: " + e.getMessage())); + return false; + } + } + + @Override + public boolean startNode(boolean asRelay, Consumer writer) { + try { + Process process = ClusterStartService.this.startNode(clusterFolder, clusterInfo, asRelay, writer); + if (process == null) + return false; + processes.add(process); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (IOException | ExecutionException | TimeoutException e) { + writer.accept(error("Could not start the Haskell node: " + e.getMessage())); + return false; + } + } + }; + } + + /** On a companion restart, catch up (or tell the user how to) when the chain is too far behind wall clock. */ + private void catchUpOnRestart(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) + throws InterruptedException { + if (devnetCatchUpService.unsupportedReason(clusterInfo).isPresent()) + return; + // The socket can exist before the node answers queries, so give it a moment + ChainLag lag = devnetCatchUpService.probeLagAfterStart(clusterInfo, java.time.Duration.ofSeconds(30)); + if (lag == null) { + writer.accept(warn("Could not read the chain tip to check whether the devnet needs a catch-up")); + return; + } + if (!lag.needsCatchUpOnStart()) + return; + + if (!devnetCatchUpService.isAutoCatchUp()) { + writer.accept(warn("The chain is %s behind wall clock, past the stability window; the node cannot forge. " + + "Run 'catch-up' to bring it to wall clock.", lag.idleTimeText())); + return; + } + var result = devnetCatchUpService.catchUpCompanion(clusterInfo, clusterFolder, + nodeControl(clusterInfo, clusterFolder), lag, writer); + if (!result.success()) + writer.accept(error(result.message())); + } + + /** + * Catch a running devnet up to wall clock (the {@code catch-up} command, the admin API and the stall watchdog). + * + * @param force catch up even when the chain is not stalled + */ + public DevnetCatchUpService.Result catchUp(ClusterInfo clusterInfo, Path clusterFolder, boolean force, + Consumer writer) { + if (!isClusterRunning()) + return new DevnetCatchUpService.Result(false, + "The devnet is not running. 'start' catches it up automatically.", -1, -1, 0, 0); + + var unsupported = devnetCatchUpService.unsupportedReason(clusterInfo); + if (unsupported.isPresent()) + return new DevnetCatchUpService.Result(false, "Cannot catch up: " + unsupported.get(), -1, -1, 0, 0); + + ChainLag lag = devnetCatchUpService.probeLag(clusterInfo); + if (lag == null) + return new DevnetCatchUpService.Result(false, "Could not read the chain tip", -1, -1, 0, 0); + if (!force && !lag.stalled()) + return new DevnetCatchUpService.Result(true, String.format( + "The chain is live: the tip is %,d slots behind wall clock (stability window %,d slots). Nothing to catch up.", + lag.lagSlots(), lag.forecastWindowSlots()), lag.tipSlot(), lag.tipSlot(), 0, 0); + + if (clusterInfo.getNodeMode() == NodeMode.YANO_ONLY) { + // Yano forges as soon as it runs, so a stalled yano-only chain means Yano stopped; restart it + yanoService.stop(); + boolean started = devnetCatchUpService.startYanoOnly(clusterInfo, clusterFolder, writer); + ChainLag after = devnetCatchUpService.probeLag(clusterInfo); + return new DevnetCatchUpService.Result(started, started ? "Yano restarted at wall clock" : "Failed to start Yano", + lag.tipSlot(), after != null ? after.tipSlot() : -1, 0, 0); + } + + return devnetCatchUpService.catchUpCompanion(clusterInfo, clusterFolder, nodeControl(clusterInfo, clusterFolder), + lag, writer); + } + private Process startSubmitApi(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) throws IOException, InterruptedException, ExecutionException, TimeoutException { String clusterFolderPath = clusterFolder.toAbsolutePath().toString(); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java index 1d8144be..c89009a1 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/ClusterAdminController.java @@ -4,6 +4,9 @@ import com.bloxbean.cardano.yacicli.localcluster.ClusterConfig; import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; import com.bloxbean.cardano.yacicli.localcluster.ClusterService; +import com.bloxbean.cardano.yacicli.localcluster.api.model.ChainLagResponse; +import com.bloxbean.cardano.yacicli.localcluster.catchup.ChainLag; +import com.bloxbean.cardano.yacicli.localcluster.catchup.DevnetCatchUpService; import com.bloxbean.cardano.yacicli.localcluster.config.ApplicationConfig; import com.bloxbean.cardano.yacicli.localcluster.config.CustomGenesisConfig; import com.bloxbean.cardano.yacicli.localcluster.service.ClusterUtilService; @@ -177,6 +180,26 @@ public String reset() { return "done"; } + @Operation(summary = "How far the devnet's chain tip is behind wall clock. A Haskell block producer cannot forge " + + "once the lag reaches the forecast window (stalled = true); POST /devnet/catch-up fixes that.") + @GetMapping("/devnet/chain-lag") + public ResponseEntity getChainLag() throws IOException { + ChainLag lag = clusterService.chainLag(DEFAULT_CLUSTER_NAME); + if (lag == null) + return ResponseEntity.status(503).build(); + return ResponseEntity.ok(ChainLagResponse.of(lag)); + } + + @Operation(summary = "Catch the devnet up to wall clock after the machine slept or the devnet was paused longer " + + "than the stability window. Keeps the chain (contracts, wallets, stake); skipped epochs are processed.") + @PostMapping("/devnet/catch-up") + public ResponseEntity catchUp( + @RequestParam(value = "force", defaultValue = "false") boolean force) throws IOException { + CommandContext.INSTANCE.setProperty("cluster_name", DEFAULT_CLUSTER_NAME); + var result = clusterService.catchUp(DEFAULT_CLUSTER_NAME, force, msg -> log.info(msg)); + return result.success() ? ResponseEntity.ok(result) : ResponseEntity.status(409).body(result); + } + @Operation(summary = "Retrieve the current KES period") @GetMapping("/devnet/kes-period") public int getKesPeriod() throws IOException { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/model/ChainLagResponse.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/model/ChainLagResponse.java new file mode 100644 index 00000000..e1318b86 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/api/model/ChainLagResponse.java @@ -0,0 +1,23 @@ +package com.bloxbean.cardano.yacicli.localcluster.api.model; + +import com.bloxbean.cardano.yacicli.localcluster.catchup.ChainLag; + +/** + * How far the devnet's chain tip is behind wall clock. + * + * @param tipSlot slot of the chain tip + * @param wallClockSlot slot of the current wall-clock time + * @param lagSlots slots the tip is behind wall clock + * @param forecastWindowSlots {@code floor(3k/f)}: a Haskell producer cannot forge once the lag reaches it + * @param stalled the lag has reached the forecast window; the chain needs a catch-up + * @param idleTime the lag as time, e.g. {@code 3d 4h} + * @param epochsBehind epoch boundaries between the tip and wall clock + */ +public record ChainLagResponse(long tipSlot, long wallClockSlot, long lagSlots, long forecastWindowSlots, + boolean stalled, String idleTime, long epochsBehind) { + + public static ChainLagResponse of(ChainLag lag) { + return new ChainLagResponse(lag.tipSlot(), lag.wallClockSlot(), lag.lagSlots(), lag.forecastWindowSlots(), + lag.stalled(), lag.idleTimeText(), lag.epochsToCross()); + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java new file mode 100644 index 00000000..4d12839a --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java @@ -0,0 +1,109 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; + +import java.time.Duration; + +/** + * How far a devnet's chain tip is behind wall clock, measured against the stability window. + *

+ * A Haskell block producer can only forge while its ledger view can be forecast to the current slot, i.e. while + * the tip is less than {@code floor(3k/f)} slots behind wall clock (the forecast window). A devnet that was paused + * for longer (laptop lid closed, devnet stopped) stays stuck until it is caught up. + * + * @param tipSlot slot of the chain tip + * @param wallClockSlot slot of the current wall-clock time + * @param forecastWindowSlots {@code floor(3k/f)} + * @param epochLength slots per epoch + * @param slotLengthSeconds slot length in seconds + */ +public record ChainLag(long tipSlot, long wallClockSlot, long forecastWindowSlots, long epochLength, + double slotLengthSeconds) { + + public static ChainLag of(ClusterInfo clusterInfo, long tipSlot, long nowMillis) { + return new ChainLag(tipSlot, + wallClockSlot(clusterInfo.getStartTime(), clusterInfo.getSlotLength(), nowMillis), + forecastWindowSlots(clusterInfo.getSecurityParam(), clusterInfo.getActiveSlotsCoeff()), + clusterInfo.getEpochLength(), + clusterInfo.getSlotLength()); + } + + /** {@code floor(3k/f)}, the slots a node can forecast its ledger view ahead of its tip. */ + public static long forecastWindowSlots(long securityParam, double activeSlotsCoeff) { + if (securityParam <= 0 || activeSlotsCoeff <= 0) + return 0; + return (long) Math.floor(3 * securityParam / activeSlotsCoeff); + } + + /** The slot at {@code nowMillis} for a chain whose slot 0 started at {@code startTimeSeconds}. */ + public static long wallClockSlot(long startTimeSeconds, double slotLengthSeconds, long nowMillis) { + if (slotLengthSeconds <= 0) + return 0; + long slotLengthMillis = Math.round(slotLengthSeconds * 1000); + return Math.max(0, (nowMillis - startTimeSeconds * 1000) / slotLengthMillis); + } + + public long lagSlots() { + return Math.max(0, wallClockSlot - tipSlot); + } + + /** The producer can no longer forge: the tip is outside the forecast window. */ + public boolean stalled() { + return forecastWindowSlots > 0 && lagSlots() >= forecastWindowSlots; + } + + /** + * Whether a (re)start should catch up before handing the chain to the producer: the lag is past three + * quarters of the forecast window, which leaves the remaining quarter for the node to start and forge. + */ + public boolean needsCatchUpOnStart() { + return forecastWindowSlots > 0 && lagSlots() * 4 > forecastWindowSlots * 3; + } + + public long tipEpoch() { + return epochLength > 0 ? tipSlot / epochLength : 0; + } + + public long wallClockEpoch() { + return epochLength > 0 ? wallClockSlot / epochLength : 0; + } + + /** Epoch boundaries between the tip and wall clock. */ + public long epochsToCross() { + return Math.max(0, wallClockEpoch() - tipEpoch()); + } + + /** + * Rough number of blocks a sparse backfill places: one per {@code window - 1} slots (Yano's automatic + * spacing for empty blocks), one at each epoch start, and one at the target. + */ + public long estimatedBackfillBlocks() { + if (lagSlots() == 0) + return 0; + long interval = Math.max(1, forecastWindowSlots - 1); + return lagSlots() / interval + epochsToCross() + 1; + } + + public Duration idleTime() { + return Duration.ofMillis(Math.round(lagSlots() * slotLengthSeconds * 1000)); + } + + /** Idle time in the largest two units, e.g. {@code 3d 4h}, {@code 2h 5m}, {@code 7m 12s}, {@code 40s}. */ + public String idleTimeText() { + return durationText(idleTime()); + } + + static String durationText(Duration d) { + long days = d.toDays(); + long hours = d.toHoursPart(); + long minutes = d.toMinutesPart(); + long seconds = d.toSecondsPart(); + if (days > 0) + return days + "d " + hours + "h"; + if (hours > 0) + return hours + "h " + minutes + "m"; + if (minutes > 0) + return minutes + "m " + seconds + "s"; + return seconds + "s"; + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java new file mode 100644 index 00000000..0f874db7 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java @@ -0,0 +1,406 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import com.bloxbean.cardano.yaci.core.protocol.chainsync.messages.Point; +import com.bloxbean.cardano.yacicli.common.AnsiColors; +import com.bloxbean.cardano.yacicli.common.CommandContext; +import com.bloxbean.cardano.yacicli.common.Tuple; +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import com.bloxbean.cardano.yacicli.localcluster.NodeMode; +import com.bloxbean.cardano.yacicli.localcluster.service.ClusterUtilService; +import com.bloxbean.cardano.yacicli.localcluster.yano.RelaySyncWaiter; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoBootstrapService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoCompanionService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoRunMode; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoService; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import static com.bloxbean.cardano.yacicli.util.ConsoleWriter.*; + +/** + * Catches a devnet up to wall clock after a pause longer than the stability window (laptop lid closed, devnet + * stopped for days), so developers keep the same chain instead of resetting it. See ADR-0017. + *

+ * Companion: the Haskell node restarts as a relay, Yano follows its chain, Yano then backfills sparse empty + * blocks to wall clock (one per forecast window plus every epoch boundary), the relay adopts them, and the node + * restarts as the block producer, now inside its forecast window. + *

+ * Yano-only: Yano starts on its chain without forging, backfills to wall clock, then restarts as the live producer. + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class DevnetCatchUpService { + private static final long SYNC_STALL_TIMEOUT_MS = 30_000; + private static final int MAX_PUMP_ROUNDS = 20; + private static final Duration YANO_READY_TIMEOUT = Duration.ofSeconds(60); + private static final Duration NODE_READY_TIMEOUT = Duration.ofSeconds(90); + + private final YanoService yanoService; + private final YanoBootstrapService yanoBootstrapService; + private final YanoCompanionService yanoCompanionService; + // ObjectProvider: ClusterUtilService -> ClusterService -> ClusterStartService -> this would be a constructor cycle + private final ObjectProvider clusterUtilServiceProvider; + + // Catch up automatically: on start when the chain is behind, and from the watchdog when a running devnet stalls + @Value("${devnet.auto.catch.up:true}") + private boolean autoCatchUp = true; + + private final AtomicBoolean inProgress = new AtomicBoolean(false); + + /** + * Outcome of a catch-up. + * + * @param success the chain is at wall clock and the producer is forging + * @param message what happened, for the console and the admin API + * @param fromSlot tip slot before the catch-up + * @param toSlot tip slot after it + * @param blocksProduced blocks Yano backfilled + * @param durationMillis wall time the catch-up took + */ + public record Result(boolean success, String message, long fromSlot, long toSlot, long blocksProduced, + long durationMillis) { + static Result failed(String message) { + return new Result(false, message, -1, -1, 0, 0); + } + + static Result nothingToDo(String message, long tipSlot) { + return new Result(true, message, tipSlot, tipSlot, 0, 0); + } + } + + public boolean isAutoCatchUp() { + return autoCatchUp; + } + + public boolean isInProgress() { + return inProgress.get(); + } + + /** + * Wait for a running catch-up to finish, so a stop or reset does not pull processes from under it. + * + * @return true when no catch-up is running any more + */ + public boolean awaitIdle(Duration timeout) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (inProgress.get() && System.currentTimeMillis() < deadline) + Thread.sleep(500); + return !inProgress.get(); + } + + /** Why this devnet cannot be caught up, or empty when it can. */ + public Optional unsupportedReason(ClusterInfo clusterInfo) { + NodeMode mode = clusterInfo.getNodeMode(); + if (clusterInfo.isLocalMultiNodeEnabled()) + return Optional.of("catch-up does not support local multi-node devnets yet"); + if (mode == NodeMode.COMPANION) { + if (clusterInfo.getActiveSlotsCoeff() < 1.0) + return Optional.of("catch-up needs activeSlotsCoeff 1 in companion mode (block time equal to slot length): " + + "Yano's catch-up on an existing chain does not check VRF eligibility, which the Haskell node requires below 1"); + return Optional.empty(); + } + if (mode == NodeMode.YANO_ONLY) + return Optional.empty(); + return Optional.of("catch-up supports companion and yano-only devnets; this one runs in " + + (mode == null ? "haskell-only" : mode.getValue()) + " mode"); + } + + /** + * The chain's lag behind wall clock, read from the Haskell node (companion) or Yano (yano-only). + * + * @return the lag, or null when the tip cannot be read + */ + public ChainLag probeLag(ClusterInfo clusterInfo) { + Tuple tip = clusterInfo.getNodeMode() == NodeMode.YANO_ONLY + ? yanoBootstrapService.getNodeTip(clusterInfo.getYanoHttpPort()) + : haskellTip(clusterInfo); + if (tip == null || tip._2 == null) + return null; + return ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()); + } + + /** + * {@link #probeLag(ClusterInfo)} for a node that has just started: retries until the tip can be read. + * + * @return the lag, or null when the tip could not be read within the timeout + */ + public ChainLag probeLagAfterStart(ClusterInfo clusterInfo, Duration timeout) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (true) { + ChainLag lag = probeLag(clusterInfo); + if (lag != null || System.currentTimeMillis() >= deadline) + return lag; + Thread.sleep(1000); + } + } + + // --------------------------------------------------------------------------------------------------------- + // Companion + // --------------------------------------------------------------------------------------------------------- + + /** + * Catch a companion devnet up to wall clock. The Haskell node must be running (as producer or relay); on + * return it runs as the block producer again, with the original topology, whether or not the catch-up worked. + */ + public Result catchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, NodeControl node, ChainLag lag, + Consumer writer) { + if (!inProgress.compareAndSet(false, true)) + return Result.failed("A catch-up is already running"); + try { + return doCatchUpCompanion(clusterInfo, clusterFolder, node, lag, writer); + } finally { + inProgress.set(false); + } + } + + private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, NodeControl node, ChainLag lag, + Consumer writer) { + long startedAt = System.currentTimeMillis(); + int httpPort = clusterInfo.getYanoHttpPort(); + long epochLength = clusterInfo.getEpochLength(); + Consumer quiet = problemsOnly(writer); + printBanner(lag, writer); + + long blocksProduced = 0; + long yanoTipSlot = lag.tipSlot(); + boolean handedBack = false; + try { + // 1. Haskell node as a relay, peering with Yano: it serves its chain to Yano and pulls Yano's backfill + if (!node.stopNode(quiet)) + return Result.failed("Could not stop the Haskell node"); + yanoCompanionService.updateTopologyForYanoPeering(clusterInfo, clusterFolder, quiet); + if (!node.startNode(true, quiet)) + return Result.failed("Could not start the Haskell node as a relay"); + Tuple relayTip = waitForHaskellTip(clusterInfo, NODE_READY_TIMEOUT); + if (relayTip == null) + return Result.failed("The Haskell relay did not answer tip queries"); + long haskellTipSlot = relayTip._2.getSlot(); + + // 2. Yano follows the Haskell chain. It already holds the bootstrap prefix, so only the blocks forged + // since the handover are fetched. + yanoService.stopQuietly(); + if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.FOLLOW, quiet) + || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) + return Result.failed("Yano did not start to follow the Haskell node (see 'yano-logs')"); + RelaySyncWaiter.Outcome followed = new RelaySyncWaiter(SYNC_STALL_TIMEOUT_MS, 0) + .awaitSlot(haskellTipSlot, epochLength, "Yano sync", + () -> yanoBootstrapService.getNodeTip(httpPort), msg -> writer.accept(info(msg))); + if (!followed.synced()) + return Result.failed("Yano could not sync the Haskell chain: " + followed.reason()); + writer.accept(success("Yano synced the chain from the Haskell node (tip slot %,d)", haskellTipSlot)); + yanoService.stopQuietly(); + + // 3. Yano backfills to wall clock. The long block time keeps it from forging a live block at the + // wall-clock slot before the backfill runs. + if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, quiet) + || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) + return Result.failed("Yano did not start as a block producer (see 'yano-logs')"); + + // 4. Backfill, let the relay adopt it, repeat until the relay is close to wall clock. The last round + // leaves the relay within a quarter of the forecast window, the rest is for the producer restart. + long slack = Math.max(2, lag.forecastWindowSlots() / 4); + RelaySyncWaiter relayWaiter = new RelaySyncWaiter(SYNC_STALL_TIMEOUT_MS, 0); + for (int round = 1; ; round++) { + JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); + if (caughtUp == null) + return Result.failed("Yano's catch-up call failed (see 'yano-logs')"); + yanoTipSlot = caughtUp.path("new_slot").asLong(yanoTipSlot); + blocksProduced += caughtUp.path("blocks_produced").asLong(0); + if (round == 1) + writer.accept(success("Backfilled to slot %,d (%,d blocks)", yanoTipSlot, blocksProduced)); + + RelaySyncWaiter.Outcome adopted = relayWaiter.awaitSlot(yanoTipSlot, epochLength, "Haskell sync", + () -> haskellTip(clusterInfo), msg -> writer.accept(info(msg))); + if (!adopted.synced()) + return Result.failed("The Haskell relay did not sync Yano's backfill: " + adopted.reason()); + + long lagNow = ChainLag.wallClockSlot(clusterInfo.getStartTime(), clusterInfo.getSlotLength(), + System.currentTimeMillis()) - yanoTipSlot; + if (lagNow <= slack) + break; + if (round >= MAX_PUMP_ROUNDS) + return Result.failed("The Haskell relay could not keep up with wall clock (still " + lagNow + + " slots behind)"); + } + writer.accept(success("Haskell node synced the backfill")); + + // 5. Hand block production back to the Haskell node, as the first-run handover does: Yano stops, the + // original topology is restored, and the node restarts as the producer. + yanoService.stopQuietly(); + yanoCompanionService.restoreOriginalTopology(clusterFolder, quiet); + boolean restarted = node.stopNode(quiet) && node.startNode(false, quiet); + handedBack = true; + if (!restarted) + return Result.failed("Could not restart the Haskell node as the block producer"); + + // 6. The producer forges on top of the backfill + Tuple forged = waitForHaskellSlotAbove(clusterInfo, yanoTipSlot, forgeTimeout(clusterInfo)); + long duration = System.currentTimeMillis() - startedAt; + if (forged == null) + return new Result(false, "The chain is caught up, but the Haskell node has not forged yet. " + + "If it does not, run 'catch-up' again.", lag.tipSlot(), yanoTipSlot, blocksProduced, duration); + + String message = resumedMessage(clusterInfo, lag, forged._2.getSlot(), duration); + writer.accept(success(message)); + return new Result(true, message, lag.tipSlot(), forged._2.getSlot(), blocksProduced, duration); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return Result.failed("Catch-up interrupted"); + } catch (IOException | RuntimeException e) { + log.error("Catch-up failed", e); + return Result.failed("Catch-up failed: " + e.getMessage()); + } finally { + if (!handedBack) { + // Never leave the devnet half-way: Yano stopped, original topology, Haskell node as producer + yanoService.stopQuietly(); + yanoCompanionService.restoreOriginalTopology(clusterFolder, quiet); + if (node.stopNode(quiet)) + node.startNode(false, quiet); + } + } + } + + // --------------------------------------------------------------------------------------------------------- + // Yano-only + // --------------------------------------------------------------------------------------------------------- + + /** + * Start Yano for an existing yano-only devnet. With auto catch-up it first runs Yano without forging, backfills + * whatever time has passed, and only then starts the live producer, so every skipped epoch gets its boundary. + */ + public boolean startYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { + if (!autoCatchUp) + return yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer); + + if (!inProgress.compareAndSet(false, true)) { + writer.accept(error("A catch-up is already running")); + return false; + } + try { + catchUpYanoOnly(clusterInfo, clusterFolder, writer); + } finally { + inProgress.set(false); + } + return yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer); + } + + /** Backfill a stopped yano-only chain to wall clock. Yano is stopped again on return. */ + private void catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { + int httpPort = clusterInfo.getYanoHttpPort(); + long startedAt = System.currentTimeMillis(); + try { + if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, problemsOnly(writer)) + || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) { + writer.accept(warn("Could not check the chain tip before starting Yano; starting without catch-up")); + return; + } + Tuple tip = yanoBootstrapService.getNodeTip(httpPort); + if (tip == null) + return; + ChainLag lag = ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()); + if (lag.lagSlots() == 0) + return; + + // Short stops need no banner: the backfill is a block or two + boolean report = lag.needsCatchUpOnStart(); + if (report) + printBanner(lag, writer); + JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); + if (caughtUp == null) { + writer.accept(error("Yano's catch-up call failed (see 'yano-logs'); the first block will skip the gap")); + return; + } + if (report) { + long toSlot = caughtUp.path("new_slot").asLong(); + writer.accept(success("Backfilled to slot %,d (%,d blocks)", toSlot, caughtUp.path("blocks_produced").asLong())); + writer.accept(success(resumedMessage(clusterInfo, lag, toSlot, System.currentTimeMillis() - startedAt))); + } + } finally { + yanoService.stopQuietly(); + } + } + + // --------------------------------------------------------------------------------------------------------- + + private void printBanner(ChainLag lag, Consumer writer) { + writer.accept(header(AnsiColors.CYAN_BOLD, "Devnet catch-up")); + writer.accept(info("Devnet was idle for %s (tip slot %,d, now slot %,d%s)", + lag.idleTimeText(), lag.tipSlot(), lag.wallClockSlot(), + lag.epochsToCross() > 0 ? String.format(", %s behind", epochs(lag.epochsToCross())) : "")); + writer.accept(info("Catching up to wall clock with Yano (sparse backfill, about %,d blocks) ...", + lag.estimatedBackfillBlocks())); + } + + private static String resumedMessage(ClusterInfo clusterInfo, ChainLag lag, long tipSlot, long durationMillis) { + long epoch = clusterInfo.getEpochLength() > 0 ? tipSlot / clusterInfo.getEpochLength() : 0; + return String.format("Devnet resumed in %ds at slot %,d, epoch %,d. Your contracts, wallets and stake are " + + "unchanged%s.", Math.max(1, durationMillis / 1000), tipSlot, epoch, + lag.epochsToCross() > 0 ? String.format("; %s passed", epochs(lag.epochsToCross())) : ""); + } + + private static String epochs(long count) { + return String.format("%,d epoch%s", count, count == 1 ? "" : "s"); + } + + /** Forward only warnings and errors: the Yano and node restarts inside a catch-up are not news to the user. */ + static Consumer problemsOnly(Consumer writer) { + return msg -> { + if (msg != null && (msg.contains("🔴") || msg.contains("❗"))) + writer.accept(msg); + }; + } + + private Tuple haskellTip(ClusterInfo clusterInfo) { + // Before the socket exists a tip query only prints "Find tip error" on the console + if (clusterInfo.getSocketPath() != null && !Files.exists(Path.of(clusterInfo.getSocketPath()))) + return null; + if (CommandContext.INSTANCE.getEra() == null && clusterInfo.getEra() != null) + CommandContext.INSTANCE.setEra(clusterInfo.getEra()); + return clusterUtilServiceProvider.getObject().getTip(msg -> {}); + } + + private Tuple waitForHaskellTip(ClusterInfo clusterInfo, Duration timeout) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (System.currentTimeMillis() < deadline) { + Tuple tip = haskellTip(clusterInfo); + if (tip != null && tip._2 != null) + return tip; + Thread.sleep(1000); + } + return null; + } + + private Tuple waitForHaskellSlotAbove(ClusterInfo clusterInfo, long slot, Duration timeout) + throws InterruptedException { + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (System.currentTimeMillis() < deadline) { + Tuple tip = haskellTip(clusterInfo); + if (tip != null && tip._2 != null && tip._2.getSlot() > slot) + return tip; + Thread.sleep(1000); + } + return null; + } + + /** + * Time to wait for the first forged block: the node has to open its ChainDB first, which took up to 30 s on a + * busy machine, so at least two minutes. + */ + private static Duration forgeTimeout(ClusterInfo clusterInfo) { + double blockTime = clusterInfo.getBlockTime() > 0 ? clusterInfo.getBlockTime() : clusterInfo.getSlotLength(); + return Duration.ofMillis(Math.max(120_000, Math.round(blockTime * 20_000))); + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java new file mode 100644 index 00000000..51dbc3f6 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java @@ -0,0 +1,121 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import com.bloxbean.cardano.yacicli.common.AnsiColors; +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import com.bloxbean.cardano.yacicli.localcluster.ClusterService; +import com.bloxbean.cardano.yacicli.localcluster.NodeMode; +import com.bloxbean.cardano.yacicli.localcluster.events.ClusterDeleted; +import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStarted; +import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStopped; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static com.bloxbean.cardano.yacicli.util.ConsoleWriter.*; + +/** + * Catches a running companion devnet up when its block producer has stalled, which is what happens when the laptop + * lid closes: the processes freeze, and on wake the node's tip is older than its forecast window, so it never forges + * again. Checks the chain lag every {@value #CHECK_INTERVAL_SECONDS} s while a devnet runs. + *

+ * Yano-only devnets are not checked: Yano forges as soon as the machine wakes, so there is no stall to detect (see + * ADR-0017, limitations). + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class DevnetStallWatchdog { + static final long CHECK_INTERVAL_SECONDS = 30; + + // ObjectProvider: ClusterService -> ClusterStartService -> DevnetCatchUpService, and this listens to both sides + private final ObjectProvider clusterServiceProvider; + private final DevnetCatchUpService devnetCatchUpService; + + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, "devnet-stall-watchdog"); + thread.setDaemon(true); + return thread; + }); + private ScheduledFuture check; + private volatile String clusterName; + private volatile boolean hintShown; + + @EventListener + public synchronized void handleClusterStarted(ClusterStarted event) { + stopChecking(); + clusterName = event.getClusterName(); + hintShown = false; + check = scheduler.scheduleWithFixedDelay(this::checkSafely, CHECK_INTERVAL_SECONDS, CHECK_INTERVAL_SECONDS, + TimeUnit.SECONDS); + } + + @EventListener + public void handleClusterStopped(ClusterStopped event) { + stopChecking(); + } + + @EventListener + public void handleClusterDeleted(ClusterDeleted event) { + stopChecking(); + } + + @PreDestroy + public void shutdown() { + stopChecking(); + scheduler.shutdownNow(); + } + + private synchronized void stopChecking() { + if (check != null) { + check.cancel(false); + check = null; + } + } + + private void checkSafely() { + try { + check(); + } catch (Exception e) { + log.debug("Stall check failed: {}", e.getMessage()); + } + } + + void check() throws Exception { + String name = clusterName; + if (name == null || devnetCatchUpService.isInProgress()) + return; + + ClusterService clusterService = clusterServiceProvider.getObject(); + ClusterInfo clusterInfo = clusterService.getClusterInfo(name); + if (clusterInfo == null || clusterInfo.getNodeMode() != NodeMode.COMPANION + || devnetCatchUpService.unsupportedReason(clusterInfo).isPresent()) + return; + + ChainLag lag = clusterService.chainLag(name); + if (lag == null || !lag.stalled()) { + hintShown = false; + return; + } + + if (!devnetCatchUpService.isAutoCatchUp()) { + if (!hintShown) + writeLn(warn("The devnet's chain is %s behind wall clock (the machine slept?) and the node cannot forge. " + + "Run 'catch-up' to bring it to wall clock.", lag.idleTimeText())); + hintShown = true; + return; + } + + writeLn(header(AnsiColors.CYAN_BOLD, "The devnet stopped producing blocks (the machine slept?)")); + var result = clusterService.catchUp(name, false, msg -> writeLn(msg)); + if (!result.success()) + writeLn(error(result.message() + " Run 'catch-up' to retry, or 'reset' to start a new chain.")); + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/NodeControl.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/NodeControl.java new file mode 100644 index 00000000..86220769 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/NodeControl.java @@ -0,0 +1,16 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import java.util.function.Consumer; + +/** + * Stops and starts the devnet's Haskell node process, as a relay (no block producer keys) or as the block producer. + * Implemented by {@link com.bloxbean.cardano.yacicli.localcluster.ClusterStartService}, which owns the process. + */ +public interface NodeControl { + + /** Stop the running node, gracefully so its ChainDB stays clean. Returns false if it could not be stopped. */ + boolean stopNode(Consumer writer); + + /** Start the node as a relay (no forging) or as the block producer. Returns false if it did not start. */ + boolean startNode(boolean asRelay, Consumer writer); +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java index 7fefd49c..66302890 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java @@ -4,6 +4,7 @@ import com.bloxbean.cardano.yacicli.common.Tuple; import java.util.function.Consumer; +import java.util.function.LongPredicate; import java.util.function.LongSupplier; /** @@ -15,7 +16,7 @@ * on long epochs: Yano was stopped, and the producer restarted with a tip already outside its own * forecast horizon, so the chain was frozen from its first minute. */ -final class RelaySyncWaiter { +public final class RelaySyncWaiter { static final long POLL_INTERVAL_MS = 1_000L; static final long PROGRESS_REPORT_INTERVAL_MS = 5_000L; static final long DEFAULT_STALL_TIMEOUT_SECONDS = 30; @@ -23,17 +24,17 @@ final class RelaySyncWaiter { private static final long MAX_SECONDS = Long.MAX_VALUE / 1000; /** Stall timeout to use for a configured value: a positive value as given, anything else the default. */ - static long stallTimeoutSeconds(long configured) { + public static long stallTimeoutSeconds(long configured) { return configured > 0 ? Math.min(configured, MAX_SECONDS) : DEFAULT_STALL_TIMEOUT_SECONDS; } /** Overall ceiling to use for a configured value: zero (no ceiling) or a positive value as given, negative is none. */ - static long maxWaitSeconds(long configured) { + public static long maxWaitSeconds(long configured) { return Math.min(Math.max(0, configured), MAX_SECONDS); } /** Reads the relay's tip: block height and point. Returns {@code null} when the relay is not reachable yet. */ - interface TipReader { + public interface TipReader { Tuple readTip(); } @@ -51,7 +52,7 @@ interface Sleeper { * @param height block height of the last tip seen, or -1 * @param reason why the wait ended when {@code synced} is false; {@code null} otherwise */ - record Outcome(boolean synced, long epoch, long slot, long height, String reason) { + public record Outcome(boolean synced, long epoch, long slot, long height, String reason) { } private final long stallTimeoutMs; @@ -59,7 +60,7 @@ record Outcome(boolean synced, long epoch, long slot, long height, String reason private final LongSupplier clock; private final Sleeper sleeper; - RelaySyncWaiter(long stallTimeoutMs, long maxWaitMs) { + public RelaySyncWaiter(long stallTimeoutMs, long maxWaitMs) { this(stallTimeoutMs, maxWaitMs, System::currentTimeMillis, Thread::sleep); } @@ -84,12 +85,43 @@ record Outcome(boolean synced, long epoch, long slot, long height, String reason * @param tipReader reads the relay's current tip * @param progress receives a progress line about every {@link #PROGRESS_REPORT_INTERVAL_MS} */ - Outcome await(long targetEpoch, long epochLength, TipReader tipReader, Consumer progress) + public Outcome await(long targetEpoch, long epochLength, TipReader tipReader, Consumer progress) throws InterruptedException { if (epochLength <= 0) { return new Outcome(false, -1, -1, -1, "epoch length is not known"); } + return awaitTip(slot -> slot / epochLength >= targetEpoch, epochLength, + (slot, height, blocksPerSecond) -> String.format( + "Relay sync in progress: epoch %d of %d, slot %d, height %d (%d blocks/s)", + slot / epochLength, targetEpoch, slot, height, blocksPerSecond), + tipReader, progress); + } + + /** + * Poll a tip until it reaches {@code targetSlot}, the tip stalls, or the ceiling is hit. + * + * @param targetSlot slot the tip must reach + * @param epochLength slots per epoch, only used for the epoch reported in the outcome + * @param label what is syncing, for the progress lines (e.g. "Yano sync") + * @param tipReader reads the current tip + * @param progress receives a progress line about every {@link #PROGRESS_REPORT_INTERVAL_MS} + */ + public Outcome awaitSlot(long targetSlot, long epochLength, String label, TipReader tipReader, + Consumer progress) throws InterruptedException { + return awaitTip(slot -> slot >= targetSlot, Math.max(1, epochLength), + (slot, height, blocksPerSecond) -> String.format( + "%s in progress: slot %d of %d, height %d (%d blocks/s)", + label, slot, targetSlot, height, blocksPerSecond), + tipReader, progress); + } + + private interface ProgressLine { + String format(long slot, long height, long blocksPerSecond); + } + + private Outcome awaitTip(LongPredicate reached, long epochLength, ProgressLine progressLine, TipReader tipReader, + Consumer progress) throws InterruptedException { final long startedAt = clock.getAsLong(); long lastProgressAt = startedAt; long lastReportAt = startedAt; @@ -118,7 +150,7 @@ Outcome await(long targetEpoch, long epochLength, TipReader tipReader, Consumer< lastReportAt = now; } - if (epoch >= targetEpoch) { + if (reached.test(slot)) { return new Outcome(true, epoch, slot, height, null); } @@ -126,9 +158,7 @@ Outcome await(long targetEpoch, long epochLength, TipReader tipReader, Consumer< long elapsedMs = Math.max(1, now - lastReportAt); long blocksSinceReport = lastReportHeight >= 0 && height >= 0 ? height - lastReportHeight : 0; long blocksPerSecond = blocksSinceReport * 1000 / elapsedMs; - progress.accept(String.format( - "Relay sync in progress: epoch %d of %d, slot %d, height %d (%d blocks/s)", - epoch, targetEpoch, slot, height, blocksPerSecond)); + progress.accept(progressLine.format(slot, height, blocksPerSecond)); lastReportAt = now; lastReportHeight = height; } @@ -138,7 +168,7 @@ Outcome await(long targetEpoch, long epochLength, TipReader tipReader, Consumer< if (sinceProgress >= stallTimeoutMs) { String where = slot >= 0 ? " at slot " + slot + " (epoch " + epoch + ")" : " before any tip was read"; return new Outcome(false, epoch, slot, height, - "relay tip did not advance for " + (sinceProgress / 1000) + "s" + where); + "tip did not advance for " + (sinceProgress / 1000) + "s" + where); } if (maxWaitMs > 0 && now - startedAt >= maxWaitMs) { return new Outcome(false, epoch, slot, height, diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoBootstrapService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoBootstrapService.java index d75993ca..9fc9a2f8 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoBootstrapService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoBootstrapService.java @@ -1,5 +1,7 @@ package com.bloxbean.cardano.yacicli.localcluster.yano; +import com.bloxbean.cardano.yaci.core.protocol.chainsync.messages.Point; +import com.bloxbean.cardano.yacicli.common.Tuple; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; @@ -80,6 +82,72 @@ public boolean shiftEpochs(int httpPort, int epochs, Consumer writer) { } } + /** + * Catch Yano's producer up to wall clock without printing anything. + * + * @return Yano's response ({@code new_slot}, {@code new_block_number}, {@code blocks_produced}), or null on failure + */ + public JsonNode catchUp(int httpPort) { + String url = "http://localhost:" + httpPort + "/api/v1/devnet/epochs/catch-up"; + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofMinutes(10)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString("{}")) + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == 200) + return objectMapper.readTree(response.body()); + log.warn("Catch-up failed: HTTP {} - {}", response.statusCode(), response.body()); + } catch (Exception e) { + log.warn("Catch-up failed: {}", e.getMessage()); + } + return null; + } + + /** + * Yano's node tip from {@code /api/v1/node/tip}, available in every run mode. + * + * @return (block number, point), or null when Yano is not reachable + */ + public Tuple getNodeTip(int httpPort) { + String url = "http://localhost:" + httpPort + "/api/v1/node/tip"; + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(5)) + .GET() + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() == 200) { + JsonNode tip = objectMapper.readTree(response.body()); + if (tip.hasNonNull("slot")) + return new Tuple<>(tip.path("blockNumber").asLong(), + new Point(tip.get("slot").asLong(), tip.path("blockHash").asText())); + } + } catch (Exception e) { + log.debug("Error getting Yano node tip: {}", e.getMessage()); + } + return null; + } + + /** Wait until {@code /api/v1/node/tip} answers (it does in every run mode, unlike the readiness probe). */ + public boolean waitForNodeTip(int httpPort, Duration timeout) { + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (System.currentTimeMillis() < deadline) { + if (getNodeTip(httpPort) != null) + return true; + try { + Thread.sleep(500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return false; + } + public boolean catchUpToWallClock(int httpPort, Consumer writer) { String url = "http://localhost:" + httpPort + "/api/v1/devnet/epochs/catch-up"; try { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java index 4dc5e92f..fd86355a 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java @@ -333,7 +333,7 @@ public void waitForPostHandoverBlock(Consumer writer) { /** * Restore the original topology.json (before Yano peering was added). */ - private void restoreOriginalTopology(Path clusterFolder, Consumer writer) { + public void restoreOriginalTopology(Path clusterFolder, Consumer writer) { try { Path topologyPath = clusterFolder.resolve("node").resolve("topology.json"); Path topologyBackup = clusterFolder.resolve("node").resolve(TOPOLOGY_BEFORE_YANO); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java index a0de3c72..8edec4a6 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java @@ -29,20 +29,21 @@ public class YanoConfigBuilder { private final ClusterConfig clusterConfig; /** - * Build and write application.properties for Yano. + * Yano's configuration for a run mode. {@link YanoService} writes it to application.properties (see + * {@link #write(Map)}) and passes the same values as environment variables, which the native binary honours + * over the file. * * @param clusterInfo cluster configuration * @param yanoConfigDir resolved path to Yano's devnet config (genesis files, keys) * @param yanoDataDir resolved path for Yano's chainstate storage * @param yanoHistoryDir resolved path for Yano's history archive - * @param pastTimeTravelMode whether past-time-travel mode is enabled + * @param runMode how Yano runs (see {@link YanoRunMode}) * @param slotLeaderTimeTravel whether the past-time-travel backfill runs Praos slot-leader checks * (see {@link YanoService#slotLeaderTimeTravelEnabled(ClusterInfo)}) - * @param backfillBlockIntervalSlots slots between blocks in the past-time-travel backfill; 0 = automatic - * @return true if config was written successfully + * @param backfillBlockIntervalSlots slots between blocks in a backfill (past-time-travel or catch-up); 0 = automatic */ - public boolean build(ClusterInfo clusterInfo, Path yanoConfigDir, Path yanoDataDir, Path yanoHistoryDir, - boolean pastTimeTravelMode, boolean slotLeaderTimeTravel, int backfillBlockIntervalSlots) { + public Map properties(ClusterInfo clusterInfo, Path yanoConfigDir, Path yanoDataDir, Path yanoHistoryDir, + YanoRunMode runMode, boolean slotLeaderTimeTravel, int backfillBlockIntervalSlots) { Map props = new LinkedHashMap<>(); // Quarkus profile @@ -76,21 +77,46 @@ public boolean build(ClusterInfo clusterInfo, Path yanoConfigDir, Path yanoDataD props.put("yano.history.projection.enabled", "false"); props.put("yano.history.dir", yanoHistoryDir.toAbsolutePath().toString()); - // Past-time-travel mode - if (pastTimeTravelMode) { - props.put("yano.block-producer.past-time-travel-mode", "true"); - if (slotLeaderTimeTravel) { - props.put("yano.block-producer.past-time-travel-slot-leader-mode", "true"); - } + switch (runMode) { + case PAST_TIME_TRAVEL -> { + props.put("yano.block-producer.past-time-travel-mode", "true"); + if (slotLeaderTimeTravel) { + props.put("yano.block-producer.past-time-travel-slot-leader-mode", "true"); + } - //Spacing for the catch-up from the shifted genesis to wall clock. 0 = automatic : Yano derives - //the widest spacing that still fits inside the forecast window a Haskell relay can validate, - //and keeps the epoch-boundary blocks needed for nonces. 1 is one block per slot (Yano's default), - //which makes create time grow with epoch length. - props.put("yano.block-producer.backfill-block-interval-slots", String.valueOf(backfillBlockIntervalSlots)); + //Spacing for the catch-up from the shifted genesis to wall clock. 0 = automatic : Yano derives + //the widest spacing that still fits inside the forecast window a Haskell relay can validate, + //and keeps the epoch-boundary blocks needed for nonces. 1 is one block per slot (Yano's default), + //which makes create time grow with epoch length. + props.put("yano.block-producer.backfill-block-interval-slots", String.valueOf(backfillBlockIntervalSlots)); + } + case CATCH_UP -> { + //Producer on the existing chain that forges nothing on its own: blocks only come from + //devnet/epochs/catch-up, so no live block can land at the wall-clock slot first + props.put("yano.block-producer.block-time-millis", String.valueOf(YanoRunMode.CATCH_UP_BLOCK_TIME_MILLIS)); + props.put("yano.block-producer.backfill-block-interval-slots", String.valueOf(backfillBlockIntervalSlots)); + } + case FOLLOW -> { + //Follow the Haskell node's chain. Yano's dev mode requires a block producer, so it is off here. + props.put("yano.dev-mode", "false"); + props.put("yano.block-producer.enabled", "false"); + props.put("yano.client.enabled", "true"); + props.put("yano.remote.host", "127.0.0.1"); + props.put("yano.remote.port", String.valueOf(clusterInfo.getNodePort())); + } + case LIVE -> { + } } - // Write to {yanoHome}/config/application.properties + return props; + } + + /** + * Write the properties to {yanoHome}/config/application.properties. + * + * @return true if config was written successfully + */ + public boolean write(Map props) { Path configPath = Path.of(clusterConfig.getYanoHome(), "config", "application.properties"); Path configFolder = configPath.getParent(); if (!configFolder.toFile().exists()) { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java new file mode 100644 index 00000000..53028a35 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java @@ -0,0 +1,25 @@ +package com.bloxbean.cardano.yacicli.localcluster.yano; + +/** + * How DevKit runs Yano. + */ +public enum YanoRunMode { + /** Block producer at wall clock (yano-only, yano-primary, restarts). */ + LIVE, + /** Block producer that starts in the past; the chain is shifted and caught up through the devnet API (first run). */ + PAST_TIME_TRAVEL, + /** Follows the Haskell node's chain (client on, producer off) so Yano's chainstate reaches the Haskell tip. */ + FOLLOW, + /** + * Block producer on an existing chain with a very long block time, so it forges nothing on its own. Blocks are + * only produced through {@code /devnet/epochs/catch-up}, and none can land at the wall-clock slot before it. + */ + CATCH_UP; + + /** Block time used in {@link #CATCH_UP}: long enough that no live block is ever scheduled during a catch-up. */ + public static final long CATCH_UP_BLOCK_TIME_MILLIS = 3_600_000L; + + public boolean producesBlocks() { + return this != FOLLOW; + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java index b7de1251..750aac2c 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java @@ -5,6 +5,7 @@ import com.bloxbean.cardano.yacicli.localcluster.NodeMode; import com.bloxbean.cardano.yacicli.localcluster.events.ClusterDeleted; import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStopped; +import com.bloxbean.cardano.yacicli.util.ConsoleWriter; import com.bloxbean.cardano.yacicli.util.PortUtil; import com.bloxbean.cardano.yacicli.util.ProcessStream; import com.bloxbean.cardano.yacicli.util.ProcessUtil; @@ -28,6 +29,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -64,7 +66,17 @@ public class YanoService { @Value("${yano.past.time.travel.slot.leader.mode:auto}") private String pastTimeTravelSlotLeaderMode = "auto"; + @Value("${yano.version:}") + private String requiredYanoVersion = ""; + + private final AtomicBoolean versionChecked = new AtomicBoolean(false); + public boolean start(ClusterInfo clusterInfo, Path clusterFolder, boolean pastTimeTravelMode, Consumer writer) { + return start(clusterInfo, clusterFolder, pastTimeTravelMode ? YanoRunMode.PAST_TIME_TRAVEL : YanoRunMode.LIVE, + writer); + } + + public boolean start(ClusterInfo clusterInfo, Path clusterFolder, YanoRunMode runMode, Consumer writer) { logs.clear(); Path yanoBin = Path.of(clusterConfig.getYanoHome(), "yano"); @@ -74,6 +86,9 @@ public boolean start(ClusterInfo clusterInfo, Path clusterFolder, boolean pastTi return false; } + if (versionChecked.compareAndSet(false, true)) + warnIfNotRequiredVersion(writer); + if (!PortUtil.isPortAvailable(clusterInfo.getYanoServerPort())) { writer.accept(error("Yano n2n port " + clusterInfo.getYanoServerPort() + " is not available")); return false; @@ -88,7 +103,7 @@ public boolean start(ClusterInfo clusterInfo, Path clusterFolder, boolean pastTi Path yanoConfigDir = prepareYanoConfig(clusterInfo, clusterFolder, writer); if (yanoConfigDir == null) return false; - Process process = startYanoProcess(clusterInfo, clusterFolder, yanoConfigDir, pastTimeTravelMode, writer); + Process process = startYanoProcess(clusterInfo, clusterFolder, yanoConfigDir, runMode, writer); if (process != null) { processes.add(process); return true; @@ -306,8 +321,42 @@ boolean slotLeaderTimeTravelEnabled(ClusterInfo clusterInfo) { return haskellNodeValidatesChain && f > 0 && f < 1.0; } + /** + * Warn when the installed Yano is not the version this DevKit pins. {@code download} keeps an existing Yano + * binary, so a DevKit upgrade does not upgrade Yano, and older releases lack features DevKit relies on + * (sparse backfill, catch-up). + */ + void warnIfNotRequiredVersion(Consumer writer) { + String installed = installedVersion(Path.of(clusterConfig.getYanoHome())); + String required = requiredYanoVersion == null ? "" : requiredYanoVersion.trim(); + if (required.isEmpty() || required.equals(installed)) + return; + + writer.accept(warn("Installed Yano is %s, this DevKit expects %s. Run 'download -c yano --overwrite' to update it.", + installed == null ? "an older release" : installed, required)); + } + + /** Version from Yano's yano-distribution-v1.json, or null when the file is missing (releases before pre15). */ + static String installedVersion(Path yanoHome) { + Path distribution = yanoHome.resolve("yano-distribution-v1.json"); + if (!Files.exists(distribution)) + return null; + try { + JsonNode version = new ObjectMapper().readTree(distribution.toFile()).get("version"); + return version != null && version.isTextual() ? version.asText() : null; + } catch (IOException e) { + log.debug("Could not read {}: {}", distribution, e.getMessage()); + return null; + } + } + + /** Quarkus environment variable for a property: {@code yano.block-producer.enabled} -> {@code YANO_BLOCK_PRODUCER_ENABLED}. */ + static String envName(String property) { + return property.replaceAll("[^A-Za-z0-9]", "_").toUpperCase(); + } + private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Path yanoConfigDir, - boolean pastTimeTravelMode, Consumer writer) + YanoRunMode runMode, Consumer writer) throws IOException, InterruptedException { Path yanoBin = Path.of(clusterConfig.getYanoHome(), "yano"); @@ -319,12 +368,16 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa // Write application.properties for Yano (persists config on disk for debugging) int backfillInterval = backfillBlockIntervalSlots(); - boolean slotLeaderTimeTravel = pastTimeTravelMode && slotLeaderTimeTravelEnabled(clusterInfo); - yanoConfigBuilder.build(clusterInfo, yanoConfigDir, yanoDataDir, yanoHistoryDir, pastTimeTravelMode, - slotLeaderTimeTravel, backfillInterval); - if (pastTimeTravelMode) + boolean slotLeaderTimeTravel = runMode == YanoRunMode.PAST_TIME_TRAVEL && slotLeaderTimeTravelEnabled(clusterInfo); + Map props = yanoConfigBuilder.properties(clusterInfo, yanoConfigDir, yanoDataDir, yanoHistoryDir, + runMode, slotLeaderTimeTravel, backfillInterval); + yanoConfigBuilder.write(props); + if (runMode == YanoRunMode.PAST_TIME_TRAVEL) writer.accept(info("Yano backfill block interval : %s", backfillInterval == 0 ? "automatic (derived from genesis)" : backfillInterval + " slots")); + if (slotLeaderTimeTravel) + writer.accept(info("Yano past-time-travel uses slot-leader checks (activeSlotsCoeff %s, mode %s)", + clusterInfo.getActiveSlotsCoeff(), pastTimeTravelSlotLeaderMode)); ProcessBuilder builder = new ProcessBuilder(); builder.directory(new File(clusterConfig.getYanoHome())); @@ -332,28 +385,7 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa // Env vars override the properties file (Quarkus priority: env > config file) // Keep them for runtime guarantee with native binaries var env = builder.environment(); - env.put("QUARKUS_PROFILE", "devnet"); - env.put("YANO_REMOTE_PROTOCOL_MAGIC", String.valueOf(clusterInfo.getProtocolMagic())); - env.put("YANO_SERVER_PORT", String.valueOf(clusterInfo.getYanoServerPort())); - env.put("QUARKUS_HTTP_PORT", String.valueOf(clusterInfo.getYanoHttpPort())); - env.put("YANO_GENESIS_SHELLEY_GENESIS_FILE", yanoConfigDir.resolve("shelley-genesis.json").toAbsolutePath().toString()); - env.put("YANO_GENESIS_BYRON_GENESIS_FILE", yanoConfigDir.resolve("byron-genesis.json").toAbsolutePath().toString()); - env.put("YANO_GENESIS_ALONZO_GENESIS_FILE", yanoConfigDir.resolve("alonzo-genesis.json").toAbsolutePath().toString()); - env.put("YANO_GENESIS_CONWAY_GENESIS_FILE", yanoConfigDir.resolve("conway-genesis.json").toAbsolutePath().toString()); - env.put("YANO_GENESIS_PROTOCOL_PARAMETERS_FILE", yanoConfigDir.resolve("protocol-param.json").toAbsolutePath().toString()); - env.put("YANO_BLOCK_PRODUCER_VRF_SKEY_FILE", yanoConfigDir.resolve("vrf.skey").toAbsolutePath().toString()); - env.put("YANO_BLOCK_PRODUCER_KES_SKEY_FILE", yanoConfigDir.resolve("kes.skey").toAbsolutePath().toString()); - env.put("YANO_BLOCK_PRODUCER_OPCERT_FILE", yanoConfigDir.resolve("opcert.cert").toAbsolutePath().toString()); - env.put("YANO_STORAGE_PATH", yanoDataDir.toAbsolutePath().toString()); - - if (pastTimeTravelMode) { - env.put("YANO_BLOCK_PRODUCER_PAST_TIME_TRAVEL_MODE", "true"); - if (slotLeaderTimeTravel) { - env.put("YANO_BLOCK_PRODUCER_PAST_TIME_TRAVEL_SLOT_LEADER_MODE", "true"); - writer.accept(info("Yano past-time-travel uses slot-leader checks (activeSlotsCoeff %s, mode %s)", - clusterInfo.getActiveSlotsCoeff(), pastTimeTravelSlotLeaderMode)); - } - } + props.forEach((key, value) -> env.put(envName(key), value)); builder.command(yanoBin.toAbsolutePath().toString()); @@ -438,21 +470,33 @@ private void deleteIfPresent(Path path, String label) { } public boolean stop() { + return stop(ConsoleWriter::writeLn); + } + + /** Stop Yano without console output, for the Yano restarts inside a catch-up. */ + public boolean stopQuietly() { + return stop(msg -> log.debug(msg)); + } + + private boolean stop(Consumer out) { try { if (processes != null && !processes.isEmpty()) - writeLn(info("Trying to stop Yano ...")); + out.accept(info("Trying to stop Yano ...")); for (Process process : processes) { if (process != null && process.isAlive()) { - process.descendants().forEach(ph -> { - ph.destroyForcibly(); - }); - process.destroyForcibly(); - process.waitFor(15, TimeUnit.SECONDS); + // SIGTERM first so Yano closes its chainstate cleanly; force only if it does not exit + process.descendants().forEach(ProcessHandle::destroy); + process.destroy(); + if (!process.waitFor(10, TimeUnit.SECONDS)) { + process.descendants().forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + process.waitFor(15, TimeUnit.SECONDS); + } if (!process.isAlive()) { - writeLn(success("Yano stopped")); + out.accept(success("Yano stopped")); } else { - writeLn(error("Yano process could not be killed")); + out.accept(error("Yano process could not be killed")); } } } @@ -462,7 +506,7 @@ public boolean stop() { logs.clear(); } catch (Exception e) { log.error("Error stopping Yano", e); - writeLn(error("Yano could not be stopped: " + e.getMessage())); + out.accept(error("Yano could not be stopped: " + e.getMessage())); return false; } finally { processes.clear(); diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java new file mode 100644 index 00000000..a7330216 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java @@ -0,0 +1,93 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class ChainLagTest { + // Default devnet: 600-slot epochs, k = 100, f = 1, 1 s slots -> forecast window 300 slots + private static ChainLag lag(long tipSlot, long wallClockSlot) { + return new ChainLag(tipSlot, wallClockSlot, 300, 600, 1.0); + } + + @Test + void forecastWindowIsThreeKOverF() { + assertThat(ChainLag.forecastWindowSlots(100, 1.0)).isEqualTo(300); + assertThat(ChainLag.forecastWindowSlots(7, 1.0)).isEqualTo(21); + assertThat(ChainLag.forecastWindowSlots(100, 0.5)).isEqualTo(600); + assertThat(ChainLag.forecastWindowSlots(2160, 0.05)).isEqualTo(129600); + assertThat(ChainLag.forecastWindowSlots(0, 1.0)).isZero(); + assertThat(ChainLag.forecastWindowSlots(100, 0)).isZero(); + } + + @Test + void wallClockSlotCountsSlotsSinceTheStartTime() { + long start = 1_790_000_000L; + assertThat(ChainLag.wallClockSlot(start, 1.0, start * 1000 + 42_500)).isEqualTo(42); + assertThat(ChainLag.wallClockSlot(start, 0.2, start * 1000 + 1_000)).isEqualTo(5); + assertThat(ChainLag.wallClockSlot(start, 1.0, start * 1000 - 5_000)).isZero(); + assertThat(ChainLag.wallClockSlot(start, 0, start * 1000 + 5_000)).isZero(); + } + + @Test + void ofReadsTheClusterInfo() { + ClusterInfo info = ClusterInfo.builder() + .startTime(1_790_000_000L).slotLength(1.0).epochLength(600).securityParam(100).activeSlotsCoeff(1.0) + .build(); + + ChainLag lag = ChainLag.of(info, 1000, 1_790_000_000_000L + 2_000_000); + + assertThat(lag.wallClockSlot()).isEqualTo(2000); + assertThat(lag.lagSlots()).isEqualTo(1000); + assertThat(lag.forecastWindowSlots()).isEqualTo(300); + } + + @Test + void stalledOnceTheLagReachesTheForecastWindow() { + assertThat(lag(1000, 1299).stalled()).isFalse(); + assertThat(lag(1000, 1300).stalled()).isTrue(); + assertThat(lag(1000, 900).stalled()).isFalse(); + assertThat(lag(1000, 900).lagSlots()).isZero(); + } + + @Test + void startCatchesUpPastThreeQuartersOfTheWindow() { + assertThat(lag(1000, 1225).needsCatchUpOnStart()).isFalse(); + assertThat(lag(1000, 1226).needsCatchUpOnStart()).isTrue(); + assertThat(new ChainLag(0, 10_000, 0, 600, 1.0).needsCatchUpOnStart()).isFalse(); + } + + @Test + void countsEpochBoundariesBetweenTipAndWallClock() { + assertThat(lag(1000, 1100).epochsToCross()).isZero(); + assertThat(lag(1100, 1300).epochsToCross()).isEqualTo(1); + assertThat(lag(1973, 2608).epochsToCross()).isEqualTo(1); + assertThat(lag(0, 6000).epochsToCross()).isEqualTo(10); + } + + @Test + void estimatesOneBlockPerWindowPlusEpochBoundariesPlusTheTarget() { + // The 2026-09-30 run: 635 slots from 1973 produced 3 blocks (2272, 2400, 2608) + assertThat(lag(1973, 2608).estimatedBackfillBlocks()).isEqualTo(635 / 299 + 1 + 1); + assertThat(lag(1000, 1000).estimatedBackfillBlocks()).isZero(); + // Three days on the default devnet + assertThat(lag(0, 259_200).estimatedBackfillBlocks()).isEqualTo(259_200 / 299 + 432 + 1); + } + + @Test + void idleTimeUsesTheSlotLength() { + assertThat(lag(0, 90).idleTime()).isEqualTo(Duration.ofSeconds(90)); + assertThat(new ChainLag(0, 90, 21, 40, 0.2).idleTime()).isEqualTo(Duration.ofMillis(18_000)); + } + + @Test + void idleTimeTextShowsTheTwoLargestUnits() { + assertThat(ChainLag.durationText(Duration.ofSeconds(40))).isEqualTo("40s"); + assertThat(ChainLag.durationText(Duration.ofSeconds(432))).isEqualTo("7m 12s"); + assertThat(ChainLag.durationText(Duration.ofMinutes(125))).isEqualTo("2h 5m"); + assertThat(ChainLag.durationText(Duration.ofHours(76).plusMinutes(12))).isEqualTo("3d 4h"); + } +} diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiterTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiterTest.java index 71ec9122..0f497a91 100644 --- a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiterTest.java +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiterTest.java @@ -155,6 +155,47 @@ void firstProgressReportMeasuresTheRateFromTheFirstTip() throws Exception { assertThat(progress).allSatisfy(line -> assertThat(line).endsWith("(100 blocks/s)")); } + @Test + void awaitSlotReturnsSyncedOnceTheTipReachesTheSlot() throws Exception { + FakeTime time = new FakeTime(); + RelaySyncWaiter waiter = new RelaySyncWaiter(30_000, 0, time::read, time::sleep); + + RelaySyncWaiter.Outcome outcome = waiter.awaitSlot(1250, EPOCH_LENGTH, "Yano sync", + script(List.of(tip(5, 1100), tip(8, 1249), tip(9, 1250))), msg -> {}); + + assertThat(outcome.synced()).isTrue(); + assertThat(outcome.slot()).isEqualTo(1250); + assertThat(outcome.epoch()).isEqualTo(2); + } + + @Test + void awaitSlotReportsProgressWithItsLabel() throws Exception { + FakeTime time = new FakeTime(); + RelaySyncWaiter waiter = new RelaySyncWaiter(30_000, 0, time::read, time::sleep); + List> tips = new ArrayList<>(); + for (long i = 1; i <= 10; i++) { + tips.add(tip(i * 10, i * 10)); + } + List progress = new ArrayList<>(); + + waiter.awaitSlot(100, EPOCH_LENGTH, "Haskell sync", script(tips), progress::add); + + assertThat(progress).isNotEmpty(); + assertThat(progress.get(0)).startsWith("Haskell sync in progress: slot").contains("of 100"); + } + + @Test + void awaitSlotGivesUpWhenTheTipStalls() throws Exception { + FakeTime time = new FakeTime(); + RelaySyncWaiter waiter = new RelaySyncWaiter(30_000, 0, time::read, time::sleep); + + RelaySyncWaiter.Outcome outcome = waiter.awaitSlot(5000, EPOCH_LENGTH, "Yano sync", + () -> tip(40, 400), msg -> {}); + + assertThat(outcome.synced()).isFalse(); + assertThat(outcome.reason()).contains("did not advance for 30s").contains("slot 400"); + } + @Test void unusableStallTimeoutFallsBackToTheDefault() { assertThat(RelaySyncWaiter.stallTimeoutSeconds(45)).isEqualTo(45); diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java new file mode 100644 index 00000000..7b1a1922 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java @@ -0,0 +1,80 @@ +package com.bloxbean.cardano.yacicli.localcluster.yano; + +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class YanoRunModeConfigTest { + private static final Path DIR = Path.of("/tmp/devnet"); + + private static Map props(YanoRunMode mode) { + ClusterInfo info = ClusterInfo.builder().nodePort(3001).protocolMagic(42).build(); + return new YanoConfigBuilder(null).properties(info, DIR.resolve("config"), DIR.resolve("data"), + DIR.resolve("history"), mode, false, 0); + } + + @Test + void liveModeSetsNoModeSpecificProperties() { + Map props = props(YanoRunMode.LIVE); + + assertThat(props).doesNotContainKeys("yano.block-producer.past-time-travel-mode", + "yano.block-producer.block-time-millis", "yano.client.enabled", "yano.dev-mode", + "yano.block-producer.backfill-block-interval-slots"); + assertThat(props).containsEntry("yano.storage.path", DIR.resolve("data").toAbsolutePath().toString()); + } + + @Test + void pastTimeTravelModeCarriesTheBackfillInterval() { + Map props = props(YanoRunMode.PAST_TIME_TRAVEL); + + assertThat(props).containsEntry("yano.block-producer.past-time-travel-mode", "true") + .containsEntry("yano.block-producer.backfill-block-interval-slots", "0"); + } + + @Test + void catchUpModeForgesNothingOnItsOwnAndBackfillsSparsely() { + Map props = props(YanoRunMode.CATCH_UP); + + assertThat(props).containsEntry("yano.block-producer.block-time-millis", "3600000") + .containsEntry("yano.block-producer.backfill-block-interval-slots", "0") + .doesNotContainKeys("yano.block-producer.past-time-travel-mode", "yano.client.enabled"); + } + + @Test + void followModeSyncsFromTheHaskellNodeWithoutProducing() { + Map props = props(YanoRunMode.FOLLOW); + + assertThat(props).containsEntry("yano.client.enabled", "true") + .containsEntry("yano.remote.host", "127.0.0.1") + .containsEntry("yano.remote.port", "3001") + .containsEntry("yano.block-producer.enabled", "false") + // Yano refuses dev mode without a block producer + .containsEntry("yano.dev-mode", "false"); + } + + @Test + void envNameFollowsQuarkusMapping() { + assertThat(YanoService.envName("yano.block-producer.block-time-millis")) + .isEqualTo("YANO_BLOCK_PRODUCER_BLOCK_TIME_MILLIS"); + assertThat(YanoService.envName("quarkus.http.port")).isEqualTo("QUARKUS_HTTP_PORT"); + assertThat(YanoService.envName("yano.dev-mode")).isEqualTo("YANO_DEV_MODE"); + } + + @Test + void installedVersionComesFromTheDistributionFile(@TempDir Path yanoHome) throws Exception { + assertThat(YanoService.installedVersion(yanoHome)).isNull(); + + Files.writeString(yanoHome.resolve("yano-distribution-v1.json"), + "{\"schemaVersion\": 1, \"product\": \"yano\", \"version\": \"0.1.0-pre17\"}"); + assertThat(YanoService.installedVersion(yanoHome)).isEqualTo("0.1.0-pre17"); + + Files.writeString(yanoHome.resolve("yano-distribution-v1.json"), "not json"); + assertThat(YanoService.installedVersion(yanoHome)).isNull(); + } +} diff --git a/docs/pages/commands.mdx b/docs/pages/commands.mdx index caca996f..40ff13b0 100644 --- a/docs/pages/commands.mdx +++ b/docs/pages/commands.mdx @@ -36,6 +36,22 @@ reset If the devnet gets stuck or you simply want to reset the data and restart with the same configuration, simply use the command "reset". It will restore your devnet to its initial state, allowing you to continue your development seamlessly. +### catch-up + +Catch the devnet up to wall clock after the machine slept or the devnet was paused for longer than its stability +window. The chain is kept (contracts, wallets, stake), and every epoch that passed is processed. Companion and +yano-only devnets only. + +- **Usage:** +```bash +catch-up [--force] +``` +- **Options:** + - `--force`: catch up even when the chain is not stalled. + +DevKit already does this on `start` and while the devnet runs, unless `devnet.auto.catch.up=false`. See +[Resuming After a Long Pause](./yano-node-modes#resuming-after-a-long-pause). + ### stop Stop the local devnet. diff --git a/docs/pages/yano-node-modes.mdx b/docs/pages/yano-node-modes.mdx index ffacd5d9..42d1912d 100644 --- a/docs/pages/yano-node-modes.mdx +++ b/docs/pages/yano-node-modes.mdx @@ -109,6 +109,67 @@ Use Yano-only mode when you want a lightweight devnet for application, SDK, and Yano-only mode is optimized for local development speed. If your workflow needs direct `cardano-node` socket access, use companion mode. +## Resuming After a Long Pause + +Close the laptop for the weekend, or stop DevKit and come back days later, and you keep the same devnet: +your contracts, wallets, stake and governance state are still there. + +Without help, a paused devnet would not produce blocks again. A Haskell block producer can only forge while +its chain tip is less than one stability window (`3k/f` slots) behind wall clock: 5 minutes on the default +devnet, 21 seconds with `--epoch-length 40`. DevKit notices a pause longer than that and catches the chain up +with Yano: + +1. The Haskell node restarts as a relay and Yano syncs its chain. +2. Yano fills the gap up to wall clock with empty blocks. The blocks are sparse (one per stability window) but + every epoch boundary gets one, so rewards, snapshots and governance run for every epoch that passed. +3. The Haskell node syncs those blocks and restarts as the block producer. + +```text +Devnet was idle for 3d 4h (tip slot 1,830, now slot 275,562, 457 epochs behind) +Catching up to wall clock with Yano (sparse backfill, about 1,380 blocks) ... +✅ Yano synced the chain from the Haskell node (tip slot 1,830) +✅ Backfilled to slot 275,561 (1,372 blocks) +✅ Haskell node synced the backfill +✅ Devnet resumed in 38s at slot 275,570, epoch 459. Your contracts, wallets and stake are unchanged; 457 epochs passed. +``` + +This happens on its own: + +- **On `start`**, before Yaci Store and the other services start. +- **While DevKit keeps running** (lid closed, machine asleep): a watchdog checks the chain every 30 seconds. + +In yano-only mode, `start` backfills before Yano produces its first live block. + +You can also run it yourself, or check how far behind the chain is: + +```shell +devnet:default> catch-up +``` + +```shell +curl http://localhost:10000/local-cluster/api/admin/devnet/chain-lag +curl -X POST http://localhost:10000/local-cluster/api/admin/devnet/catch-up +``` + +To only report the stall and catch up by hand, set this in `config/application.properties`: + +```properties +devnet.auto.catch.up=false +``` + + +Time moves forward to wall clock, so every epoch that passed is processed. Governance actions and DRep activity +can expire while the devnet sleeps, just as they would on a real network. + + +Limitations: + +- Companion devnets need `activeSlotsCoeff` 1 (block time equal to slot length). +- Local multi-node, `yano-primary` and `haskell-only` devnets are not supported. +- If DevKit keeps running in yano-only mode, Yano produces a block as soon as the machine wakes, before DevKit can + react. That block jumps over the missed epochs. Stop the devnet before a long pause, so the next `start` catches + up first. + ## Configuration For Docker and zip distributions, set the mode in `config/node.properties` before creating the devnet: From cd4a3374688b6d42018ee6dfcf23ce96a76670d5 Mon Sep 17 00:00:00 2001 From: Satya Date: Thu, 1 Oct 2026 23:47:13 +0800 Subject: [PATCH 2/9] Show long-running CLI work as steps with progress bars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A catch-up restarts the node twice and can take a while; the console used to print a handful of ✅ lines with silences in between, plus leftover noise ("Waiting for node socket file ...", "Deleted pid file : yano.pid"), and a watchdog catch-up was printed straight after the shell prompt. New util.progress.ConsoleProgress renders work as numbered steps: [1/4] ✅ Haskell node → relay ................. tip slot 1,067 (1s) [2/4] ⠹ Yano following the chain ▕██████░░░░▏ 61% slot 92,310 / 150,004 a spinner while a step runs, a bar with a percentage when the total is known, and each step's duration. In a terminal the line is redrawn in place; anywhere else (Docker logs, pipes, the admin API's writer) the same steps are plain lines with progress throttled to every 5 s. YACI_PROGRESS=plain forces plain lines. Only ConsoleWriter.console() gets in-place rendering, so other writers are never fed carriage returns. Used for: - devnet catch-up (4 steps; both syncs show real percentages) - the companion first-run relay sync - Yano and Yaci Store start-up (one spinner line instead of a line per second) - Yaci Store's sync to the chain tip (indexer height / node height) - component downloads (MB / MB) The watchdog now starts below the shell prompt and redraws it when done. RelaySyncWaiter gains a numeric tip listener for the bars. Verified in a pseudo-terminal: companion create, a 70 s freeze healed by the watchdog with Ogmios, Kupo, submit-api and Yaci Store running (all reconnected; SDK e2e suites 13/13 afterwards), and an Ogmios download. Co-Authored-By: Claude Opus 5.5 --- .../commands/common/DownloadService.java | 21 ++- .../yacicli/localcluster/ClusterCommands.java | 2 +- .../yacicli/localcluster/ClusterService.java | 2 +- .../localcluster/ClusterStartService.java | 4 +- .../catchup/DevnetCatchUpService.java | 118 ++++++++----- .../catchup/DevnetStallWatchdog.java | 25 ++- .../yacistore/YaciStoreService.java | 26 +-- .../localcluster/yano/RelaySyncWaiter.java | 40 ++++- .../yano/YanoCompanionService.java | 18 +- .../localcluster/yano/YanoService.java | 14 +- .../cardano/yacicli/util/ConsoleWriter.java | 23 +++ .../cardano/yacicli/util/ProcessUtil.java | 6 +- .../util/progress/ConsoleProgress.java | 154 ++++++++++++++++ .../yacicli/util/progress/ProgressTask.java | 46 +++++ .../cardano/yacicli/util/progress/Step.java | 164 ++++++++++++++++++ .../util/progress/ConsoleProgressTest.java | 82 +++++++++ docs/pages/yano-node-modes.mdx | 18 +- 17 files changed, 674 insertions(+), 89 deletions(-) create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ProgressTask.java create mode 100644 applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/Step.java create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgressTest.java diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java index 2ae290ad..b2c32225 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java @@ -1,6 +1,8 @@ package com.bloxbean.cardano.yacicli.commands.common; import com.bloxbean.cardano.yacicli.localcluster.ClusterConfig; +import com.bloxbean.cardano.yacicli.util.progress.ConsoleProgress; +import com.bloxbean.cardano.yacicli.util.progress.Step; import lombok.RequiredArgsConstructor; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -490,29 +492,30 @@ private Path download(String component, String downloadUrl, String targetDir, St Path targetPath = Paths.get(targetDir, targetFileName); Files.createDirectories(targetPath.getParent()); + Step step = ConsoleProgress.step("Downloading " + component, console()); try (InputStream inputStream = httpConn.getInputStream(); FileOutputStream outputStream = new FileOutputStream(targetPath.toFile())) { - byte[] buffer = new byte[4096]; + byte[] buffer = new byte[65536]; int bytesRead; long totalBytesRead = 0; - int percentCompleted; while ((bytesRead = inputStream.read(buffer)) != -1) { if (Thread.currentThread().isInterrupted()) { - writeLn(warnLabel("Download", "Download interrupted by user.")); + step.fail("interrupted by user"); Files.deleteIfExists(targetPath); return null; } outputStream.write(buffer, 0, bytesRead); totalBytesRead += bytesRead; - percentCompleted = (int) ((totalBytesRead * 100) / fileSize); - write("\rDownloading: " + percentCompleted + "%"); + step.progress(totalBytesRead, fileSize, fileSize > 0 + ? megabytes(totalBytesRead) + " / " + megabytes(fileSize) + : megabytes(totalBytesRead)); } - System.out.println(); - writeLn(success("Download complete for %s!", component)); + step.done(megabytes(totalBytesRead)); } catch (Exception e) { + step.fail(e.getMessage()); e.printStackTrace(); } @@ -524,6 +527,10 @@ private Path download(String component, String downloadUrl, String targetDir, St return null; } + private static String megabytes(long bytes) { + return String.format("%.1f MB", bytes / (1024.0 * 1024.0)); + } + public void extractTarGz(String filePath, String extractDir) throws IOException { try (FileInputStream fis = new FileInputStream(filePath); BufferedInputStream bis = new BufferedInputStream(fis); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java index d3513c41..36504155 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java @@ -285,7 +285,7 @@ public void catchUp(@ShellOption(value = {"--force"}, defaultValue = "false", help = "Catch up even when the chain is not stalled") boolean force) { String clusterName = CommandContext.INSTANCE.getProperty(ClusterConfig.CLUSTER_NAME); try { - var result = localClusterService.catchUp(clusterName, force, msg -> writeLn(msg)); + var result = localClusterService.catchUp(clusterName, force, console()); if (!result.success()) writeLn(error(result.message())); else if (result.blocksProduced() == 0) diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java index 451ce14f..c18cc9ef 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java @@ -124,7 +124,7 @@ public void startClusterContext(String clusterName, Consumer writer) { public RunStatus startCluster(String clusterName) { try { - RunStatus startedSuccessfully = clusterStartService.startCluster(getClusterInfo(clusterName), getClusterFolder(clusterName), msg -> writeLn(msg)); + RunStatus startedSuccessfully = clusterStartService.startCluster(getClusterInfo(clusterName), getClusterFolder(clusterName), console()); if (startedSuccessfully.stared()) writeLn(info("Swagger Url to interact with the cluster's node : " + "http://localhost:" + server.getWebServer().getPort() +"/swagger-ui.html")); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java index a7d1aef2..d9d55c2a 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java @@ -386,8 +386,8 @@ private Process startNode(Path clusterFolder, ClusterInfo clusterInfo, boolean a int counter = 0; while (!Files.exists(nodeSocketPath) && counter < 10) { //wait 5 sec max Thread.sleep(500); - if (counter > 4) - writeLn(info("Waiting for node socket file to be created ...")); + if (counter == 5) + writer.accept(info("Waiting for node socket file to be created ...")); counter++; } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java index 0f874db7..7319aec3 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java @@ -12,6 +12,9 @@ import com.bloxbean.cardano.yacicli.localcluster.yano.YanoCompanionService; import com.bloxbean.cardano.yacicli.localcluster.yano.YanoRunMode; import com.bloxbean.cardano.yacicli.localcluster.yano.YanoService; +import com.bloxbean.cardano.yacicli.util.progress.ConsoleProgress; +import com.bloxbean.cardano.yacicli.util.progress.ProgressTask; +import com.bloxbean.cardano.yacicli.util.progress.Step; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -171,97 +174,115 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N long startedAt = System.currentTimeMillis(); int httpPort = clusterInfo.getYanoHttpPort(); long epochLength = clusterInfo.getEpochLength(); - Consumer quiet = problemsOnly(writer); - printBanner(lag, writer); + ProgressTask task = ConsoleProgress.task(null, 4, writer); + printBanner(lag, task.logWriter()); + // The node and Yano restarts inside a catch-up are not news to the user; warnings and errors still are + Consumer quiet = problemsOnly(task.logWriter()); long blocksProduced = 0; long yanoTipSlot = lag.tipSlot(); boolean handedBack = false; + Step step = null; try { // 1. Haskell node as a relay, peering with Yano: it serves its chain to Yano and pulls Yano's backfill + step = task.step("Haskell node → relay"); if (!node.stopNode(quiet)) - return Result.failed("Could not stop the Haskell node"); + return failed(step, "Could not stop the Haskell node"); yanoCompanionService.updateTopologyForYanoPeering(clusterInfo, clusterFolder, quiet); if (!node.startNode(true, quiet)) - return Result.failed("Could not start the Haskell node as a relay"); + return failed(step, "Could not start the Haskell node as a relay"); Tuple relayTip = waitForHaskellTip(clusterInfo, NODE_READY_TIMEOUT); if (relayTip == null) - return Result.failed("The Haskell relay did not answer tip queries"); + return failed(step, "The Haskell relay did not answer tip queries"); long haskellTipSlot = relayTip._2.getSlot(); + step.done(String.format("tip slot %,d", haskellTipSlot)); // 2. Yano follows the Haskell chain. It already holds the bootstrap prefix, so only the blocks forged // since the handover are fetched. + step = task.step("Yano following the chain"); yanoService.stopQuietly(); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.FOLLOW, quiet) || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) - return Result.failed("Yano did not start to follow the Haskell node (see 'yano-logs')"); + return failed(step, "Yano did not start to follow the Haskell node (see 'yano-logs')"); + Tuple yanoStart = yanoBootstrapService.getNodeTip(httpPort); + long followFrom = yanoStart != null ? yanoStart._2.getSlot() : 0; + final Step following = step; RelaySyncWaiter.Outcome followed = new RelaySyncWaiter(SYNC_STALL_TIMEOUT_MS, 0) - .awaitSlot(haskellTipSlot, epochLength, "Yano sync", - () -> yanoBootstrapService.getNodeTip(httpPort), msg -> writer.accept(info(msg))); + .awaitSlot(haskellTipSlot, epochLength, () -> yanoBootstrapService.getNodeTip(httpPort), + (slot, height, blocksPerSecond) -> following.progress(slot - followFrom, + haskellTipSlot - followFrom, slotDetail(slot, haskellTipSlot, blocksPerSecond))); if (!followed.synced()) - return Result.failed("Yano could not sync the Haskell chain: " + followed.reason()); - writer.accept(success("Yano synced the chain from the Haskell node (tip slot %,d)", haskellTipSlot)); + return failed(step, "Yano could not sync the Haskell chain: " + followed.reason()); yanoService.stopQuietly(); + step.done(String.format("at slot %,d", haskellTipSlot)); - // 3. Yano backfills to wall clock. The long block time keeps it from forging a live block at the - // wall-clock slot before the backfill runs. + // 3. Yano backfills to wall clock and the relay adopts it. The long block time keeps Yano from forging + // a live block at the wall-clock slot before the backfill runs. Repeat until the relay is within a + // quarter of the forecast window; the rest is for the producer restart. + step = task.step("Backfill to wall clock"); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, quiet) || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) - return Result.failed("Yano did not start as a block producer (see 'yano-logs')"); - - // 4. Backfill, let the relay adopt it, repeat until the relay is close to wall clock. The last round - // leaves the relay within a quarter of the forecast window, the rest is for the producer restart. + return failed(step, "Yano did not start as a block producer (see 'yano-logs')"); long slack = Math.max(2, lag.forecastWindowSlots() / 4); RelaySyncWaiter relayWaiter = new RelaySyncWaiter(SYNC_STALL_TIMEOUT_MS, 0); + final Step backfilling = step; for (int round = 1; ; round++) { JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); if (caughtUp == null) - return Result.failed("Yano's catch-up call failed (see 'yano-logs')"); + return failed(step, "Yano's catch-up call failed (see 'yano-logs')"); yanoTipSlot = caughtUp.path("new_slot").asLong(yanoTipSlot); blocksProduced += caughtUp.path("blocks_produced").asLong(0); - if (round == 1) - writer.accept(success("Backfilled to slot %,d (%,d blocks)", yanoTipSlot, blocksProduced)); - RelaySyncWaiter.Outcome adopted = relayWaiter.awaitSlot(yanoTipSlot, epochLength, "Haskell sync", - () -> haskellTip(clusterInfo), msg -> writer.accept(info(msg))); + final long target = yanoTipSlot; + final long produced = blocksProduced; + RelaySyncWaiter.Outcome adopted = relayWaiter.awaitSlot(target, epochLength, + () -> haskellTip(clusterInfo), + (slot, height, blocksPerSecond) -> backfilling.progress(slot - haskellTipSlot, + target - haskellTipSlot, String.format("%,d blocks, relay at slot %,d", produced, slot))); if (!adopted.synced()) - return Result.failed("The Haskell relay did not sync Yano's backfill: " + adopted.reason()); + return failed(step, "The Haskell relay did not sync Yano's backfill: " + adopted.reason()); long lagNow = ChainLag.wallClockSlot(clusterInfo.getStartTime(), clusterInfo.getSlotLength(), System.currentTimeMillis()) - yanoTipSlot; if (lagNow <= slack) break; if (round >= MAX_PUMP_ROUNDS) - return Result.failed("The Haskell relay could not keep up with wall clock (still " + lagNow + return failed(step, "The Haskell relay could not keep up with wall clock (still " + lagNow + " slots behind)"); } - writer.accept(success("Haskell node synced the backfill")); + step.done(String.format("%,d blocks to slot %,d", blocksProduced, yanoTipSlot)); - // 5. Hand block production back to the Haskell node, as the first-run handover does: Yano stops, the - // original topology is restored, and the node restarts as the producer. + // 4. Hand block production back to the Haskell node, as the first-run handover does: Yano stops, the + // original topology is restored, and the node restarts as the producer and forges on top. + step = task.step("Haskell node back as block producer"); yanoService.stopQuietly(); yanoCompanionService.restoreOriginalTopology(clusterFolder, quiet); boolean restarted = node.stopNode(quiet) && node.startNode(false, quiet); handedBack = true; if (!restarted) - return Result.failed("Could not restart the Haskell node as the block producer"); + return failed(step, "Could not restart the Haskell node as the block producer"); - // 6. The producer forges on top of the backfill + step.detail("waiting for the first block"); Tuple forged = waitForHaskellSlotAbove(clusterInfo, yanoTipSlot, forgeTimeout(clusterInfo)); long duration = System.currentTimeMillis() - startedAt; - if (forged == null) - return new Result(false, "The chain is caught up, but the Haskell node has not forged yet. " - + "If it does not, run 'catch-up' again.", lag.tipSlot(), yanoTipSlot, blocksProduced, duration); + if (forged == null) { + String message = "The chain is caught up, but the Haskell node has not forged yet. " + + "If it does not, run 'catch-up' again."; + step.fail("no block yet"); + task.log(warn(message)); + return new Result(false, message, lag.tipSlot(), yanoTipSlot, blocksProduced, duration); + } + step.done(String.format("forging at slot %,d", forged._2.getSlot())); String message = resumedMessage(clusterInfo, lag, forged._2.getSlot(), duration); - writer.accept(success(message)); + task.log(success(message)); return new Result(true, message, lag.tipSlot(), forged._2.getSlot(), blocksProduced, duration); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - return Result.failed("Catch-up interrupted"); + return failed(step, "Catch-up interrupted"); } catch (IOException | RuntimeException e) { log.error("Catch-up failed", e); - return Result.failed("Catch-up failed: " + e.getMessage()); + return failed(step, "Catch-up failed: " + e.getMessage()); } finally { if (!handedBack) { // Never leave the devnet half-way: Yano stopped, original topology, Haskell node as producer @@ -273,6 +294,16 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N } } + private static Result failed(Step step, String message) { + if (step != null) + step.fail(message); + return Result.failed(message); + } + + private static String slotDetail(long slot, long target, long blocksPerSecond) { + return String.format("slot %,d / %,d%s", slot, target, blocksPerSecond > 0 ? ", " + blocksPerSecond + " blocks/s" : ""); + } + // --------------------------------------------------------------------------------------------------------- // Yano-only // --------------------------------------------------------------------------------------------------------- @@ -316,17 +347,23 @@ private void catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consum // Short stops need no banner: the backfill is a block or two boolean report = lag.needsCatchUpOnStart(); + ProgressTask task = report ? ConsoleProgress.task(null, 1, writer) : null; if (report) - printBanner(lag, writer); + printBanner(lag, task.logWriter()); + Step step = report ? task.step("Backfill to wall clock") : null; JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); if (caughtUp == null) { - writer.accept(error("Yano's catch-up call failed (see 'yano-logs'); the first block will skip the gap")); + String message = "Yano's catch-up call failed (see 'yano-logs'); the first block will skip the gap"; + if (step != null) + step.fail(message); + else + writer.accept(error(message)); return; } if (report) { long toSlot = caughtUp.path("new_slot").asLong(); - writer.accept(success("Backfilled to slot %,d (%,d blocks)", toSlot, caughtUp.path("blocks_produced").asLong())); - writer.accept(success(resumedMessage(clusterInfo, lag, toSlot, System.currentTimeMillis() - startedAt))); + step.done(String.format("%,d blocks to slot %,d", caughtUp.path("blocks_produced").asLong(), toSlot)); + task.log(success(resumedMessage(clusterInfo, lag, toSlot, System.currentTimeMillis() - startedAt))); } } finally { yanoService.stopQuietly(); @@ -336,10 +373,9 @@ private void catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consum // --------------------------------------------------------------------------------------------------------- private void printBanner(ChainLag lag, Consumer writer) { - writer.accept(header(AnsiColors.CYAN_BOLD, "Devnet catch-up")); - writer.accept(info("Devnet was idle for %s (tip slot %,d, now slot %,d%s)", + writer.accept(header(AnsiColors.CYAN_BOLD, String.format("⏸ Devnet was idle for %s (tip slot %,d, now slot %,d%s).", lag.idleTimeText(), lag.tipSlot(), lag.wallClockSlot(), - lag.epochsToCross() > 0 ? String.format(", %s behind", epochs(lag.epochsToCross())) : "")); + lag.epochsToCross() > 0 ? String.format(", %s behind", epochs(lag.epochsToCross())) : ""))); writer.accept(info("Catching up to wall clock with Yano (sparse backfill, about %,d blocks) ...", lag.estimatedBackfillBlocks())); } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java index 51dbc3f6..69b39375 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java @@ -9,6 +9,7 @@ import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStopped; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; +import org.jline.reader.LineReader; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.ObjectProvider; import org.springframework.context.event.EventListener; @@ -38,6 +39,8 @@ public class DevnetStallWatchdog { // ObjectProvider: ClusterService -> ClusterStartService -> DevnetCatchUpService, and this listens to both sides private final ObjectProvider clusterServiceProvider; private final DevnetCatchUpService devnetCatchUpService; + // The interactive shell's line reader, absent in non-interactive runs + private final ObjectProvider lineReaderProvider; private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(runnable -> { Thread thread = new Thread(runnable, "devnet-stall-watchdog"); @@ -113,9 +116,29 @@ void check() throws Exception { return; } + // The shell prompt is waiting for input on the current line: start below it, redraw it when done + LineReader reader = lineReaderProvider.getIfAvailable(); + boolean atPrompt = reader != null && reader.isReading(); + if (atPrompt) + System.out.println(); + writeLn(header(AnsiColors.CYAN_BOLD, "The devnet stopped producing blocks (the machine slept?)")); - var result = clusterService.catchUp(name, false, msg -> writeLn(msg)); + var result = clusterService.catchUp(name, false, console()); if (!result.success()) writeLn(error(result.message() + " Run 'catch-up' to retry, or 'reset' to start a new chain.")); + + if (atPrompt) + redrawPrompt(reader); + } + + private static void redrawPrompt(LineReader reader) { + try { + if (reader.isReading()) { + reader.callWidget(LineReader.REDRAW_LINE); + reader.callWidget(LineReader.REDISPLAY); + } + } catch (RuntimeException e) { + log.debug("Could not redraw the prompt: {}", e.getMessage()); + } } } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java index bd8a61ac..a3c04852 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yacistore/YaciStoreService.java @@ -24,6 +24,8 @@ import com.bloxbean.cardano.yacicli.util.PortUtil; import com.bloxbean.cardano.yacicli.util.ProcessStream; import com.bloxbean.cardano.yacicli.util.ProcessUtil; +import com.bloxbean.cardano.yacicli.util.progress.ConsoleProgress; +import com.bloxbean.cardano.yacicli.util.progress.Step; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.EvictingQueue; @@ -82,7 +84,7 @@ public class YaciStoreService { public void handleClusterStarted(ClusterStarted clusterStarted) { String clusterName = clusterStarted.getClusterName(); - start(clusterName, msg -> writeLn(msg)); + start(clusterName, console()); } public boolean start(String clusterName, Consumer writer) { @@ -186,11 +188,12 @@ private void waitForSyncToTip(ClusterInfo clusterInfo, Consumer writer) ObjectMapper mapper = new ObjectMapper(); long deadlineMs = System.currentTimeMillis() + SYNC_WAIT_DEADLINE_MS; - writer.accept(info("Waiting for Yaci Store to sync to chain tip (lag tolerance " + maxLagBlocks + " blocks)...")); + Step step = ConsoleProgress.step("Yaci Store syncing to chain tip", writer); while (true) { long remaining = deadlineMs - System.currentTimeMillis(); if (remaining <= 0) { + step.fail("not at the tip after 60s"); writer.accept(warn("Yaci Store did not reach chain tip within 60s. Proceeding anyway.")); return; } @@ -204,10 +207,10 @@ private void waitForSyncToTip(ClusterInfo clusterInfo, Consumer writer) // Non-negative guard: if indexer is ahead of node (stale DB / aborted reset), // don't false-pass — keep waiting until the indexer's height makes sense. if (lag >= 0 && lag <= maxLagBlocks) { - writer.accept(success("Yaci Store synced to chain tip (indexer height " - + indexerHeight + ", node height " + nodeHeight + ", lag " + lag + ")")); + step.done("height " + indexerHeight + " (node " + nodeHeight + ", lag " + lag + ")"); return; } + step.progress(indexerHeight, nodeHeight, String.format("height %,d / %,d", indexerHeight, nodeHeight)); } // null/negative-lag cases fall through to the sleep + retry within the deadline. @@ -468,22 +471,21 @@ private StoreStartResult startStoreApp(ClusterInfo clusterInfo, Era era, Consume Process process = builder.start(); - writeLn(success("Yaci store starting ...")); AtomicBoolean started = new AtomicBoolean(false); AtomicBoolean intersectNotFoundAlreadyShown = new AtomicBoolean(false); + Step step = ConsoleProgress.step("Starting Yaci Store", writer); ProcessStream processStream = new ProcessStream(process.getInputStream(), line -> { logs.add(line); if (line != null && line.contains("Started YaciStoreApplication")) { - writeLn(infoLabel("OK", "Yaci Store Started")); started.set(true); } if (line != null && customDBHelper.getStoreDbUrl() != null && !customDBHelper.getStoreDbUrl().isEmpty()) { if (!intersectNotFoundAlreadyShown.get() && line.contains("IntersactNotFound")) { - writeLn(warn("Looks like some issue while starting yaci store.")); - writeLn(warn("Please check the logs for more details. Command: yaci-store-logs")); - writeLn(warn("Please verify if you are using an empty schema while creating a new devnet.")); + step.log(warn("Looks like some issue while starting yaci store.")); + step.log(warn("Please check the logs for more details. Command: yaci-store-logs")); + step.log(warn("Please verify if you are using an empty schema while creating a new devnet.")); intersectNotFoundAlreadyShown.set(true); } } @@ -496,10 +498,12 @@ private StoreStartResult startStoreApp(ClusterInfo clusterInfo, Era era, Consume if (started.get()) break; Thread.sleep(1000); - writeLn("Waiting for Yaci Store to start ..."); } - if (!started.get()) { + if (started.get()) { + step.done("started"); + } else { + step.fail("not started after 40s"); writeLn(error("Waited too long. Could not start Yaci Store. Something is wrong..")); writeLn(error("Use \"yaci-store-logs\" to see the logs")); writeLn(error("Please verify if another yaci-store in running in the same port. " + diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java index 66302890..29a9aed3 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/RelaySyncWaiter.java @@ -95,7 +95,7 @@ public Outcome await(long targetEpoch, long epochLength, TipReader tipReader, Co (slot, height, blocksPerSecond) -> String.format( "Relay sync in progress: epoch %d of %d, slot %d, height %d (%d blocks/s)", slot / epochLength, targetEpoch, slot, height, blocksPerSecond), - tipReader, progress); + tipReader, progress, null); } /** @@ -113,7 +113,33 @@ public Outcome awaitSlot(long targetSlot, long epochLength, String label, TipRea (slot, height, blocksPerSecond) -> String.format( "%s in progress: slot %d of %d, height %d (%d blocks/s)", label, slot, targetSlot, height, blocksPerSecond), - tipReader, progress); + tipReader, progress, null); + } + + /** + * Like {@link #awaitSlot(long, long, String, TipReader, Consumer)}, but reports every tip read to + * {@code listener} instead of writing progress lines, for a progress bar. + */ + public Outcome awaitSlot(long targetSlot, long epochLength, TipReader tipReader, TipListener listener) + throws InterruptedException { + return awaitTip(slot -> slot >= targetSlot, Math.max(1, epochLength), null, tipReader, msg -> {}, listener); + } + + /** + * Like {@link #await(long, long, TipReader, Consumer)}, but reports every tip read to {@code listener} instead + * of writing progress lines, for a progress bar. + */ + public Outcome await(long targetEpoch, long epochLength, TipReader tipReader, TipListener listener) + throws InterruptedException { + if (epochLength <= 0) { + return new Outcome(false, -1, -1, -1, "epoch length is not known"); + } + return awaitTip(slot -> slot / epochLength >= targetEpoch, epochLength, null, tipReader, msg -> {}, listener); + } + + /** Receives every tip read during a wait. */ + public interface TipListener { + void tip(long slot, long height, long blocksPerSecond); } private interface ProgressLine { @@ -121,7 +147,7 @@ private interface ProgressLine { } private Outcome awaitTip(LongPredicate reached, long epochLength, ProgressLine progressLine, TipReader tipReader, - Consumer progress) throws InterruptedException { + Consumer progress, TipListener listener) throws InterruptedException { final long startedAt = clock.getAsLong(); long lastProgressAt = startedAt; long lastReportAt = startedAt; @@ -150,11 +176,17 @@ private Outcome awaitTip(LongPredicate reached, long epochLength, ProgressLine p lastReportAt = now; } + if (listener != null) { + long elapsedMs = Math.max(1, now - lastReportAt); + long blocks = lastReportHeight >= 0 && height >= 0 ? height - lastReportHeight : 0; + listener.tip(slot, height, blocks * 1000 / elapsedMs); + } + if (reached.test(slot)) { return new Outcome(true, epoch, slot, height, null); } - if (now - lastReportAt >= PROGRESS_REPORT_INTERVAL_MS) { + if (progressLine != null && now - lastReportAt >= PROGRESS_REPORT_INTERVAL_MS) { long elapsedMs = Math.max(1, now - lastReportAt); long blocksSinceReport = lastReportHeight >= 0 && height >= 0 ? height - lastReportHeight : 0; long blocksPerSecond = blocksSinceReport * 1000 / elapsedMs; diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java index fd86355a..38f0aa5e 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java @@ -8,6 +8,8 @@ import com.bloxbean.cardano.yacicli.localcluster.yano.bootstrap.ChainBootstrapContext; import com.bloxbean.cardano.yacicli.localcluster.yano.bootstrap.ChainBootstrapRunner; import com.bloxbean.cardano.yaci.core.protocol.chainsync.messages.Point; +import com.bloxbean.cardano.yacicli.util.progress.ConsoleProgress; +import com.bloxbean.cardano.yacicli.util.progress.Step; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -252,8 +254,6 @@ public void updateTopologyForYanoPeering(ClusterInfo clusterInfo, Path clusterFo * @param writer console output */ public void performHandover(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { - writer.accept(info("Waiting for Haskell node to sync from Yano...")); - try { // Wait for node socket to appear (Haskell node is ready) Path socketPath = Path.of(clusterInfo.getSocketPath()); @@ -280,23 +280,23 @@ public void performHandover(ClusterInfo clusterInfo, Path clusterFolder, Consume writer.accept(warn("yano.relay.sync.max.wait.seconds must be 0 (no ceiling) or positive, got %d. Using %d.", relaySyncMaxWaitSeconds, maxWaitSeconds)); - String ceiling = maxWaitSeconds > 0 ? ", at most " + maxWaitSeconds + "s in total" : ""; - writer.accept(info("Waiting for relay to sync past epoch %d (giving up if the tip stalls for %ds%s)...", - BOOTSTRAP_EPOCH_SHIFT, stallTimeoutSeconds, ceiling)); - final ClusterUtilService clusterUtilService = clusterUtilServiceProvider.getObject(); // ClusterUtilService.getTip prints "Find tip error ..." directly via static // ConsoleWriter on exception (bypassing the consumer); transient noise during // the first one or two polls after the socket appears is expected. RelaySyncWaiter waiter = new RelaySyncWaiter(stallTimeoutSeconds * 1000L, maxWaitSeconds * 1000L); + long targetSlot = BOOTSTRAP_EPOCH_SHIFT * epochLength; + Step step = ConsoleProgress.step(String.format("Haskell relay syncing to epoch %d", BOOTSTRAP_EPOCH_SHIFT), + writer); RelaySyncWaiter.Outcome outcome = waiter.await(BOOTSTRAP_EPOCH_SHIFT, epochLength, () -> clusterUtilService.getTip(msg -> {}), - msg -> writer.accept(info(msg))); + (slot, height, blocksPerSecond) -> step.progress(slot, targetSlot, String.format( + "height %,d%s", height, blocksPerSecond > 0 ? ", " + blocksPerSecond + " blocks/s" : ""))); if (outcome.synced()) { - writer.accept(success("Relay synced to epoch " + outcome.epoch() - + " (slot " + outcome.slot() + ", height " + outcome.height() + ")")); + step.done("epoch " + outcome.epoch() + " (slot " + outcome.slot() + ", height " + outcome.height() + ")"); } else { + step.fail("stopped at slot " + outcome.slot()); writer.accept(warn("Relay sync did not reach epoch " + BOOTSTRAP_EPOCH_SHIFT + ": " + outcome.reason() + ". Proceeding anyway.")); writer.accept(warn("If the Haskell node forges no blocks after the restart, its tip is outside " diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java index 750aac2c..ddca3fb8 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java @@ -9,6 +9,8 @@ import com.bloxbean.cardano.yacicli.util.PortUtil; import com.bloxbean.cardano.yacicli.util.ProcessStream; import com.bloxbean.cardano.yacicli.util.ProcessUtil; +import com.bloxbean.cardano.yacicli.util.progress.ConsoleProgress; +import com.bloxbean.cardano.yacicli.util.progress.Step; import com.google.common.collect.EvictingQueue; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -390,8 +392,9 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa builder.command(yanoBin.toAbsolutePath().toString()); Process process = builder.start(); - writer.accept(info("Starting Yano (n2n port: %d, HTTP port: %d) ...", - clusterInfo.getYanoServerPort(), clusterInfo.getYanoHttpPort())); + Step step = ConsoleProgress.step(String.format("Starting Yano (%s, n2n port %d, HTTP port %d)", + runMode.name().toLowerCase().replace('_', '-'), clusterInfo.getYanoServerPort(), + clusterInfo.getYanoHttpPort()), writer); AtomicBoolean started = new AtomicBoolean(false); ProcessStream processStream = new ProcessStream(process.getInputStream(), line -> { @@ -416,19 +419,20 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa counter++; if (started.get()) break; if (!process.isAlive()) { + step.fail("exited"); writer.accept(error("Yano process exited unexpectedly. Check logs with 'yano-logs'")); return null; } Thread.sleep(1000); - writer.accept("Waiting for Yano to start ..."); } if (!started.get()) { + step.fail("not started after 30s"); writer.accept(error("Yano did not start within timeout. Check logs with 'yano-logs'")); return null; } - writer.accept(success("Yano started successfully")); + step.done("started"); processUtil.createProcessId(YANO_PROCESS_NAME, process); return process; } @@ -500,7 +504,7 @@ private boolean stop(Consumer out) { } } } - processUtil.deletePidFile(YANO_PROCESS_NAME); + processUtil.deletePidFile(YANO_PROCESS_NAME, out); executors.forEach(ExecutorService::shutdownNow); executors.clear(); logs.clear(); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ConsoleWriter.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ConsoleWriter.java index 73fd9855..3b94489a 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ConsoleWriter.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ConsoleWriter.java @@ -2,7 +2,30 @@ import com.bloxbean.cardano.yacicli.common.AnsiColors; +import java.util.function.Consumer; + public class ConsoleWriter { + /** + * A writer that prints to the console. Progress output ({@code util.progress}) redraws its line in place only + * for this writer, and only when stdout is a terminal. + */ + public static final class ConsoleSink implements Consumer { + private static final ConsoleSink INSTANCE = new ConsoleSink(); + + private ConsoleSink() { + } + + @Override + public void accept(String line) { + writeLn(line); + } + } + + /** The console as a {@code Consumer} writer. */ + public static Consumer console() { + return ConsoleSink.INSTANCE; + } + public static void writeLn(String str, Object...args) { if (args.length != 0) System.out.println(String.format(str, args)); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java index b77a69cf..a144c3d5 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java @@ -139,6 +139,10 @@ public String createProcessId(String processName, Process process) { } public void deletePidFile(String processName) { + deletePidFile(processName, ConsoleWriter::writeLn); + } + + public void deletePidFile(String processName, Consumer writer) { var yaciCliHome = clusterConfig.getYaciCliHome(); // Validate the yaciCliHome directory Path homePath = Paths.get(yaciCliHome); @@ -147,7 +151,7 @@ public void deletePidFile(String processName) { var pidPath = pids.resolve(processName + ".pid"); if (Files.exists(pidPath)) { pidPath.toFile().delete(); - writeLn(info("Deleted pid file : " + processName + ".pid")); + writer.accept(info("Deleted pid file : " + processName + ".pid")); } } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java new file mode 100644 index 00000000..4edf1a24 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java @@ -0,0 +1,154 @@ +package com.bloxbean.cardano.yacicli.util.progress; + +import com.bloxbean.cardano.yacicli.util.ConsoleWriter; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; + +/** + * Progress output for long-running CLI work: numbered steps ({@code [2/4] Yano following the chain}), a spinner + * while a step runs, a bar with a percentage when the total is known, and the step's duration when it is done. + *

+ * In a terminal the current step is redrawn in place. Anywhere else (Docker logs, pipes, the admin API's writer) + * the same information is written as plain lines, with progress updates throttled, so logs stay readable. + *

+ * In-place rendering needs the console: pass {@link ConsoleWriter#console()} as the writer. Any other writer gets + * plain lines. + */ +public final class ConsoleProgress { + static final long PLAIN_PROGRESS_INTERVAL_MS = 5_000; + static final long SPINNER_INTERVAL_MS = 120; + + private static final Object LOCK = new Object(); + private static volatile ScheduledExecutorService spinner; + + private ConsoleProgress() { + } + + /** + * A task made of a known number of steps. The title is written first. + */ + public static ProgressTask task(String title, int totalSteps, Consumer writer) { + ProgressTask task = new ProgressTask(totalSteps, sinkFor(writer)); + if (title != null) + task.log(title); + return task; + } + + /** A single step without a {@code [i/n]} prefix, for one long wait. */ + public static Step step(String label, Consumer writer) { + return new ProgressTask(0, sinkFor(writer)).step(label); + } + + static Sink sinkFor(Consumer writer) { + if (writer instanceof ConsoleWriter.ConsoleSink && interactiveTerminal()) + return new TerminalSink(); + return new PlainSink(writer); + } + + /** + * Whether stdout is an interactive terminal that can redraw a line. {@code YACI_PROGRESS=plain} (env or system + * property) forces plain lines, e.g. for a terminal that does not handle carriage returns. + */ + static boolean interactiveTerminal() { + String mode = System.getProperty("yaci.progress", System.getenv("YACI_PROGRESS")); + if ("plain".equalsIgnoreCase(mode)) + return false; + if ("dumb".equalsIgnoreCase(System.getenv("TERM"))) + return false; + return System.console() != null; + } + + static Object lock() { + return LOCK; + } + + static ScheduledExecutorService spinner() { + if (spinner == null) { + synchronized (LOCK) { + if (spinner == null) + spinner = Executors.newSingleThreadScheduledExecutor(runnable -> { + Thread thread = new Thread(runnable, "console-progress"); + thread.setDaemon(true); + return thread; + }); + } + } + return spinner; + } + + /** Where progress goes: the terminal (line redrawn in place) or a plain line writer. */ + interface Sink { + boolean inPlace(); + + /** Write a complete line (above the live step line, if any). */ + void line(String text); + + /** Replace the live step line. Only used when {@link #inPlace()}. */ + default void live(String text) { + } + + /** Remove the live step line. Only used when {@link #inPlace()}. */ + default void clearLive() { + } + } + + static final class PlainSink implements Sink { + private final Consumer writer; + + PlainSink(Consumer writer) { + this.writer = writer; + } + + @Override + public boolean inPlace() { + return false; + } + + @Override + public void line(String text) { + writer.accept(text); + } + } + + static final class TerminalSink implements Sink { + private static final String CLEAR_LINE = "\r\033[K"; + private boolean liveShown; + + @Override + public boolean inPlace() { + return true; + } + + @Override + public void line(String text) { + synchronized (LOCK) { + if (liveShown) + System.out.print(CLEAR_LINE); + System.out.println(text); + liveShown = false; + System.out.flush(); + } + } + + @Override + public void live(String text) { + synchronized (LOCK) { + System.out.print(CLEAR_LINE + text); + liveShown = true; + System.out.flush(); + } + } + + @Override + public void clearLive() { + synchronized (LOCK) { + if (liveShown) + System.out.print(CLEAR_LINE); + liveShown = false; + System.out.flush(); + } + } + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ProgressTask.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ProgressTask.java new file mode 100644 index 00000000..cb547bea --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ProgressTask.java @@ -0,0 +1,46 @@ +package com.bloxbean.cardano.yacicli.util.progress; + +import java.util.function.Consumer; + +/** + * A task of numbered steps, rendered as {@code [i/n] label}. Create one with + * {@link ConsoleProgress#task(String, int, Consumer)}. + */ +public final class ProgressTask { + private final int totalSteps; + private final ConsoleProgress.Sink sink; + private int stepNumber; + private Step current; + + ProgressTask(int totalSteps, ConsoleProgress.Sink sink) { + this.totalSteps = totalSteps; + this.sink = sink; + } + + /** Start the next step. A step still running is marked done first. */ + public synchronized Step step(String label) { + if (current != null && current.isRunning()) + current.done("done"); + stepNumber++; + String prefix = totalSteps > 0 ? String.format(" [%d/%d] ", stepNumber, totalSteps) : " "; + current = new Step(prefix, label, sink); + current.start(); + return current; + } + + /** Write a line above the running step (a warning, the summary), keeping the step line intact. */ + public void log(String line) { + Step step = current; + if (step != null && step.isRunning()) + step.log(line); + else + sink.line(line); + } + + /** + * A writer that prints above the running step, for code that reports through a {@code Consumer}. + */ + public Consumer logWriter() { + return this::log; + } +} diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/Step.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/Step.java new file mode 100644 index 00000000..080b8554 --- /dev/null +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/Step.java @@ -0,0 +1,164 @@ +package com.bloxbean.cardano.yacicli.util.progress; + +import com.bloxbean.cardano.yacicli.common.AnsiColors; + +import java.time.Duration; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * One step of a {@link ProgressTask}: a spinner while it runs, a bar with a percentage once + * {@link #progress(long, long, String)} reports a total, and a final line with the result and duration. + */ +public final class Step { + private static final String[] SPINNER = {"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}; + private static final int BAR_WIDTH = 20; + private static final int LABEL_WIDTH = 38; + private static final int MAX_DETAIL = 48; + + private final String prefix; + private final String label; + private final ConsoleProgress.Sink sink; + private final long startedAt = System.currentTimeMillis(); + + private volatile boolean running; + private volatile long done; + private volatile long total; + private volatile String detail = ""; + private int frame; + private long lastPlainReportAt; + private ScheduledFuture animation; + + Step(String prefix, String label, ConsoleProgress.Sink sink) { + this.prefix = prefix; + this.label = label; + this.sink = sink; + } + + void start() { + running = true; + lastPlainReportAt = startedAt; + if (sink.inPlace()) { + redraw(); + animation = ConsoleProgress.spinner().scheduleAtFixedRate(this::redraw, + ConsoleProgress.SPINNER_INTERVAL_MS, ConsoleProgress.SPINNER_INTERVAL_MS, TimeUnit.MILLISECONDS); + } else { + sink.line(prefix + label + " ..."); + } + } + + boolean isRunning() { + return running; + } + + /** + * Report progress. With {@code total > 0} a bar and percentage are shown; otherwise only the detail text. + * + * @param done units done so far (relative to the start of this step) + * @param total units in total, or 0 when unknown + * @param detail short text after the bar, e.g. {@code slot 92,310 / 150,004} + */ + public void progress(long done, long total, String detail) { + if (!running) + return; + this.done = Math.max(0, Math.min(done, total > 0 ? total : done)); + this.total = total; + this.detail = detail == null ? "" : detail; + if (!sink.inPlace()) { + long now = System.currentTimeMillis(); + if (now - lastPlainReportAt >= ConsoleProgress.PLAIN_PROGRESS_INTERVAL_MS) { + lastPlainReportAt = now; + String pct = total > 0 ? percent() + "% " : ""; + sink.line(prefix + label + ": " + pct + (this.detail.isEmpty() ? "" : "(" + this.detail + ")")); + } + } + } + + /** Update the text shown after the label without a percentage. */ + public void detail(String detail) { + progress(0, 0, detail); + } + + /** Write a line above the step line (warnings, errors) without breaking the live rendering. */ + public void log(String line) { + sink.line(line); + if (running && sink.inPlace()) + redraw(); + } + + /** Finish the step, e.g. {@code done("1,372 blocks")}. */ + public void done(String result) { + finish(AnsiColors.BLUE + "✅ " + AnsiColors.ANSI_RESET + dotted(label) + " " + result + " (" + elapsed() + ")", + prefix + label + ": " + result + " (" + elapsed() + ")"); + } + + /** Finish the step as failed. */ + public void fail(String reason) { + finish(AnsiColors.RED + "🔴 " + AnsiColors.ANSI_RESET + label + ": " + reason, + prefix + label + ": FAILED: " + reason); + } + + private void finish(String terminalLine, String plainLine) { + if (!running) + return; + running = false; + if (animation != null) + animation.cancel(false); + synchronized (ConsoleProgress.lock()) { + sink.line(sink.inPlace() ? prefix + terminalLine : plainLine); + } + } + + private void redraw() { + if (!running) + return; + synchronized (ConsoleProgress.lock()) { + if (!running) + return; + String spin = AnsiColors.CYAN_BOLD + SPINNER[frame++ % SPINNER.length] + AnsiColors.ANSI_RESET; + StringBuilder line = new StringBuilder(prefix).append(spin).append(' ').append(pad(label)); + if (total > 0) + line.append(' ').append(bar()).append(String.format(" %3d%%", percent())); + if (!detail.isEmpty()) + line.append(" ").append(truncate(detail)); + line.append(" (").append(elapsed()).append(')'); + sink.live(line.toString()); + } + } + + int percent() { + if (total <= 0) + return 0; + return (int) Math.min(100, done * 100 / total); + } + + private String bar() { + int filled = total > 0 ? (int) Math.min(BAR_WIDTH, done * BAR_WIDTH / total) : 0; + return "▕" + "█".repeat(filled) + "░".repeat(BAR_WIDTH - filled) + "▏"; + } + + private String elapsed() { + return format(Duration.ofMillis(System.currentTimeMillis() - startedAt)); + } + + static String format(Duration d) { + long seconds = d.getSeconds(); + if (seconds < 60) + return seconds + "s"; + if (seconds < 3600) + return d.toMinutesPart() + "m " + d.toSecondsPart() + "s"; + return d.toHours() + "h " + d.toMinutesPart() + "m"; + } + + private static String pad(String text) { + return text.length() >= LABEL_WIDTH ? text : text + " ".repeat(LABEL_WIDTH - text.length()); + } + + private static String dotted(String text) { + return text.length() >= LABEL_WIDTH ? text : text + " " + ".".repeat(LABEL_WIDTH - text.length() - 1); + } + + private static String truncate(String text) { + return text.length() <= MAX_DETAIL ? text : text.substring(0, MAX_DETAIL - 1) + "…"; + } +} diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgressTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgressTest.java new file mode 100644 index 00000000..2c809003 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgressTest.java @@ -0,0 +1,82 @@ +package com.bloxbean.cardano.yacicli.util.progress; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ConsoleProgressTest { + + @Test + void anyWriterOtherThanTheConsoleGetsPlainLines() { + List lines = new ArrayList<>(); + + ProgressTask task = ConsoleProgress.task("Catching up ...", 2, lines::add); + Step first = task.step("Haskell node → relay"); + first.done("tip slot 1,832"); + Step second = task.step("Backfill to wall clock"); + second.fail("Yano's catch-up call failed"); + + assertThat(lines).containsExactly( + "Catching up ...", + " [1/2] Haskell node → relay ...", + " [1/2] Haskell node → relay: tip slot 1,832 (0s)", + " [2/2] Backfill to wall clock ...", + " [2/2] Backfill to wall clock: FAILED: Yano's catch-up call failed"); + } + + @Test + void plainProgressIsThrottled() { + List lines = new ArrayList<>(); + Step step = ConsoleProgress.step("Downloading yano", lines::add); + + // Within the first five seconds updates are not written as lines + step.progress(10, 100, "1 MB / 10 MB"); + step.progress(50, 100, "5 MB / 10 MB"); + step.done("10.0 MB"); + + assertThat(lines).containsExactly(" Downloading yano ...", " Downloading yano: 10.0 MB (0s)"); + } + + @Test + void percentClampsToTheTotal() { + Step step = ConsoleProgress.step("Sync", line -> {}); + step.progress(50, 200, ""); + assertThat(step.percent()).isEqualTo(25); + step.progress(300, 200, ""); + assertThat(step.percent()).isEqualTo(100); + step.progress(5, 0, "no total"); + assertThat(step.percent()).isZero(); + } + + @Test + void finishingTwiceWritesOnce() { + List lines = new ArrayList<>(); + Step step = ConsoleProgress.step("Starting Yano", lines::add); + step.done("started"); + step.fail("late failure"); + step.progress(1, 1, "ignored"); + + assertThat(lines).hasSize(2); + } + + @Test + void startingANewStepFinishesTheRunningOne() { + List lines = new ArrayList<>(); + ProgressTask task = ConsoleProgress.task(null, 2, lines::add); + task.step("first"); + task.step("second"); + + assertThat(lines).contains(" [1/2] first: done (0s)"); + } + + @Test + void durationsUseTheLargestUnits() { + assertThat(Step.format(Duration.ofSeconds(9))).isEqualTo("9s"); + assertThat(Step.format(Duration.ofSeconds(125))).isEqualTo("2m 5s"); + assertThat(Step.format(Duration.ofMinutes(75))).isEqualTo("1h 15m"); + } +} diff --git a/docs/pages/yano-node-modes.mdx b/docs/pages/yano-node-modes.mdx index 42d1912d..66e34b7c 100644 --- a/docs/pages/yano-node-modes.mdx +++ b/docs/pages/yano-node-modes.mdx @@ -125,12 +125,14 @@ with Yano: 3. The Haskell node syncs those blocks and restarts as the block producer. ```text -Devnet was idle for 3d 4h (tip slot 1,830, now slot 275,562, 457 epochs behind) -Catching up to wall clock with Yano (sparse backfill, about 1,380 blocks) ... -✅ Yano synced the chain from the Haskell node (tip slot 1,830) -✅ Backfilled to slot 275,561 (1,372 blocks) -✅ Haskell node synced the backfill -✅ Devnet resumed in 38s at slot 275,570, epoch 459. Your contracts, wallets and stake are unchanged; 457 epochs passed. +The devnet stopped producing blocks (the machine slept?) +⏸ Devnet was idle for 3d 4h (tip slot 1,830, now slot 275,562, 457 epochs behind). +💡 Catching up to wall clock with Yano (sparse backfill, about 1,380 blocks) ... + [1/4] ✅ Haskell node → relay ................. tip slot 1,830 (2s) + [2/4] ✅ Yano following the chain ............. at slot 1,830 (4s) + [3/4] ✅ Backfill to wall clock ............... 1,372 blocks to slot 275,561 (12s) + [4/4] ✅ Haskell node back as block producer .. forging at slot 275,570 (1s) +✅ Devnet resumed in 19s at slot 275,570, epoch 459. Your contracts, wallets and stake are unchanged; 457 epochs passed. ``` This happens on its own: @@ -138,6 +140,10 @@ This happens on its own: - **On `start`**, before Yaci Store and the other services start. - **While DevKit keeps running** (lid closed, machine asleep): a watchdog checks the chain every 30 seconds. +In a terminal each step shows a spinner, and a progress bar with a percentage while Yano or the Haskell node +syncs. In Docker logs and other non-terminal output the same steps are written as plain lines. Set +`YACI_PROGRESS=plain` to get plain lines in a terminal too. + In yano-only mode, `start` backfills before Yano produces its first live block. You can also run it yourself, or check how far behind the chain is: From af7cf5fcb0eb207c8ce2023cb48e372ca9e08632 Mon Sep 17 00:00:00 2001 From: Satya Date: Fri, 2 Oct 2026 13:26:31 +0800 Subject: [PATCH 3/9] Keep other console output off the live progress line On a machine where the node socket took a while to appear, the relay sync step was interleaved with "Find tip error : Node Socket file is not available yet" on every poll, each glued to the end of the spinner line. - ClusterUtilService.getTip reports errors through the caller's writer instead of printing them directly. Polling callers (relay sync, catch-up, Yaci Store sync wait, admin API, MCP) already pass a silent or debug writer; the tip command still prints them. - While a progress line is shown in a terminal, System.out is wrapped: any other write clears the live line first and the spinner redraws it below, and output left without a trailing newline is not overwritten. Output from anywhere (relayed process stderr, warnings) now lands on its own line. Co-Authored-By: Claude Opus 5.5 --- .../service/ClusterUtilService.java | 6 +- .../yano/YanoCompanionService.java | 3 - .../util/progress/ConsoleProgress.java | 115 +++++++++++++++--- 3 files changed, 104 insertions(+), 20 deletions(-) diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/ClusterUtilService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/ClusterUtilService.java index cfb71d26..79217399 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/ClusterUtilService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/ClusterUtilService.java @@ -64,8 +64,10 @@ public Tuple getTip(Consumer writer, String nodeName) { return localNodeService.getTip(); } catch (Exception e) { - log.error("Error", e); - writeLn(error("Find tip error : " + e.getMessage())); + log.debug("Find tip error", e); + // Through the caller's writer: polling callers (relay sync, catch-up, store sync) pass a silent one, and + // a direct print would land in the middle of their progress line + writer.accept(error("Find tip error : " + e.getMessage())); return null; } finally { rootLogService.setLogLevel(orgLevel); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java index 38f0aa5e..347608fc 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoCompanionService.java @@ -281,9 +281,6 @@ public void performHandover(ClusterInfo clusterInfo, Path clusterFolder, Consume relaySyncMaxWaitSeconds, maxWaitSeconds)); final ClusterUtilService clusterUtilService = clusterUtilServiceProvider.getObject(); - // ClusterUtilService.getTip prints "Find tip error ..." directly via static - // ConsoleWriter on exception (bypassing the consumer); transient noise during - // the first one or two polls after the socket appears is expected. RelaySyncWaiter waiter = new RelaySyncWaiter(stallTimeoutSeconds * 1000L, maxWaitSeconds * 1000L); long targetSlot = BOOTSTRAP_EPOCH_SHIFT * epochLength; Step step = ConsoleProgress.step(String.format("Haskell relay syncing to epoch %d", BOOTSTRAP_EPOCH_SHIFT), diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java index 4edf1a24..6e886c30 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/progress/ConsoleProgress.java @@ -2,6 +2,8 @@ import com.bloxbean.cardano.yacicli.util.ConsoleWriter; +import java.io.PrintStream; + import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Consumer; @@ -113,8 +115,9 @@ public void line(String text) { } static final class TerminalSink implements Sink { - private static final String CLEAR_LINE = "\r\033[K"; - private boolean liveShown; + TerminalSink() { + LiveAwareOut.install(); + } @Override public boolean inPlace() { @@ -123,31 +126,113 @@ public boolean inPlace() { @Override public void line(String text) { + LiveAwareOut.line(text); + } + + @Override + public void live(String text) { + LiveAwareOut.live(text); + } + + @Override + public void clearLive() { + LiveAwareOut.clearLive(); + } + } + + /** + * Wraps {@code System.out} while progress is rendered in place, so output from anywhere else (process stderr + * relayed to the console, warnings, other services) never lands on the end of the live step line: the live line + * is cleared before the write, and the spinner redraws it below on its next tick. + */ + static final class LiveAwareOut extends PrintStream { + private static final String CLEAR_LINE = "\r\033[K"; + private static PrintStream terminal; + // A live step line is on screen + private static boolean liveShown; + // Other output left the cursor in the middle of a line + private static boolean foreignLineOpen; + + private LiveAwareOut(PrintStream target) { + super(target, true); + } + + static void install() { synchronized (LOCK) { - if (liveShown) - System.out.print(CLEAR_LINE); - System.out.println(text); - liveShown = false; - System.out.flush(); + if (System.out instanceof LiveAwareOut) + return; + terminal = System.out; + System.setOut(new LiveAwareOut(terminal)); } } @Override - public void live(String text) { + public void write(int b) { synchronized (LOCK) { - System.out.print(CLEAR_LINE + text); - liveShown = true; - System.out.flush(); + beforeForeignOutput(); + terminal.write(b); + foreignLineOpen = b != '\n'; } } @Override - public void clearLive() { + public void write(byte[] buf, int off, int len) { + if (len <= 0) + return; synchronized (LOCK) { - if (liveShown) - System.out.print(CLEAR_LINE); + beforeForeignOutput(); + terminal.write(buf, off, len); + foreignLineOpen = buf[off + len - 1] != '\n'; + } + } + + @Override + public void flush() { + terminal.flush(); + } + + private static void beforeForeignOutput() { + if (liveShown) { + terminal.print(CLEAR_LINE); + liveShown = false; + } + } + + static void line(String text) { + synchronized (LOCK) { + clearLiveLocked(); + terminal.println(text); + terminal.flush(); + } + } + + static void live(String text) { + synchronized (LOCK) { + if (foreignLineOpen) { + terminal.println(); + foreignLineOpen = false; + } + terminal.print(CLEAR_LINE + text); + liveShown = true; + terminal.flush(); + } + } + + static void clearLive() { + synchronized (LOCK) { + clearLiveLocked(); + terminal.flush(); + } + } + + private static void clearLiveLocked() { + if (foreignLineOpen) { + terminal.println(); + foreignLineOpen = false; + } + if (liveShown) { + terminal.print(CLEAR_LINE); liveShown = false; - System.out.flush(); } } } From c06d73dd75181888f85389ee7bc19bf124a8dd1d Mon Sep 17 00:00:00 2001 From: Satya Date: Sun, 4 Oct 2026 22:27:54 +0800 Subject: [PATCH 4/9] Address review: cancel catch-up on stop, never skip epochs on a failed yano-only backfill - stop/reset cancels a running catch-up instead of waiting up to 3 minutes and then stopping next to it. The catch-up's thread is interrupted (its waits, Yano HTTP calls and sleeps all stop), it starts no further process, and its clean-up does not restart the Haskell node, so a stop cannot find the node resurrected and a reset cannot race a restart while deleting the database. - Yano-only: a failed backfill no longer starts live production when the gap crosses an epoch boundary (its first block would skip the missed epochs). Inside one epoch it still starts, since no boundary is skipped. - The catch-up command and POST /devnet/catch-up always backfill a yano-only devnet, whatever devnet.auto.catch.up says. With auto catch-up off, start now leaves a chain that is behind with Yano running but not forging (like a stalled companion producer) and points to 'catch-up', instead of starting live production over the gap. - Companion hand-back counts as done only after the producer restarted; a failed restart goes through the recovery path, which reports if it cannot recover either. DevnetCatchUpServiceTest covers each case. Co-Authored-By: Claude Opus 5.5 --- .../localcluster/ClusterStartService.java | 17 +- .../catchup/DevnetCatchUpService.java | 209 ++++++++++++--- .../catchup/DevnetCatchUpServiceTest.java | 238 ++++++++++++++++++ 3 files changed, 417 insertions(+), 47 deletions(-) create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java index d9d55c2a..887992c8 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java @@ -282,10 +282,12 @@ private static boolean portAvailabilityCheck(ClusterInfo clusterInfo, Consumer writer) { try { + // Cancel a running catch-up first: it starts no further process and does not restart the node behind + // this stop (or a reset deleting the database) if (devnetCatchUpService.isInProgress()) { - writer.accept(info("Waiting for the running catch-up to finish before stopping ...")); - if (!devnetCatchUpService.awaitIdle(java.time.Duration.ofMinutes(3))) - writer.accept(warn("The catch-up is still running; stopping anyway")); + writer.accept(info("Cancelling the running catch-up ...")); + if (!devnetCatchUpService.cancelAndAwait(java.time.Duration.ofSeconds(60))) + writer.accept(warn("The catch-up did not stop within 60s; it will not start any further process")); } if (processes != null && processes.size() > 0) writer.accept(info("Trying to stop the running cluster ...")); @@ -513,12 +515,9 @@ public DevnetCatchUpService.Result catchUp(ClusterInfo clusterInfo, Path cluster lag.lagSlots(), lag.forecastWindowSlots()), lag.tipSlot(), lag.tipSlot(), 0, 0); if (clusterInfo.getNodeMode() == NodeMode.YANO_ONLY) { - // Yano forges as soon as it runs, so a stalled yano-only chain means Yano stopped; restart it - yanoService.stop(); - boolean started = devnetCatchUpService.startYanoOnly(clusterInfo, clusterFolder, writer); - ChainLag after = devnetCatchUpService.probeLag(clusterInfo); - return new DevnetCatchUpService.Result(started, started ? "Yano restarted at wall clock" : "Failed to start Yano", - lag.tipSlot(), after != null ? after.tipSlot() : -1, 0, 0); + // A stalled yano-only chain means Yano is not forging (stopped, or left idle by a start with auto + // catch-up off): backfill and start the live producer, whatever devnet.auto.catch.up says + return devnetCatchUpService.catchUpYanoOnly(clusterInfo, clusterFolder, writer); } return devnetCatchUpService.catchUpCompanion(clusterInfo, clusterFolder, nodeControl(clusterInfo, clusterFolder), diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java index 7319aec3..53153bfc 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java @@ -62,6 +62,9 @@ public class DevnetCatchUpService { private boolean autoCatchUp = true; private final AtomicBoolean inProgress = new AtomicBoolean(false); + // The thread running the current catch-up, and whether stop/reset asked it to give up + private volatile Thread runner; + private volatile boolean cancelled; /** * Outcome of a catch-up. @@ -93,17 +96,50 @@ public boolean isInProgress() { } /** - * Wait for a running catch-up to finish, so a stop or reset does not pull processes from under it. + * Cancel a running catch-up and wait for it to give up, so a stop or reset does not race it: the catch-up's + * thread is interrupted (its waits, Yano HTTP calls and sleeps all stop), it starts no further process, and its + * clean-up does not restart the node. Whatever it already started is in the process list the stop then stops. * * @return true when no catch-up is running any more */ - public boolean awaitIdle(Duration timeout) throws InterruptedException { + public boolean cancelAndAwait(Duration timeout) throws InterruptedException { + if (!inProgress.get()) + return true; + cancelled = true; + Thread thread = runner; + if (thread != null && thread != Thread.currentThread()) + thread.interrupt(); long deadline = System.currentTimeMillis() + timeout.toMillis(); while (inProgress.get() && System.currentTimeMillis() < deadline) - Thread.sleep(500); + Thread.sleep(100); return !inProgress.get(); } + /** Claim the catch-up slot for the current thread. */ + private boolean begin(Consumer writer) { + if (!inProgress.compareAndSet(false, true)) { + writer.accept(error("A catch-up is already running")); + return false; + } + cancelled = false; + runner = Thread.currentThread(); + return true; + } + + private void end() { + runner = null; + // A cancel interrupts this thread; do not leak the flag into whatever the thread does next + if (cancelled) + Thread.interrupted(); + inProgress.set(false); + } + + /** Called before anything that starts a process: once stop/reset cancelled the catch-up, nothing is started. */ + private void checkCancelled() throws InterruptedException { + if (cancelled || Thread.currentThread().isInterrupted()) + throw new InterruptedException("catch-up cancelled"); + } + /** Why this devnet cannot be caught up, or empty when it can. */ public Optional unsupportedReason(ClusterInfo clusterInfo) { NodeMode mode = clusterInfo.getNodeMode(); @@ -160,12 +196,12 @@ public ChainLag probeLagAfterStart(ClusterInfo clusterInfo, Duration timeout) th */ public Result catchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, NodeControl node, ChainLag lag, Consumer writer) { - if (!inProgress.compareAndSet(false, true)) + if (!begin(writer)) return Result.failed("A catch-up is already running"); try { return doCatchUpCompanion(clusterInfo, clusterFolder, node, lag, writer); } finally { - inProgress.set(false); + end(); } } @@ -186,9 +222,11 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N try { // 1. Haskell node as a relay, peering with Yano: it serves its chain to Yano and pulls Yano's backfill step = task.step("Haskell node → relay"); + checkCancelled(); if (!node.stopNode(quiet)) return failed(step, "Could not stop the Haskell node"); yanoCompanionService.updateTopologyForYanoPeering(clusterInfo, clusterFolder, quiet); + checkCancelled(); if (!node.startNode(true, quiet)) return failed(step, "Could not start the Haskell node as a relay"); Tuple relayTip = waitForHaskellTip(clusterInfo, NODE_READY_TIMEOUT); @@ -201,6 +239,7 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N // since the handover are fetched. step = task.step("Yano following the chain"); yanoService.stopQuietly(); + checkCancelled(); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.FOLLOW, quiet) || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) return failed(step, "Yano did not start to follow the Haskell node (see 'yano-logs')"); @@ -220,6 +259,7 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N // a live block at the wall-clock slot before the backfill runs. Repeat until the relay is within a // quarter of the forecast window; the rest is for the producer restart. step = task.step("Backfill to wall clock"); + checkCancelled(); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, quiet) || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) return failed(step, "Yano did not start as a block producer (see 'yano-logs')"); @@ -227,6 +267,7 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N RelaySyncWaiter relayWaiter = new RelaySyncWaiter(SYNC_STALL_TIMEOUT_MS, 0); final Step backfilling = step; for (int round = 1; ; round++) { + checkCancelled(); JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); if (caughtUp == null) return failed(step, "Yano's catch-up call failed (see 'yano-logs')"); @@ -257,10 +298,11 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N step = task.step("Haskell node back as block producer"); yanoService.stopQuietly(); yanoCompanionService.restoreOriginalTopology(clusterFolder, quiet); - boolean restarted = node.stopNode(quiet) && node.startNode(false, quiet); - handedBack = true; - if (!restarted) + checkCancelled(); + // Only a successful restart counts as handed back; otherwise the recovery below retries it + if (!node.stopNode(quiet) || !node.startNode(false, quiet)) return failed(step, "Could not restart the Haskell node as the block producer"); + handedBack = true; step.detail("waiting for the first block"); Tuple forged = waitForHaskellSlotAbove(clusterInfo, yanoTipSlot, forgeTimeout(clusterInfo)); @@ -278,23 +320,30 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N task.log(success(message)); return new Result(true, message, lag.tipSlot(), forged._2.getSlot(), blocksProduced, duration); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return failed(step, "Catch-up interrupted"); + return failed(step, cancelled ? "Catch-up cancelled" : "Catch-up interrupted"); } catch (IOException | RuntimeException e) { log.error("Catch-up failed", e); return failed(step, "Catch-up failed: " + e.getMessage()); } finally { + // A cancel interrupts this thread: clear it so the clean-up's own waits (stopping Yano) still work + Thread.interrupted(); if (!handedBack) { - // Never leave the devnet half-way: Yano stopped, original topology, Haskell node as producer + // Never leave the devnet half-way: Yano stopped, original topology. Then the Haskell node back as + // the producer, unless a stop/reset cancelled the catch-up: that stops whatever is running and must + // not find the node restarted behind its back. yanoService.stopQuietly(); yanoCompanionService.restoreOriginalTopology(clusterFolder, quiet); - if (node.stopNode(quiet)) - node.startNode(false, quiet); + if (!cancelled && !(node.stopNode(quiet) && node.startNode(false, quiet))) + writer.accept(error("Could not restart the Haskell node as the block producer. " + + "Run 'stop' and 'start' to recover.")); } } } - private static Result failed(Step step, String message) { + private Result failed(Step step, String message) { + // A step fails as a side effect of a cancel (an interrupted wait, a Yano start cut short): say so + if (cancelled) + message = "Catch-up cancelled"; if (step != null) step.fail(message); return Result.failed(message); @@ -309,41 +358,113 @@ private static String slotDetail(long slot, long target, long blocksPerSecond) { // --------------------------------------------------------------------------------------------------------- /** - * Start Yano for an existing yano-only devnet. With auto catch-up it first runs Yano without forging, backfills - * whatever time has passed, and only then starts the live producer, so every skipped epoch gets its boundary. + * Start Yano for an existing yano-only devnet. + *

+ * With auto catch-up, Yano first runs without forging, the time that passed is backfilled, and only then the + * live producer starts, so every skipped epoch gets its boundary. If the backfill fails while the gap crosses an + * epoch boundary, live production is not started: its first block would jump over the missed epochs. + *

+ * With auto catch-up off, a chain that is too far behind is left with Yano running but not forging (like a + * stalled companion producer), and the user is told to run {@code catch-up}. + * + * @return true when Yano is running */ public boolean startYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { - if (!autoCatchUp) - return yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer); + if (!begin(writer)) + return false; + try { + if (!autoCatchUp) + return startYanoOnlyWithoutCatchUp(clusterInfo, clusterFolder, writer); - if (!inProgress.compareAndSet(false, true)) { - writer.accept(error("A catch-up is already running")); + YanoBackfill backfill = backfillYanoOnly(clusterInfo, clusterFolder, writer); + if (!backfill.liveStartAllowed()) { + writer.accept(error(backfill.message())); + return false; + } + checkCancelled(); + return yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); return false; + } finally { + end(); } + } + + /** + * Catch a yano-only devnet up on request ({@code catch-up}, the admin API), whatever + * {@code devnet.auto.catch.up} says: Yano is stopped, the gap backfilled, and the live producer started. + */ + public Result catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { + if (!begin(writer)) + return Result.failed("A catch-up is already running"); + long startedAt = System.currentTimeMillis(); try { - catchUpYanoOnly(clusterInfo, clusterFolder, writer); + yanoService.stopQuietly(); + YanoBackfill backfill = backfillYanoOnly(clusterInfo, clusterFolder, writer); + if (!backfill.liveStartAllowed()) + return Result.failed(backfill.message()); + checkCancelled(); + if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer)) + return Result.failed("Yano did not start as the live producer (see 'yano-logs')"); + long duration = System.currentTimeMillis() - startedAt; + return new Result(true, backfill.message(), backfill.fromSlot(), backfill.toSlot(), backfill.blocks(), duration); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return Result.failed("Catch-up cancelled"); } finally { - inProgress.set(false); + end(); + } + } + + private boolean startYanoOnlyWithoutCatchUp(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) + throws InterruptedException { + int httpPort = clusterInfo.getYanoHttpPort(); + // Look at the tip before anything can forge, as the companion start does with its stalled producer + if (yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, problemsOnly(writer)) + && yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) { + Tuple tip = yanoBootstrapService.getNodeTip(httpPort); + ChainLag lag = tip != null ? ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()) : null; + if (lag != null && lag.needsCatchUpOnStart()) { + writer.accept(warn("The chain is %s behind wall clock%s. Yano is running but not producing blocks; " + + "run 'catch-up' to bring it to wall clock.", lag.idleTimeText(), + lag.epochsToCross() > 0 ? String.format(" (%s)", epochs(lag.epochsToCross())) : "")); + return true; + } } + yanoService.stopQuietly(); + checkCancelled(); return yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer); } - /** Backfill a stopped yano-only chain to wall clock. Yano is stopped again on return. */ - private void catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { + /** + * Outcome of a yano-only backfill. + * + * @param liveStartAllowed the live producer can start: the chain is at wall clock, or the gap stays inside one + * epoch so its first block skips no boundary + */ + record YanoBackfill(boolean liveStartAllowed, String message, long fromSlot, long toSlot, long blocks) { + } + + /** Backfill a stopped yano-only chain to wall clock with Yano in catch-up mode. Yano is stopped again on return. */ + private YanoBackfill backfillYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) + throws InterruptedException { int httpPort = clusterInfo.getYanoHttpPort(); long startedAt = System.currentTimeMillis(); try { + checkCancelled(); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, problemsOnly(writer)) - || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) { - writer.accept(warn("Could not check the chain tip before starting Yano; starting without catch-up")); - return; - } + || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) + return new YanoBackfill(false, "Yano did not start to check the chain tip (see 'yano-logs'); " + + "not starting block production, so no missed epoch is skipped. Retry with 'start' or 'catch-up'.", + -1, -1, 0); Tuple tip = yanoBootstrapService.getNodeTip(httpPort); if (tip == null) - return; + return new YanoBackfill(false, "Could not read the chain tip from Yano; not starting block production. " + + "Retry with 'start' or 'catch-up'.", -1, -1, 0); ChainLag lag = ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()); if (lag.lagSlots() == 0) - return; + return new YanoBackfill(true, "The chain is at wall clock", lag.tipSlot(), lag.tipSlot(), 0); // Short stops need no banner: the backfill is a block or two boolean report = lag.needsCatchUpOnStart(); @@ -351,22 +472,34 @@ private void catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consum if (report) printBanner(lag, task.logWriter()); Step step = report ? task.step("Backfill to wall clock") : null; + checkCancelled(); JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); + checkCancelled(); if (caughtUp == null) { - String message = "Yano's catch-up call failed (see 'yano-logs'); the first block will skip the gap"; if (step != null) - step.fail(message); - else - writer.accept(error(message)); - return; + step.fail("Yano's catch-up call failed"); + // Inside one epoch the first live block skips no boundary, so starting is harmless + if (lag.epochsToCross() == 0) + return new YanoBackfill(true, "Catch-up failed; the gap is inside one epoch", lag.tipSlot(), + lag.tipSlot(), 0); + return new YanoBackfill(false, String.format("Yano's catch-up call failed (see 'yano-logs'). Not starting " + + "block production: its first block would skip %s. Retry with 'catch-up'.", + epochs(lag.epochsToCross())), lag.tipSlot(), lag.tipSlot(), 0); } + long toSlot = caughtUp.path("new_slot").asLong(); + long blocks = caughtUp.path("blocks_produced").asLong(); + String message = resumedMessage(clusterInfo, lag, toSlot, System.currentTimeMillis() - startedAt); if (report) { - long toSlot = caughtUp.path("new_slot").asLong(); - step.done(String.format("%,d blocks to slot %,d", caughtUp.path("blocks_produced").asLong(), toSlot)); - task.log(success(resumedMessage(clusterInfo, lag, toSlot, System.currentTimeMillis() - startedAt))); + step.done(String.format("%,d blocks to slot %,d", blocks, toSlot)); + task.log(success(message)); } + return new YanoBackfill(true, message, lag.tipSlot(), toSlot, blocks); } finally { + // A cancel interrupts this thread: keep the flag for the caller, but let stopping Yano wait properly + boolean interrupted = Thread.interrupted(); yanoService.stopQuietly(); + if (interrupted) + Thread.currentThread().interrupt(); } } diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java new file mode 100644 index 00000000..114f89a0 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java @@ -0,0 +1,238 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import com.bloxbean.cardano.yaci.core.protocol.chainsync.messages.Point; +import com.bloxbean.cardano.yacicli.common.Tuple; +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import com.bloxbean.cardano.yacicli.localcluster.NodeMode; +import com.bloxbean.cardano.yacicli.localcluster.service.ClusterUtilService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoBootstrapService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoCompanionService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoRunMode; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoService; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +class DevnetCatchUpServiceTest { + private static final Path FOLDER = Path.of("/tmp/devnet"); + private static final int HTTP_PORT = 6060; + + private YanoService yano; + private YanoBootstrapService bootstrap; + private YanoCompanionService companion; + private ClusterUtilService clusterUtil; + private DevnetCatchUpService service; + private final List output = new ArrayList<>(); + private final Consumer writer = output::add; + + // Wall clock is about slot 1,010 of a chain with 40-slot epochs (window 21 slots) + private final long startTime = System.currentTimeMillis() / 1000 - 1010; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + yano = mock(YanoService.class); + bootstrap = mock(YanoBootstrapService.class); + companion = mock(YanoCompanionService.class); + clusterUtil = mock(ClusterUtilService.class); + ObjectProvider provider = mock(ObjectProvider.class); + when(provider.getObject()).thenReturn(clusterUtil); + service = new DevnetCatchUpService(yano, bootstrap, companion, provider); + + when(yano.start(any(), any(), any(YanoRunMode.class), any())).thenReturn(true); + when(bootstrap.waitForNodeTip(eq(HTTP_PORT), any())).thenReturn(true); + } + + private ClusterInfo clusterInfo(NodeMode mode) { + return ClusterInfo.builder() + .nodeMode(mode).startTime(startTime).slotLength(1.0).blockTime(1.0).epochLength(40) + .securityParam(7).activeSlotsCoeff(1.0).yanoHttpPort(HTTP_PORT) + .build(); + } + + private long wallSlot() { + return ChainLag.wallClockSlot(startTime, 1.0, System.currentTimeMillis()); + } + + private static Tuple tip(long slot) { + return new Tuple<>(slot, new Point(slot, "hash-" + slot)); + } + + private static com.fasterxml.jackson.databind.JsonNode caughtUp(long newSlot, long blocks) { + return new ObjectMapper().createObjectNode().put("new_slot", newSlot).put("blocks_produced", blocks); + } + + private void verifyNoLiveStart() { + verify(yano, never()).start(any(), any(), eq(YanoRunMode.LIVE), any()); + } + + // --- Yano-only ------------------------------------------------------------------------------------------ + + @Test + void yanoOnlyFailedBackfillAcrossEpochsDoesNotStartLiveProduction() { + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); // epoch 2, wall clock is epoch 25 + when(bootstrap.catchUp(HTTP_PORT)).thenReturn(null); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isFalse(); + verifyNoLiveStart(); + assertThat(String.join("\n", output)).contains("would skip 23 epochs"); + } + + @Test + void yanoOnlyFailedBackfillInsideOneEpochStillStartsLive() { + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(1001)); // same epoch as wall clock (1,000-1,039) + when(bootstrap.catchUp(HTTP_PORT)).thenReturn(null); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isTrue(); + verify(yano).start(any(), any(), eq(YanoRunMode.LIVE), any()); + } + + @Test + void yanoOnlyStartBackfillsThenStartsLive() { + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); + when(bootstrap.catchUp(HTTP_PORT)).thenReturn(caughtUp(1010, 30)); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isTrue(); + var order = inOrder(yano, bootstrap); + order.verify(yano).start(any(), any(), eq(YanoRunMode.CATCH_UP), any()); + order.verify(bootstrap).catchUp(HTTP_PORT); + order.verify(yano).stopQuietly(); + order.verify(yano).start(any(), any(), eq(YanoRunMode.LIVE), any()); + } + + @Test + void explicitYanoOnlyCatchUpBackfillsEvenWithAutoCatchUpOff() { + ReflectionTestUtils.setField(service, "autoCatchUp", false); + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); + when(bootstrap.catchUp(HTTP_PORT)).thenReturn(caughtUp(1010, 30)); + + DevnetCatchUpService.Result result = service.catchUpYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(result.success()).isTrue(); + assertThat(result.blocksProduced()).isEqualTo(30); + verify(bootstrap).catchUp(HTTP_PORT); + verify(yano).start(any(), any(), eq(YanoRunMode.LIVE), any()); + } + + @Test + void explicitYanoOnlyCatchUpReportsAFailedBackfill() { + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); + when(bootstrap.catchUp(HTTP_PORT)).thenReturn(null); + + DevnetCatchUpService.Result result = service.catchUpYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(result.success()).isFalse(); + verifyNoLiveStart(); + } + + @Test + void yanoOnlyStartWithAutoCatchUpOffLeavesYanoIdleWhenBehind() { + ReflectionTestUtils.setField(service, "autoCatchUp", false); + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isTrue(); + verify(yano).start(any(), any(), eq(YanoRunMode.CATCH_UP), any()); + verify(bootstrap, never()).catchUp(anyInt()); + verifyNoLiveStart(); + assertThat(String.join("\n", output)).contains("run 'catch-up'"); + } + + @Test + void yanoOnlyStartWithAutoCatchUpOffGoesLiveAtTheTip() { + ReflectionTestUtils.setField(service, "autoCatchUp", false); + when(bootstrap.getNodeTip(HTTP_PORT)).thenAnswer(invocation -> tip(wallSlot())); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isTrue(); + verify(yano).start(any(), any(), eq(YanoRunMode.LIVE), any()); + } + + // --- Companion ------------------------------------------------------------------------------------------ + + @Test + void companionFailedProducerRestartGoesThroughRecovery() { + NodeControl node = mock(NodeControl.class); + when(node.stopNode(any())).thenReturn(true); + when(node.startNode(eq(true), any())).thenReturn(true); + when(node.startNode(eq(false), any())).thenReturn(false); + // Both Haskell node and Yano are at wall clock, so every sync is immediate + when(clusterUtil.getTip(any())).thenAnswer(invocation -> tip(wallSlot())); + when(bootstrap.getNodeTip(HTTP_PORT)).thenAnswer(invocation -> tip(wallSlot())); + when(bootstrap.catchUp(HTTP_PORT)).thenAnswer(invocation -> caughtUp(wallSlot(), 3)); + ClusterInfo info = clusterInfo(NodeMode.COMPANION); + + DevnetCatchUpService.Result result = service.catchUpCompanion(info, FOLDER, node, + ChainLag.of(info, wallSlot() - 100, System.currentTimeMillis()), writer); + + assertThat(result.success()).isFalse(); + // Once in the hand-back step, once more by the recovery that a failed hand-back must still run + verify(node, times(2)).startNode(eq(false), any()); + assertThat(String.join("\n", output)).contains("Run 'stop' and 'start' to recover"); + } + + @Test + void cancelledCompanionCatchUpStartsNothingFurtherAndDoesNotRestartTheNode() throws Exception { + NodeControl node = mock(NodeControl.class); + when(node.stopNode(any())).thenReturn(true); + when(node.startNode(eq(true), any())).thenReturn(true); + when(clusterUtil.getTip(any())).thenAnswer(invocation -> tip(wallSlot() - 100)); + CountDownLatch yanoFollowing = new CountDownLatch(1); + // Yano's follow start blocks until the catch-up is interrupted + when(yano.start(any(), any(), eq(YanoRunMode.FOLLOW), any())).thenAnswer(invocation -> { + yanoFollowing.countDown(); + try { + Thread.sleep(30_000); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + }); + ClusterInfo info = clusterInfo(NodeMode.COMPANION); + + CompletableFuture running = CompletableFuture.supplyAsync(() -> + service.catchUpCompanion(info, FOLDER, node, ChainLag.of(info, wallSlot() - 100, + System.currentTimeMillis()), writer)); + assertThat(yanoFollowing.await(10, TimeUnit.SECONDS)).isTrue(); + + assertThat(service.cancelAndAwait(Duration.ofSeconds(10))).isTrue(); + DevnetCatchUpService.Result result = running.get(10, TimeUnit.SECONDS); + + assertThat(result.success()).isFalse(); + assertThat(result.message()).isEqualTo("Catch-up cancelled"); + assertThat(service.isInProgress()).isFalse(); + verify(yano, never()).start(any(), any(), eq(YanoRunMode.CATCH_UP), any()); + verify(node, never()).startNode(eq(false), any()); + } + + @Test + void cancelWithNothingRunningReturnsImmediately() throws Exception { + assertThat(service.cancelAndAwait(Duration.ofSeconds(1))).isTrue(); + } +} From feba8933ab69dabfca302e17476f22970723afd1 Mon Sep 17 00:00:00 2001 From: Satya Date: Sun, 4 Oct 2026 22:28:07 +0800 Subject: [PATCH 5/9] Document what devnet.auto.catch.up=false does in each node mode Co-Authored-By: Claude Opus 5.5 --- docs/pages/yano-node-modes.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/yano-node-modes.mdx b/docs/pages/yano-node-modes.mdx index 66e34b7c..9b84d443 100644 --- a/docs/pages/yano-node-modes.mdx +++ b/docs/pages/yano-node-modes.mdx @@ -163,6 +163,9 @@ To only report the stall and catch up by hand, set this in `config/application.p devnet.auto.catch.up=false ``` +The chain then waits for you: a companion block producer cannot forge, and in yano-only mode Yano starts without +producing blocks, so no missed epoch is skipped. `catch-up` brings it to wall clock and starts block production. + Time moves forward to wall clock, so every epoch that passed is processed. Governance actions and DRep activity can expire while the devnet sleeps, just as they would on a real network. From 2d4dbe7506d38558da105a85390c1720be0a3e1b Mon Sep 17 00:00:00 2001 From: Satya Date: Sun, 4 Oct 2026 22:44:59 +0800 Subject: [PATCH 6/9] Address review: no untracked process on a cancelled start, a truly idle Yano, fresh lag on fallback - A process is tracked as soon as it is spawned. If its start-up is interrupted (stop/reset cancelling a catch-up), times out or exits, it is terminated before the error propagates: Yano (YanoService registers it right after spawning and terminates it on any failed start, including the 30 s timeout that used to leave it running), the 1 s start check in ProcessUtil, and the node's socket wait. ProcessUtil.terminate stops a process tree gracefully then forcibly and also works on an interrupted thread. - New YanoRunMode.IDLE: Yano serves the chain and its HTTP API with no block producer and no client, so it can never forge. A yano-only start with auto catch-up off now waits in this mode instead of CATCH_UP, whose long block time only postponed forging (by an hour). CATCH_UP, which only lives for the duration of a catch-up, now uses Yano's maximum block time. - A failed yano-only backfill re-measures the lag before allowing live production inside one epoch: the failed call may have taken minutes. Tests: a real child process whose start is interrupted is gone afterwards; terminate on an interrupted thread; IDLE properties; the lag re-check with a clock that moves five epochs during the failing call. Also verified with the Yano binary: auto catch-up off, idle Yano does not forge for 20 s+, then 'catch-up' backfills 3 epochs and goes live. Co-Authored-By: Claude Opus 5.5 --- .../localcluster/ClusterStartService.java | 10 ++- .../catchup/DevnetCatchUpService.java | 39 +++++----- .../localcluster/yano/YanoConfigBuilder.java | 6 ++ .../localcluster/yano/YanoRunMode.java | 19 +++-- .../localcluster/yano/YanoService.java | 41 +++++++--- .../cardano/yacicli/util/ProcessUtil.java | 43 ++++++++++- .../catchup/DevnetCatchUpServiceTest.java | 24 +++++- .../yano/YanoRunModeConfigTest.java | 13 +++- .../yano/YanoServiceStartTest.java | 74 +++++++++++++++++++ 9 files changed, 232 insertions(+), 37 deletions(-) create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceStartTest.java diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java index 887992c8..f354f03b 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java @@ -387,7 +387,15 @@ private Process startNode(Path clusterFolder, ClusterInfo clusterInfo, boolean a Path nodeSocketPath = clusterFolder.resolve(ClusterConfig.NODE_FOLDER_PREFIX).resolve("node.sock"); int counter = 0; while (!Files.exists(nodeSocketPath) && counter < 10) { //wait 5 sec max - Thread.sleep(500); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + // A cancelled catch-up: the caller never gets this process to track, so do not leave it running + if (haskellNodeProcess == process) + haskellNodeProcess = null; + ProcessUtil.terminate(process); + throw e; + } if (counter == 5) writer.accept(info("Waiting for node socket file to be created ...")); counter++; diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java index 53153bfc..c86af964 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.LongSupplier; import static com.bloxbean.cardano.yacicli.util.ConsoleWriter.*; @@ -62,6 +63,8 @@ public class DevnetCatchUpService { private boolean autoCatchUp = true; private final AtomicBoolean inProgress = new AtomicBoolean(false); + // Wall clock, replaceable in tests + LongSupplier clock = System::currentTimeMillis; // The thread running the current catch-up, and whether stop/reset asked it to give up private volatile Thread runner; private volatile boolean cancelled; @@ -168,7 +171,7 @@ public ChainLag probeLag(ClusterInfo clusterInfo) { : haskellTip(clusterInfo); if (tip == null || tip._2 == null) return null; - return ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()); + return ChainLag.of(clusterInfo, tip._2.getSlot(), clock.getAsLong()); } /** @@ -207,7 +210,7 @@ public Result catchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, Node private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, NodeControl node, ChainLag lag, Consumer writer) { - long startedAt = System.currentTimeMillis(); + long startedAt = clock.getAsLong(); int httpPort = clusterInfo.getYanoHttpPort(); long epochLength = clusterInfo.getEpochLength(); ProgressTask task = ConsoleProgress.task(null, 4, writer); @@ -284,7 +287,7 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N return failed(step, "The Haskell relay did not sync Yano's backfill: " + adopted.reason()); long lagNow = ChainLag.wallClockSlot(clusterInfo.getStartTime(), clusterInfo.getSlotLength(), - System.currentTimeMillis()) - yanoTipSlot; + clock.getAsLong()) - yanoTipSlot; if (lagNow <= slack) break; if (round >= MAX_PUMP_ROUNDS) @@ -306,7 +309,7 @@ private Result doCatchUpCompanion(ClusterInfo clusterInfo, Path clusterFolder, N step.detail("waiting for the first block"); Tuple forged = waitForHaskellSlotAbove(clusterInfo, yanoTipSlot, forgeTimeout(clusterInfo)); - long duration = System.currentTimeMillis() - startedAt; + long duration = clock.getAsLong() - startedAt; if (forged == null) { String message = "The chain is caught up, but the Haskell node has not forged yet. " + "If it does not, run 'catch-up' again."; @@ -398,7 +401,7 @@ public boolean startYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consum public Result catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { if (!begin(writer)) return Result.failed("A catch-up is already running"); - long startedAt = System.currentTimeMillis(); + long startedAt = clock.getAsLong(); try { yanoService.stopQuietly(); YanoBackfill backfill = backfillYanoOnly(clusterInfo, clusterFolder, writer); @@ -407,7 +410,7 @@ public Result catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consu checkCancelled(); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.LIVE, writer)) return Result.failed("Yano did not start as the live producer (see 'yano-logs')"); - long duration = System.currentTimeMillis() - startedAt; + long duration = clock.getAsLong() - startedAt; return new Result(true, backfill.message(), backfill.fromSlot(), backfill.toSlot(), backfill.blocks(), duration); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -420,11 +423,11 @@ public Result catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consu private boolean startYanoOnlyWithoutCatchUp(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) throws InterruptedException { int httpPort = clusterInfo.getYanoHttpPort(); - // Look at the tip before anything can forge, as the companion start does with its stalled producer - if (yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, problemsOnly(writer)) + // Look at the tip with Yano serving but never forging, as the companion start does with its stalled producer + if (yanoService.start(clusterInfo, clusterFolder, YanoRunMode.IDLE, problemsOnly(writer)) && yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) { Tuple tip = yanoBootstrapService.getNodeTip(httpPort); - ChainLag lag = tip != null ? ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()) : null; + ChainLag lag = tip != null ? ChainLag.of(clusterInfo, tip._2.getSlot(), clock.getAsLong()) : null; if (lag != null && lag.needsCatchUpOnStart()) { writer.accept(warn("The chain is %s behind wall clock%s. Yano is running but not producing blocks; " + "run 'catch-up' to bring it to wall clock.", lag.idleTimeText(), @@ -450,7 +453,7 @@ record YanoBackfill(boolean liveStartAllowed, String message, long fromSlot, lon private YanoBackfill backfillYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) throws InterruptedException { int httpPort = clusterInfo.getYanoHttpPort(); - long startedAt = System.currentTimeMillis(); + long startedAt = clock.getAsLong(); try { checkCancelled(); if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.CATCH_UP, problemsOnly(writer)) @@ -462,7 +465,7 @@ private YanoBackfill backfillYanoOnly(ClusterInfo clusterInfo, Path clusterFolde if (tip == null) return new YanoBackfill(false, "Could not read the chain tip from Yano; not starting block production. " + "Retry with 'start' or 'catch-up'.", -1, -1, 0); - ChainLag lag = ChainLag.of(clusterInfo, tip._2.getSlot(), System.currentTimeMillis()); + ChainLag lag = ChainLag.of(clusterInfo, tip._2.getSlot(), clock.getAsLong()); if (lag.lagSlots() == 0) return new YanoBackfill(true, "The chain is at wall clock", lag.tipSlot(), lag.tipSlot(), 0); @@ -478,17 +481,19 @@ private YanoBackfill backfillYanoOnly(ClusterInfo clusterInfo, Path clusterFolde if (caughtUp == null) { if (step != null) step.fail("Yano's catch-up call failed"); - // Inside one epoch the first live block skips no boundary, so starting is harmless - if (lag.epochsToCross() == 0) - return new YanoBackfill(true, "Catch-up failed; the gap is inside one epoch", lag.tipSlot(), - lag.tipSlot(), 0); + // Inside one epoch the first live block skips no boundary, so starting is harmless. Measure again: + // the failed call may have taken minutes, and epochs may have passed meanwhile. + ChainLag now = ChainLag.of(clusterInfo, lag.tipSlot(), clock.getAsLong()); + if (now.epochsToCross() == 0) + return new YanoBackfill(true, "Catch-up failed; the gap is inside one epoch", now.tipSlot(), + now.tipSlot(), 0); return new YanoBackfill(false, String.format("Yano's catch-up call failed (see 'yano-logs'). Not starting " + "block production: its first block would skip %s. Retry with 'catch-up'.", - epochs(lag.epochsToCross())), lag.tipSlot(), lag.tipSlot(), 0); + epochs(now.epochsToCross())), now.tipSlot(), now.tipSlot(), 0); } long toSlot = caughtUp.path("new_slot").asLong(); long blocks = caughtUp.path("blocks_produced").asLong(); - String message = resumedMessage(clusterInfo, lag, toSlot, System.currentTimeMillis() - startedAt); + String message = resumedMessage(clusterInfo, lag, toSlot, clock.getAsLong() - startedAt); if (report) { step.done(String.format("%,d blocks to slot %,d", blocks, toSlot)); task.log(success(message)); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java index 8edec4a6..1bf1ec1e 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoConfigBuilder.java @@ -104,6 +104,12 @@ public Map properties(ClusterInfo clusterInfo, Path yanoConfigDi props.put("yano.remote.host", "127.0.0.1"); props.put("yano.remote.port", String.valueOf(clusterInfo.getNodePort())); } + case IDLE -> { + //Serve the chain without forging: no producer, no client. Yano's dev mode requires a producer. + props.put("yano.dev-mode", "false"); + props.put("yano.block-producer.enabled", "false"); + props.put("yano.client.enabled", "false"); + } case LIVE -> { } } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java index 53028a35..629b1606 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunMode.java @@ -11,15 +11,22 @@ public enum YanoRunMode { /** Follows the Haskell node's chain (client on, producer off) so Yano's chainstate reaches the Haskell tip. */ FOLLOW, /** - * Block producer on an existing chain with a very long block time, so it forges nothing on its own. Blocks are - * only produced through {@code /devnet/epochs/catch-up}, and none can land at the wall-clock slot before it. + * Block producer on an existing chain with the longest block time Yano accepts (about 24.8 days), so it forges + * nothing on its own during a catch-up. Blocks are only produced through {@code /devnet/epochs/catch-up}, and none + * can land at the wall-clock slot before it. Only for the duration of a catch-up: use {@link #IDLE} to keep a + * chain paused. */ - CATCH_UP; + CATCH_UP, + /** + * Serves the chain (n2n server and HTTP API) with no block producer and no client, so it never forges: a + * yano-only chain that is behind wall clock, waiting for {@code catch-up}. + */ + IDLE; - /** Block time used in {@link #CATCH_UP}: long enough that no live block is ever scheduled during a catch-up. */ - public static final long CATCH_UP_BLOCK_TIME_MILLIS = 3_600_000L; + /** Block time used in {@link #CATCH_UP}: Yano's maximum, so no live block is scheduled during a catch-up. */ + public static final long CATCH_UP_BLOCK_TIME_MILLIS = Integer.MAX_VALUE; public boolean producesBlocks() { - return this != FOLLOW; + return this != FOLLOW && this != IDLE; } } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java index ddca3fb8..2e9a5037 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java @@ -105,11 +105,12 @@ public boolean start(ClusterInfo clusterInfo, Path clusterFolder, YanoRunMode ru Path yanoConfigDir = prepareYanoConfig(clusterInfo, clusterFolder, writer); if (yanoConfigDir == null) return false; - Process process = startYanoProcess(clusterInfo, clusterFolder, yanoConfigDir, runMode, writer); - if (process != null) { - processes.add(process); - return true; - } + // startYanoProcess registers the process as soon as it is spawned and removes it again if it fails + return startYanoProcess(clusterInfo, clusterFolder, yanoConfigDir, runMode, writer) != null; + } catch (InterruptedException e) { + // A cancelled catch-up: the spawned process was already terminated + Thread.currentThread().interrupt(); + writer.accept(warn("Yano start interrupted")); } catch (Exception e) { log.error("Error starting Yano", e); writer.accept(error("Failed to start Yano: " + e.getMessage())); @@ -392,6 +393,23 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa builder.command(yanoBin.toAbsolutePath().toString()); Process process = builder.start(); + // Track it right away: a stop, or a cancel interrupting the wait below, must be able to find it + processes.add(process); + boolean started = false; + try { + started = awaitYanoStart(process, clusterInfo, runMode, writer); + return started ? process : null; + } finally { + if (!started) { + // Not started (exited, timed out, interrupted): never leave it running untracked + processes.remove(process); + ProcessUtil.terminate(process); + } + } + } + + private boolean awaitYanoStart(Process process, ClusterInfo clusterInfo, YanoRunMode runMode, + Consumer writer) throws InterruptedException { Step step = ConsoleProgress.step(String.format("Starting Yano (%s, n2n port %d, HTTP port %d)", runMode.name().toLowerCase().replace('_', '-'), clusterInfo.getYanoServerPort(), clusterInfo.getYanoHttpPort()), writer); @@ -421,20 +439,25 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa if (!process.isAlive()) { step.fail("exited"); writer.accept(error("Yano process exited unexpectedly. Check logs with 'yano-logs'")); - return null; + return false; + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + step.fail("interrupted"); + throw e; } - Thread.sleep(1000); } if (!started.get()) { step.fail("not started after 30s"); writer.accept(error("Yano did not start within timeout. Check logs with 'yano-logs'")); - return null; + return false; } step.done("started"); processUtil.createProcessId(YANO_PROCESS_NAME, process); - return process; + return true; } @EventListener diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java index a144c3d5..639299cd 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/util/ProcessUtil.java @@ -80,7 +80,7 @@ public Process startLongRunningProcess(String processName, ProcessBuilder builde executorHelper.getExecutor().submit(processStream); executorHelper.getExecutor().submit(errorStream); - process.waitFor(1, TimeUnit.SECONDS); + awaitStartOrTerminate(process); if (!process.isAlive()) { writer.accept(error("%s process could not be started.", processName)); return null; @@ -96,7 +96,7 @@ public Process startLongRunningProcess(String processName, ProcessBuilder builde public Process startLongRunningProcess(String processName, ProcessBuilder builder, Consumer writer) throws IOException, InterruptedException { Process process = builder.start(); - process.waitFor(1, TimeUnit.SECONDS); + awaitStartOrTerminate(process); if (!process.isAlive()) { writer.accept(error("%s process could not be started.", processName)); return null; @@ -107,6 +107,45 @@ public Process startLongRunningProcess(String processName, ProcessBuilder builde return process; } + /** + * Give a freshly spawned process a second to fail fast. If the wait is interrupted (a cancelled catch-up), the + * process is not returned to anyone who could stop it later, so terminate it here. + */ + private static void awaitStartOrTerminate(Process process) throws InterruptedException { + try { + process.waitFor(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + terminate(process); + throw e; + } + } + + /** + * Stop a process tree: SIGTERM, then a forced kill after a timeout. Works on an interrupted thread too; the + * interrupt flag is restored afterwards. + */ + public static void terminate(Process process) { + boolean interrupted = Thread.interrupted(); + try { + var children = process.descendants().toList(); + children.forEach(ProcessHandle::destroy); + process.destroy(); + if (!process.waitFor(10, TimeUnit.SECONDS)) { + children.forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + process.waitFor(5, TimeUnit.SECONDS); + } + children.stream().filter(ProcessHandle::isAlive).forEach(ProcessHandle::destroyForcibly); + } catch (InterruptedException e) { + process.descendants().forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + interrupted = true; + } finally { + if (interrupted) + Thread.currentThread().interrupt(); + } + } + public String createProcessId(String processName, Process process) { try { var yaciCliHome = clusterConfig.getYaciCliHome(); diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java index 114f89a0..007e4f42 100644 --- a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java @@ -156,12 +156,34 @@ void yanoOnlyStartWithAutoCatchUpOffLeavesYanoIdleWhenBehind() { boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); assertThat(started).isTrue(); - verify(yano).start(any(), any(), eq(YanoRunMode.CATCH_UP), any()); + // Waiting for catch-up in a mode that cannot forge at all, not merely one with a long block time + verify(yano).start(any(), any(), eq(YanoRunMode.IDLE), any()); + verify(yano, never()).start(any(), any(), eq(YanoRunMode.CATCH_UP), any()); + verify(yano, never()).stopQuietly(); verify(bootstrap, never()).catchUp(anyInt()); verifyNoLiveStart(); assertThat(String.join("\n", output)).contains("run 'catch-up'"); } + @Test + void yanoOnlyFailedBackfillRechecksTheLagBeforeAllowingLiveProduction() { + // At the start the gap is inside one epoch (tip 1,000, wall clock 1,001) ... + long[] now = {(startTime + 1001) * 1000}; + service.clock = () -> now[0]; + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(1000)); + // ... but the failing call takes long enough for five epochs to pass + when(bootstrap.catchUp(HTTP_PORT)).thenAnswer(invocation -> { + now[0] += 200_000; + return null; + }); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isFalse(); + verifyNoLiveStart(); + assertThat(String.join("\n", output)).contains("would skip 5 epochs"); + } + @Test void yanoOnlyStartWithAutoCatchUpOffGoesLiveAtTheTip() { ReflectionTestUtils.setField(service, "autoCatchUp", false); diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java index 7b1a1922..d45801c3 100644 --- a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoRunModeConfigTest.java @@ -41,7 +41,7 @@ void pastTimeTravelModeCarriesTheBackfillInterval() { void catchUpModeForgesNothingOnItsOwnAndBackfillsSparsely() { Map props = props(YanoRunMode.CATCH_UP); - assertThat(props).containsEntry("yano.block-producer.block-time-millis", "3600000") + assertThat(props).containsEntry("yano.block-producer.block-time-millis", String.valueOf(Integer.MAX_VALUE)) .containsEntry("yano.block-producer.backfill-block-interval-slots", "0") .doesNotContainKeys("yano.block-producer.past-time-travel-mode", "yano.client.enabled"); } @@ -58,6 +58,17 @@ void followModeSyncsFromTheHaskellNodeWithoutProducing() { .containsEntry("yano.dev-mode", "false"); } + @Test + void idleModeServesTheChainWithoutAnyProducerOrClient() { + Map props = props(YanoRunMode.IDLE); + + assertThat(props).containsEntry("yano.block-producer.enabled", "false") + .containsEntry("yano.client.enabled", "false") + .containsEntry("yano.dev-mode", "false") + .doesNotContainKeys("yano.block-producer.block-time-millis", "yano.remote.host"); + assertThat(YanoRunMode.IDLE.producesBlocks()).isFalse(); + } + @Test void envNameFollowsQuarkusMapping() { assertThat(YanoService.envName("yano.block-producer.block-time-millis")) diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceStartTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceStartTest.java new file mode 100644 index 00000000..f1825f30 --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoServiceStartTest.java @@ -0,0 +1,74 @@ +package com.bloxbean.cardano.yacicli.localcluster.yano; + +import com.bloxbean.cardano.yacicli.localcluster.ClusterConfig; +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import com.bloxbean.cardano.yacicli.util.ProcessUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A Yano start that is interrupted (a stop or reset cancelling a catch-up) must not leave the spawned process + * running untracked. + */ +@DisabledOnOs(OS.WINDOWS) +class YanoServiceStartTest { + + private static int freePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + @Test + void interruptedStartTerminatesTheSpawnedProcess(@TempDir Path home) throws Exception { + // A "yano" that never reports it started + Path pidFile = home.resolve("yano.pid"); + Path yanoBin = home.resolve("yano"); + Files.writeString(yanoBin, "#!/bin/sh\necho $$ > " + pidFile + "\nexec sleep 120\n"); + assertThat(yanoBin.toFile().setExecutable(true)).isTrue(); + + ClusterConfig clusterConfig = mock(ClusterConfig.class); + when(clusterConfig.getYanoHome()).thenReturn(home.toString()); + YanoService service = new YanoService(clusterConfig, mock(ProcessUtil.class), new YanoConfigBuilder(clusterConfig)); + ClusterInfo info = ClusterInfo.builder().yanoServerPort(freePort()).yanoHttpPort(freePort()).nodePort(3001).build(); + Path clusterFolder = Files.createDirectories(home.resolve("cluster")); + + Thread starting = new Thread(() -> service.start(info, clusterFolder, YanoRunMode.CATCH_UP, msg -> {})); + starting.start(); + for (int i = 0; i < 100 && !Files.exists(pidFile); i++) + Thread.sleep(100); + assertThat(pidFile).exists(); + Thread.sleep(200); + long pid = Long.parseLong(Files.readString(pidFile).trim()); + assertThat(ProcessHandle.of(pid).map(ProcessHandle::isAlive)).contains(true); + + starting.interrupt(); + starting.join(20_000); + + assertThat(starting.isAlive()).isFalse(); + assertThat(ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false)).isFalse(); + assertThat(service.isRunning()).isFalse(); + } + + @Test + void terminateWorksOnAnInterruptedThreadAndKeepsTheFlag() throws Exception { + Process process = new ProcessBuilder("sleep", "120").start(); + + Thread.currentThread().interrupt(); + ProcessUtil.terminate(process); + + assertThat(Thread.interrupted()).isTrue(); + assertThat(process.isAlive()).isFalse(); + } +} From 8f92d08156a05562ba72b00d0715623978dfdea1 Mon Sep 17 00:00:00 2001 From: Satya Date: Sat, 10 Oct 2026 23:22:41 +0800 Subject: [PATCH 7/9] Address review: a stop aborts the start, no live Yano without a known safe lag or a fresh backfill - F1: a stop/reset during start tells the start to give up: it starts no further process (submit-api, producer restart) and reports failure, and the stop waits for the start before stopping what it started. - F2: with devnet.auto.catch.up=false, yano-only starts LIVE only after a tip read and a safe lag; an unreadable tip fails the start. A lag that would cross more than one epoch boundary also stays idle. - F3: the yano-only backfill repeats against fresh wall clock until the gap is within the handoff slack, with a bounded failure path. - catch-up runs for an idle yano-only Yano even before the lag reaches the stability window. Co-Authored-By: Claude Opus 5.5 --- .../localcluster/ClusterStartService.java | 53 +++- .../localcluster/catchup/ChainLag.java | 22 ++ .../catchup/DevnetCatchUpService.java | 85 ++++-- .../localcluster/yano/YanoService.java | 10 + .../ClusterStartServiceStopTest.java | 243 ++++++++++++++++++ .../localcluster/catchup/ChainLagTest.java | 16 ++ .../catchup/DevnetCatchUpServiceTest.java | 84 ++++++ docs/pages/yano-node-modes.mdx | 1 + 8 files changed, 485 insertions(+), 29 deletions(-) create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java index f354f03b..46b33739 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java @@ -13,6 +13,7 @@ import com.bloxbean.cardano.yacicli.localcluster.peer.LocalPeerService; import com.bloxbean.cardano.yacicli.localcluster.yano.YanoBootstrapService; import com.bloxbean.cardano.yacicli.localcluster.yano.YanoCompanionService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoRunMode; import com.bloxbean.cardano.yacicli.localcluster.yano.YanoService; import com.bloxbean.cardano.yacicli.localcluster.yano.YanoTimeTravelBootstrap; import com.bloxbean.cardano.yacicli.util.PortUtil; @@ -66,11 +67,32 @@ public class ClusterStartService { private List processes = new CopyOnWriteArrayList<>(); // The Haskell node process (the `sh node.sh` wrapper), replaced when a catch-up restarts the node private volatile Process haskellNodeProcess; + // The thread running startCluster, and whether a stop/reset asked it to give up: it then starts no further + // process and reports the start as failed, and the stop waits for it before stopping what it started + private volatile Thread startingThread; + private volatile boolean stopRequested; private Queue logs = EvictingQueue.create(200); private Queue submitApiLogs = EvictingQueue.create(100); public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { + stopRequested = false; + startingThread = Thread.currentThread(); + try { + return doStartCluster(clusterInfo, clusterFolder, writer); + } finally { + startingThread = null; + } + } + + /** A stop or reset arrived during the start: start nothing more. */ + private boolean startCancelled(Consumer writer) { + if (stopRequested) + writer.accept(warn("Start cancelled by a stop")); + return stopRequested; + } + + private RunStatus doStartCluster(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) { logs.clear(); submitApiLogs.clear(); @@ -116,6 +138,8 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu // Restart: resume the existing chain. With auto catch-up, whatever time has passed is // backfilled first, so the live producer does not jump over the skipped epochs. boolean yanoStarted = devnetCatchUpService.startYanoOnly(clusterInfo, clusterFolder, writer); + if (startCancelled(writer)) + return new RunStatus(false, firstRun); if (!yanoStarted) { writer.accept(error("Failed to start Yano.")); return new RunStatus(false, firstRun); @@ -176,6 +200,8 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu // In companion mode, start as relay first (no forging) to sync Yano's chain cleanly. // A block producer would forge its own block at slot 1 (activeSlotsCoeff=1.0), // creating a divergent chain that prevents chain-sync from Yano. + if (startCancelled(writer)) + return new RunStatus(false, firstRun); Process nodeProcess = startNode(clusterFolder, clusterInfo, companionBootstrapDone, writer); if (nodeProcess == null) { writer.accept(error("Node process could not be started.")); @@ -204,6 +230,8 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu localPeerService.preparePeersAfterCompanionHandover(clusterName, writer); } + if (startCancelled(writer)) + return new RunStatus(false, firstRun); nodeProcess = startNode(clusterFolder, clusterInfo, false, writer); if (nodeProcess != null) { processes.add(nodeProcess); @@ -222,6 +250,9 @@ public RunStatus startCluster(ClusterInfo clusterInfo, Path clusterFolder, Consu if (companionMode && !firstRun) catchUpOnRestart(clusterInfo, clusterFolder, writer); + // A stop cancels a catch-up above; either way the start must not go on to submit-api and report success + if (startCancelled(writer)) + return new RunStatus(false, firstRun); Process submitApiProcess = startSubmitApi(clusterInfo, clusterFolder, writer); if (submitApiProcess == null) { writer.accept(error("Submit API process could not be started.")); @@ -282,6 +313,9 @@ private static boolean portAvailabilityCheck(ClusterInfo clusterInfo, Consumer writer) { try { + // Tell a start in progress to give up before cancelling its catch-up, so it does not carry on once + // the catch-up returns + stopRequested = true; // Cancel a running catch-up first: it starts no further process and does not restart the node behind // this stop (or a reset deleting the database) if (devnetCatchUpService.isInProgress()) { @@ -289,6 +323,9 @@ public void stopCluster(Consumer writer) { if (!devnetCatchUpService.cancelAndAwait(java.time.Duration.ofSeconds(60))) writer.accept(warn("The catch-up did not stop within 60s; it will not start any further process")); } + // Then wait for the start itself: whatever it started is in the process list by the time it returns + if (!awaitStart(java.time.Duration.ofSeconds(90))) + writer.accept(warn("The start in progress did not finish within 90s; it will not start any further process")); if (processes != null && processes.size() > 0) writer.accept(info("Trying to stop the running cluster ...")); @@ -322,6 +359,17 @@ public void stopCluster(Consumer writer) { processes.clear(); } + /** Wait for a start running on another thread to return. */ + private boolean awaitStart(java.time.Duration timeout) throws InterruptedException { + Thread starting = startingThread; + if (starting == null || starting == Thread.currentThread()) + return true; + long deadline = System.currentTimeMillis() + timeout.toMillis(); + while (startingThread != null && System.currentTimeMillis() < deadline) + Thread.sleep(100); + return startingThread == null; + } + public void showLogs(Consumer consumer) { if (logs.isEmpty()) { consumer.accept("No log to show"); @@ -517,7 +565,10 @@ public DevnetCatchUpService.Result catchUp(ClusterInfo clusterInfo, Path cluster ChainLag lag = devnetCatchUpService.probeLag(clusterInfo); if (lag == null) return new DevnetCatchUpService.Result(false, "Could not read the chain tip", -1, -1, 0, 0); - if (!force && !lag.stalled()) + // A yano-only devnet whose Yano is not producing (left idle by a start with auto catch-up off) needs a + // catch-up even before the lag reaches the stability window + boolean yanoIdle = clusterInfo.getNodeMode() == NodeMode.YANO_ONLY && yanoService.runMode() != YanoRunMode.LIVE; + if (!force && !yanoIdle && !lag.stalled()) return new DevnetCatchUpService.Result(true, String.format( "The chain is live: the tip is %,d slots behind wall clock (stability window %,d slots). Nothing to catch up.", lag.lagSlots(), lag.forecastWindowSlots()), lag.tipSlot(), lag.tipSlot(), 0, 0); diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java index 4d12839a..28bb53e7 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLag.java @@ -60,6 +60,28 @@ public boolean needsCatchUpOnStart() { return forecastWindowSlots > 0 && lagSlots() * 4 > forecastWindowSlots * 3; } + /** + * Yano-only: whether the live producer can start at wall clock with no backfill. Its first block processes one + * epoch boundary as usual but jumps over any further ones ("epoch boundary 3 → 6"), which leaves those epochs + * without protocol parameters and rewards. A securityParam set by hand can make the forecast window longer than + * an epoch, so the lag threshold alone does not rule that out. + */ + public boolean liveStartSafe() { + return !needsCatchUpOnStart() && epochsToCross() <= 1; + } + + /** + * Yano-only: how close a backfill must get to wall clock before the live producer starts. The Yano restart in + * between must not let the gap cross a second epoch boundary, so at most half an epoch, and at most a quarter of + * the forecast window, as in companion mode. + */ + public long yanoHandoffSlackSlots() { + long slack = forecastWindowSlots > 0 ? forecastWindowSlots / 4 : epochLength / 2; + if (epochLength > 0) + slack = Math.min(slack, epochLength / 2); + return Math.max(2, slack); + } + public long tipEpoch() { return epochLength > 0 ? tipSlot / epochLength : 0; } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java index c86af964..d476d49a 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpService.java @@ -423,17 +423,28 @@ public Result catchUpYanoOnly(ClusterInfo clusterInfo, Path clusterFolder, Consu private boolean startYanoOnlyWithoutCatchUp(ClusterInfo clusterInfo, Path clusterFolder, Consumer writer) throws InterruptedException { int httpPort = clusterInfo.getYanoHttpPort(); - // Look at the tip with Yano serving but never forging, as the companion start does with its stalled producer - if (yanoService.start(clusterInfo, clusterFolder, YanoRunMode.IDLE, problemsOnly(writer)) - && yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) { - Tuple tip = yanoBootstrapService.getNodeTip(httpPort); - ChainLag lag = tip != null ? ChainLag.of(clusterInfo, tip._2.getSlot(), clock.getAsLong()) : null; - if (lag != null && lag.needsCatchUpOnStart()) { - writer.accept(warn("The chain is %s behind wall clock%s. Yano is running but not producing blocks; " - + "run 'catch-up' to bring it to wall clock.", lag.idleTimeText(), - lag.epochsToCross() > 0 ? String.format(" (%s)", epochs(lag.epochsToCross())) : "")); - return true; - } + // Look at the tip with Yano serving but never forging, as the companion start does with its stalled producer. + // Live production starts only once the tip is known and the lag is safe: an unknown lag is no permission. + if (!yanoService.start(clusterInfo, clusterFolder, YanoRunMode.IDLE, problemsOnly(writer)) + || !yanoBootstrapService.waitForNodeTip(httpPort, YANO_READY_TIMEOUT)) { + yanoService.stopQuietly(); + writer.accept(error("Yano did not start to check the chain tip (see 'yano-logs'); not starting block " + + "production, so no missed epoch is skipped. Retry with 'start'.")); + return false; + } + Tuple tip = yanoBootstrapService.getNodeTip(httpPort); + if (tip == null || tip._2 == null) { + yanoService.stopQuietly(); + writer.accept(error("Could not read the chain tip from Yano; not starting block production. " + + "Retry with 'start'.")); + return false; + } + ChainLag lag = ChainLag.of(clusterInfo, tip._2.getSlot(), clock.getAsLong()); + if (!lag.liveStartSafe()) { + writer.accept(warn("The chain is %s behind wall clock%s. Yano is running but not producing blocks; " + + "run 'catch-up' to bring it to wall clock.", lag.idleTimeText(), + lag.epochsToCross() > 0 ? String.format(" (%s)", epochs(lag.epochsToCross())) : "")); + return true; } yanoService.stopQuietly(); checkCancelled(); @@ -475,24 +486,28 @@ private YanoBackfill backfillYanoOnly(ClusterInfo clusterInfo, Path clusterFolde if (report) printBanner(lag, task.logWriter()); Step step = report ? task.step("Backfill to wall clock") : null; - checkCancelled(); - JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); - checkCancelled(); - if (caughtUp == null) { - if (step != null) - step.fail("Yano's catch-up call failed"); - // Inside one epoch the first live block skips no boundary, so starting is harmless. Measure again: - // the failed call may have taken minutes, and epochs may have passed meanwhile. - ChainLag now = ChainLag.of(clusterInfo, lag.tipSlot(), clock.getAsLong()); - if (now.epochsToCross() == 0) - return new YanoBackfill(true, "Catch-up failed; the gap is inside one epoch", now.tipSlot(), - now.tipSlot(), 0); - return new YanoBackfill(false, String.format("Yano's catch-up call failed (see 'yano-logs'). Not starting " - + "block production: its first block would skip %s. Retry with 'catch-up'.", - epochs(now.epochsToCross())), now.tipSlot(), now.tipSlot(), 0); + long toSlot = lag.tipSlot(); + long blocks = 0; + long slack = lag.yanoHandoffSlackSlots(); + // Yano backfills to the wall clock it reads when the call comes in, so a long backfill ends behind the + // current one. Repeat until the gap left for the live producer's restart is small. + for (int round = 1; ; round++) { + checkCancelled(); + JsonNode caughtUp = yanoBootstrapService.catchUp(httpPort); + checkCancelled(); + // Measure again after every call: it may have taken minutes, and epochs may have passed meanwhile + if (caughtUp == null) + return gaveUp(step, "Yano's catch-up call failed", + ChainLag.of(clusterInfo, toSlot, clock.getAsLong())); + toSlot = caughtUp.path("new_slot").asLong(toSlot); + blocks += caughtUp.path("blocks_produced").asLong(0); + ChainLag now = ChainLag.of(clusterInfo, toSlot, clock.getAsLong()); + if (now.lagSlots() <= slack) + break; + if (round >= MAX_PUMP_ROUNDS) + return gaveUp(step, String.format("Yano's backfill could not keep up with wall clock (still %,d " + + "slots behind)", now.lagSlots()), now); } - long toSlot = caughtUp.path("new_slot").asLong(); - long blocks = caughtUp.path("blocks_produced").asLong(); String message = resumedMessage(clusterInfo, lag, toSlot, clock.getAsLong() - startedAt); if (report) { step.done(String.format("%,d blocks to slot %,d", blocks, toSlot)); @@ -508,6 +523,20 @@ private YanoBackfill backfillYanoOnly(ClusterInfo clusterInfo, Path clusterFolde } } + /** + * A yano-only backfill that stopped short of wall clock. Inside one epoch the first live block skips no boundary, + * so starting is harmless; otherwise live production is not started. + */ + private YanoBackfill gaveUp(Step step, String why, ChainLag now) { + if (step != null) + step.fail(why); + if (now.epochsToCross() == 0) + return new YanoBackfill(true, why + "; the gap is inside one epoch", now.tipSlot(), now.tipSlot(), 0); + return new YanoBackfill(false, String.format("%s (see 'yano-logs'). Not starting block production: its first " + + "block would skip %s. Retry with 'catch-up'.", why, epochs(now.epochsToCross())), + now.tipSlot(), now.tipSlot(), 0); + } + // --------------------------------------------------------------------------------------------------------- private void printBanner(ChainLag lag, Consumer writer) { diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java index 2e9a5037..f624891c 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/yano/YanoService.java @@ -72,6 +72,8 @@ public class YanoService { private String requiredYanoVersion = ""; private final AtomicBoolean versionChecked = new AtomicBoolean(false); + // How the running Yano was started, or null when it is not running + private volatile YanoRunMode runMode; public boolean start(ClusterInfo clusterInfo, Path clusterFolder, boolean pastTimeTravelMode, Consumer writer) { return start(clusterInfo, clusterFolder, pastTimeTravelMode ? YanoRunMode.PAST_TIME_TRAVEL : YanoRunMode.LIVE, @@ -398,6 +400,8 @@ private Process startYanoProcess(ClusterInfo clusterInfo, Path clusterFolder, Pa boolean started = false; try { started = awaitYanoStart(process, clusterInfo, runMode, writer); + if (started) + this.runMode = runMode; return started ? process : null; } finally { if (!started) { @@ -537,6 +541,7 @@ private boolean stop(Consumer out) { return false; } finally { processes.clear(); + runMode = null; } return true; } @@ -545,6 +550,11 @@ public boolean isRunning() { return processes.stream().anyMatch(Process::isAlive); } + /** How the running Yano was started ({@link YanoRunMode#IDLE} waits for a catch-up), or null when it is not running. */ + public YanoRunMode runMode() { + return isRunning() ? runMode : null; + } + public void showLogs(Consumer consumer) { if (logs.isEmpty()) { consumer.accept("No Yano logs to show"); diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java new file mode 100644 index 00000000..0d81ca6f --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java @@ -0,0 +1,243 @@ +package com.bloxbean.cardano.yacicli.localcluster; + +import com.bloxbean.cardano.yacicli.localcluster.catchup.ChainLag; +import com.bloxbean.cardano.yacicli.localcluster.catchup.DevnetCatchUpService; +import com.bloxbean.cardano.yacicli.localcluster.config.CustomGenesisConfig; +import com.bloxbean.cardano.yacicli.localcluster.config.GenesisConfig; +import com.bloxbean.cardano.yacicli.localcluster.model.RunStatus; +import com.bloxbean.cardano.yacicli.localcluster.peer.LocalPeerService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoBootstrapService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoCompanionService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoRunMode; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoService; +import com.bloxbean.cardano.yacicli.localcluster.yano.YanoTimeTravelBootstrap; +import com.bloxbean.cardano.yacicli.util.ProcessUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** A stop or reset during a start: the start gives up and the stop waits for it (PR #196 review, F1). */ +class ClusterStartServiceStopTest { + @TempDir + Path tmp; + + private Path clusterFolder; + private ProcessUtil processUtil; + private DevnetCatchUpService catchUp; + private YanoService yano; + private ClusterStartService service; + private final List started = new CopyOnWriteArrayList<>(); + private final List output = new CopyOnWriteArrayList<>(); + private final Consumer writer = output::add; + + @BeforeEach + void setUp() throws Exception { + clusterFolder = tmp.resolve("devnet"); + Path node = clusterFolder.resolve(ClusterConfig.NODE_FOLDER_PREFIX); + Files.createDirectories(node.resolve("db")); // an existing devnet: a restart, not a first run + Files.createFile(node.resolve("node.sock")); // the node's socket is there at once + Path bin = Files.createDirectories(tmp.resolve("bin")); + Files.createFile(bin.resolve("cardano-submit-api")); + + ClusterConfig clusterConfig = mock(ClusterConfig.class); + when(clusterConfig.getCLIBinFolder()).thenReturn(bin.toString()); + processUtil = mock(ProcessUtil.class); + when(processUtil.startLongRunningProcess(any(), any(), any(), any())).thenAnswer(invocation -> { + FakeProcess process = new FakeProcess(invocation.getArgument(0)); + started.add(process); + return process; + }); + catchUp = mock(DevnetCatchUpService.class); + yano = mock(YanoService.class); + service = new ClusterStartService(clusterConfig, mock(ClusterPortInfoHelper.class), processUtil, + mock(GenesisConfig.class), mock(CustomGenesisConfig.class), mock(LocalPeerService.class), + mock(YanoCompanionService.class), yano, mock(YanoBootstrapService.class), + mock(YanoTimeTravelBootstrap.class), catchUp); + + when(catchUp.unsupportedReason(any())).thenReturn(Optional.empty()); + when(catchUp.isAutoCatchUp()).thenReturn(true); + // Far behind wall clock: the restart catches up + when(catchUp.probeLagAfterStart(any(), any())).thenReturn(new ChainLag(100, 1010, 21, 40, 1.0)); + } + + private ClusterInfo clusterInfo(NodeMode mode) throws Exception { + return ClusterInfo.builder() + .nodeMode(mode).masterNode(true).nodePort(freePort()).submitApiPort(freePort()) + .startTime(System.currentTimeMillis() / 1000 - 1010).slotLength(1.0).blockTime(1.0).epochLength(40) + .securityParam(7).activeSlotsCoeff(1.0).yanoHttpPort(6060) + .build(); + } + + private static int freePort() throws Exception { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + private void verifySubmitApiNeverStarted() throws Exception { + verify(processUtil, never()).startLongRunningProcess(eq(ClusterStartService.SUBMIT_API_PROCESS_NAME), + any(), any(), any()); + } + + @Test + void stopDuringTheRestartCatchUpAbortsTheStartAndWaitsForIt() throws Exception { + AtomicBoolean inProgress = new AtomicBoolean(false); + CountDownLatch inCatchUp = new CountDownLatch(1); + CountDownLatch cancel = new CountDownLatch(1); + when(catchUp.isInProgress()).thenAnswer(invocation -> inProgress.get()); + when(catchUp.catchUpCompanion(any(), any(), any(), any(), any())).thenAnswer(invocation -> { + inProgress.set(true); + inCatchUp.countDown(); + cancel.await(10, TimeUnit.SECONDS); + // A cancelled catch-up clears its interrupt and only reports the cancel + Thread.sleep(200); + inProgress.set(false); + return new DevnetCatchUpService.Result(false, "Catch-up cancelled", -1, -1, 0, 0); + }); + when(catchUp.cancelAndAwait(any())).thenAnswer(invocation -> { + cancel.countDown(); + while (inProgress.get()) + Thread.sleep(20); + return true; + }); + ClusterInfo info = clusterInfo(NodeMode.COMPANION); + + CompletableFuture start = CompletableFuture.supplyAsync(() -> + service.startCluster(info, clusterFolder, writer)); + assertThat(inCatchUp.await(10, TimeUnit.SECONDS)).isTrue(); + + service.stopCluster(writer); + + // The node the start had already launched was stopped, and nothing was started or registered after that + assertThat(started).hasSize(1); + assertThat(started.get(0).isAlive()).isFalse(); + RunStatus status = start.get(10, TimeUnit.SECONDS); + assertThat(status.stared()).isFalse(); + verifySubmitApiNeverStarted(); + assertThat(started).hasSize(1); + assertThat(service.isClusterRunning()).isFalse(); + assertThat(String.join("\n", output)).contains("Start cancelled by a stop"); + } + + @Test + void aFailedCatchUpWithoutAStopStillStartsTheDevnet() throws Exception { + // A stop with nothing running must not leave the next start cancelled + service.stopCluster(writer); + when(catchUp.catchUpCompanion(any(), any(), any(), any(), any())).thenReturn( + new DevnetCatchUpService.Result(false, "Yano's catch-up call failed", -1, -1, 0, 0)); + + RunStatus status = service.startCluster(clusterInfo(NodeMode.COMPANION), clusterFolder, writer); + + // The catch-up leaves the producer running; the devnet stays usable and 'catch-up' can be retried + assertThat(status.stared()).isTrue(); + verify(processUtil).startLongRunningProcess(eq(ClusterStartService.SUBMIT_API_PROCESS_NAME), any(), any(), any()); + service.stopCluster(writer); + } + + @Test + void catchUpCommandRunsForAnIdleYanoOnlyDevnetInsideTheWindow() throws Exception { + when(yano.isRunning()).thenReturn(true); + when(yano.runMode()).thenReturn(YanoRunMode.IDLE); + // 16 slots behind with a 21-slot window: not stalled, but Yano is not producing + when(catchUp.probeLag(any())).thenReturn(new ChainLag(994, 1010, 21, 40, 1.0)); + when(catchUp.catchUpYanoOnly(any(), any(), any())).thenReturn( + new DevnetCatchUpService.Result(true, "resumed", 994, 1010, 2, 1000)); + + DevnetCatchUpService.Result result = service.catchUp(clusterInfo(NodeMode.YANO_ONLY), clusterFolder, false, writer); + + assertThat(result.success()).isTrue(); + verify(catchUp).catchUpYanoOnly(any(), any(), any()); + } + + @Test + void catchUpCommandIsANoOpForALiveYanoOnlyDevnet() throws Exception { + when(yano.isRunning()).thenReturn(true); + when(yano.runMode()).thenReturn(YanoRunMode.LIVE); + when(catchUp.probeLag(any())).thenReturn(new ChainLag(1009, 1010, 21, 40, 1.0)); + + DevnetCatchUpService.Result result = service.catchUp(clusterInfo(NodeMode.YANO_ONLY), clusterFolder, false, writer); + + assertThat(result.message()).contains("Nothing to catch up"); + verify(catchUp, never()).catchUpYanoOnly(any(), any(), any()); + } + + /** A process that runs until it is destroyed. */ + private static final class FakeProcess extends Process { + private final String name; + private volatile boolean alive = true; + + FakeProcess(String name) { + this.name = name; + } + + @Override + public OutputStream getOutputStream() { + return OutputStream.nullOutputStream(); + } + + @Override + public InputStream getInputStream() { + return InputStream.nullInputStream(); + } + + @Override + public InputStream getErrorStream() { + return InputStream.nullInputStream(); + } + + @Override + public int waitFor() { + return 0; + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) { + return !alive; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() { + alive = false; + } + + @Override + public boolean isAlive() { + return alive; + } + + @Override + public Stream descendants() { + return Stream.empty(); + } + + @Override + public String toString() { + return "FakeProcess[" + name + "]"; + } + } +} diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java index a7330216..bb196113 100644 --- a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/ChainLagTest.java @@ -90,4 +90,20 @@ void idleTimeTextShowsTheTwoLargestUnits() { assertThat(ChainLag.durationText(Duration.ofMinutes(125))).isEqualTo("2h 5m"); assertThat(ChainLag.durationText(Duration.ofHours(76).plusMinutes(12))).isEqualTo("3d 4h"); } + + @Test + void liveStartIsSafeInsideTheThresholdAndAcrossAtMostOneEpochBoundary() { + assertThat(lag(1000, 1100).liveStartSafe()).isTrue(); // epoch 1, nothing crossed + assertThat(lag(1100, 1250).liveStartSafe()).isTrue(); // one boundary (epoch 1 -> 2) + assertThat(lag(1000, 1226).liveStartSafe()).isFalse(); // past 3/4 of the window + // A window longer than the epoch (securityParam set by hand): a short lag can cross several boundaries + assertThat(new ChainLag(100, 200, 210, 40, 1.0).liveStartSafe()).isFalse(); + } + + @Test + void yanoHandoffSlackIsAQuarterWindowAndAtMostHalfAnEpoch() { + assertThat(lag(0, 0).yanoHandoffSlackSlots()).isEqualTo(75); // 300 / 4 + assertThat(new ChainLag(0, 0, 210, 40, 1.0).yanoHandoffSlackSlots()).isEqualTo(20); // 40 / 2 + assertThat(new ChainLag(0, 0, 4, 40, 1.0).yanoHandoffSlackSlots()).isEqualTo(2); // floor of 2 + } } diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java index 007e4f42..a9c92bb8 100644 --- a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetCatchUpServiceTest.java @@ -195,6 +195,90 @@ void yanoOnlyStartWithAutoCatchUpOffGoesLiveAtTheTip() { verify(yano).start(any(), any(), eq(YanoRunMode.LIVE), any()); } + @Test + void yanoOnlyStartWithAutoCatchUpOffFailsWhenYanoNeverAnswers() { + ReflectionTestUtils.setField(service, "autoCatchUp", false); + when(bootstrap.waitForNodeTip(eq(HTTP_PORT), any())).thenReturn(false); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + // Not knowing the lag is no permission to forge + assertThat(started).isFalse(); + verifyNoLiveStart(); + verify(yano).stopQuietly(); + assertThat(String.join("\n", output)).contains("Retry with 'start'"); + } + + @Test + void yanoOnlyStartWithAutoCatchUpOffFailsWhenTheTipCannotBeRead() { + ReflectionTestUtils.setField(service, "autoCatchUp", false); + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(null); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isFalse(); + verifyNoLiveStart(); + verify(yano).stopQuietly(); + assertThat(String.join("\n", output)).contains("Could not read the chain tip"); + } + + @Test + void yanoOnlyStartWithAutoCatchUpOffStaysIdleWhenAShortLagCrossesSeveralEpochs() { + ReflectionTestUtils.setField(service, "autoCatchUp", false); + // securityParam 70: a 210-slot window, longer than the 40-slot epoch. 100 slots behind is inside the + // threshold, yet a live block would jump from epoch 22 to 25. + ClusterInfo info = clusterInfo(NodeMode.YANO_ONLY); + info.setSecurityParam(70); + when(bootstrap.getNodeTip(HTTP_PORT)).thenAnswer(invocation -> tip(wallSlot() - 100)); + + boolean started = service.startYanoOnly(info, FOLDER, writer); + + assertThat(started).isTrue(); + verify(yano).start(any(), any(), eq(YanoRunMode.IDLE), any()); + verifyNoLiveStart(); + assertThat(String.join("\n", output)).contains("run 'catch-up'"); + } + + @Test + void yanoOnlySlowBackfillIsRepeatedUntilItReachesTheCurrentWallClock() { + long[] now = {(startTime + 1010) * 1000}; + service.clock = () -> now[0]; + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); + // The first call reaches the wall clock it saw when it came in (slot 1,010), but takes 200 s: five epochs + // pass meanwhile. The second reaches the wall clock of its own start. + when(bootstrap.catchUp(HTTP_PORT)).thenAnswer(invocation -> { + now[0] += 200_000; + return caughtUp(1010, 30); + }).thenAnswer(invocation -> caughtUp(1210, 6)); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isTrue(); + var order = inOrder(yano, bootstrap); + order.verify(bootstrap, times(2)).catchUp(HTTP_PORT); + order.verify(yano).start(any(), any(), eq(YanoRunMode.LIVE), any()); + assertThat(String.join("\n", output)).contains("36 blocks to slot 1,210"); + } + + @Test + void yanoOnlyBackfillThatNeverCatchesUpDoesNotStartLiveProduction() { + long[] now = {(startTime + 1010) * 1000}; + service.clock = () -> now[0]; + when(bootstrap.getNodeTip(HTTP_PORT)).thenReturn(tip(100)); + // Every call ends 200 s behind the wall clock + when(bootstrap.catchUp(HTTP_PORT)).thenAnswer(invocation -> { + long target = ChainLag.wallClockSlot(startTime, 1.0, now[0]); + now[0] += 200_000; + return caughtUp(target, 6); + }); + + boolean started = service.startYanoOnly(clusterInfo(NodeMode.YANO_ONLY), FOLDER, writer); + + assertThat(started).isFalse(); + verifyNoLiveStart(); + assertThat(String.join("\n", output)).contains("could not keep up with wall clock").contains("would skip 5 epochs"); + } + // --- Companion ------------------------------------------------------------------------------------------ @Test diff --git a/docs/pages/yano-node-modes.mdx b/docs/pages/yano-node-modes.mdx index 9b84d443..7fdf8fed 100644 --- a/docs/pages/yano-node-modes.mdx +++ b/docs/pages/yano-node-modes.mdx @@ -165,6 +165,7 @@ devnet.auto.catch.up=false The chain then waits for you: a companion block producer cannot forge, and in yano-only mode Yano starts without producing blocks, so no missed epoch is skipped. `catch-up` brings it to wall clock and starts block production. +If Yano cannot read its chain tip on `start`, the start fails instead of producing blocks; run `start` again. Time moves forward to wall clock, so every epoch that passed is processed. Governance actions and DRep activity From e2c88bfe8c4d56657593b6903037a9537de89331 Mon Sep 17 00:00:00 2001 From: Satya Date: Mon, 12 Oct 2026 22:14:56 +0800 Subject: [PATCH 8/9] Bump Yaci Store to 2.0.7 2.0.7 upgrades yaci to 0.4.7, which fixes reading mainnet block 14,053,493 as having no transactions (bloxbean/yaci-store#1247). Devnets are not affected by that block, but DevKit should ship the current Store. Native binaries from rel-native-2.0.7, jar from v2.0.7, Docker image built from the v2.0.7 tag (4bca8bf). Asset names are unchanged from 2.0.6. Verified with native builds only: the native Docker image (Store built from v2.0.7) and the native macOS CLI with the rel-native-2.0.7 binaries downloaded by 'download -c yaci-store'. SDK e2e 13/13 in Docker companion, native companion and native yano-only; Store restarts on its existing index. Co-Authored-By: Claude Opus 5.5 --- Earthfile | 2 +- applications/cli/Earthfile | 2 +- applications/cli/config/download.properties | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Earthfile b/Earthfile index b45f05f7..14f60218 100644 --- a/Earthfile +++ b/Earthfile @@ -5,7 +5,7 @@ ARG --global tag="dev" ARG --global local="true" ARG --global REGISTRY_ORG = "bloxbean" ARG --global build_type="native" -ARG --global STORE_NATIVE_BRANCH="v2.0.6" +ARG --global STORE_NATIVE_BRANCH="v2.0.7" ARG --global YANO_TAG="v0.1.0-pre17" ARG --global YANO_VERSION="0.1.0-pre17" diff --git a/applications/cli/Earthfile b/applications/cli/Earthfile index 9e7c96e0..364d9c05 100644 --- a/applications/cli/Earthfile +++ b/applications/cli/Earthfile @@ -67,7 +67,7 @@ docker-build: ARG APP_VERSION ARG REGISTRY_ORG ARG COMMIT_ID - ARG STORE_NATIVE_BRANCH="v2.0.6" + ARG STORE_NATIVE_BRANCH="v2.0.7" ARG YANO_TAG="v0.1.0-pre17" ARG YANO_VERSION="0.1.0-pre17" diff --git a/applications/cli/config/download.properties b/applications/cli/config/download.properties index c1628536..3106e3e2 100644 --- a/applications/cli/config/download.properties +++ b/applications/cli/config/download.properties @@ -3,9 +3,9 @@ node.version=11.0.1 ogmios.version=7.0.0 kupo.version=2.12.0 -yaci.store.tag=rel-native-2.0.6 -yaci.store.version=2.0.6 -yaci.store.jar.version=2.0.6 +yaci.store.tag=rel-native-2.0.7 +yaci.store.version=2.0.7 +yaci.store.jar.version=2.0.7 yano.tag=v0.1.0-pre17 yano.version=0.1.0-pre17 From 44e0fd28e598e6aa03b2c960852efe1652902078 Mon Sep 17 00:00:00 2001 From: Satya Date: Mon, 12 Oct 2026 22:15:05 +0800 Subject: [PATCH 9/9] Catch-up in Docker: stop the node before its shell; free a watchdog stuck on a stopped node Found running the native Docker image after a long pause (Store 2.0.7 test): - The restart's catch-up failed at step 1 with "Could not stop the Haskell node", and the devnet stayed stalled. stopProcessTree signalled 'sh node.sh' and cardano-node together; the shell died first, the node was re-parented to PID 1, which in the Docker image is 'sleep infinity' and never reaps it. The zombie still counts as alive for ProcessHandle.isAlive(), so the stop never succeeded. Now the children are stopped first, while their shell is there to reap them, and a Linux zombie (/proc state Z) counts as gone. - A watchdog check that ran while 'stop' was stopping the node found the stale node.sock and hung in yaci's connect retry (every 8 s, forever) on the watchdog's only thread, logging an ERROR with a stack trace each time. ClusterStopped/ClusterDeleted now cancel the check with an interrupt, which ends the retry, and LocalNodeService shuts its client down when start() fails, so the client does not keep reconnecting. Tests: stopProcessTreeStopsTheNodeWhileItsShellStillRunsToReapIt, aProcessThatHasExitedIsNotRunning, and DevnetStallWatchdogTest.aStopInterruptsACheckStillReadingTheStoppedNodesTip. Both regression tests fail with the old code. Co-Authored-By: Claude Opus 5.5 --- .../localcluster/ClusterStartService.java | 25 +++++-- .../catchup/DevnetStallWatchdog.java | 19 +++--- .../service/LocalNodeService.java | 8 ++- .../ClusterStartServiceStopTest.java | 27 ++++++++ .../catchup/DevnetStallWatchdogTest.java | 65 +++++++++++++++++++ 5 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdogTest.java diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java index 46b33739..007b1fd3 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartService.java @@ -456,26 +456,43 @@ private Process startNode(Path clusterFolder, ClusterInfo clusterInfo, boolean a * Stop a process and its children, gracefully first. The node runs as a child of {@code sh node.sh}, so the * signal has to reach it directly: SIGTERM lets cardano-node close its ChainDB (0.1 s), while a forced kill * leaves it unclean and the next start revalidates every chunk, which grows with chain length. + *

+ * The children go first, while their shell still runs and reaps them. Stopping both at once orphans the node, + * and in Docker its new parent ({@code sleep infinity}, PID 1) never reaps it: it stays a zombie, which + * {@link ProcessHandle#isAlive()} still reports as alive. * * @return true if the process tree is gone */ static boolean stopProcessTree(Process process) throws InterruptedException { List children = process.descendants().toList(); children.forEach(ProcessHandle::destroy); - process.destroy(); long deadline = System.currentTimeMillis() + 15_000; for (ProcessHandle child : children) { - while (child.isAlive() && System.currentTimeMillis() < deadline) + while (isRunning(child) && System.currentTimeMillis() < deadline) Thread.sleep(100); - if (child.isAlive()) + if (isRunning(child)) child.destroyForcibly(); } + process.destroy(); if (!process.waitFor(Math.max(1, deadline - System.currentTimeMillis()), TimeUnit.MILLISECONDS)) { process.destroyForcibly(); process.waitFor(5, TimeUnit.SECONDS); } - return !process.isAlive() && children.stream().noneMatch(ProcessHandle::isAlive); + return !process.isAlive() && children.stream().noneMatch(ClusterStartService::isRunning); + } + + /** Alive and not a zombie (Linux {@code /proc//stat} state Z); where there is no /proc, just alive. */ + static boolean isRunning(ProcessHandle handle) { + if (!handle.isAlive()) + return false; + try { + String stat = Files.readString(Path.of("/proc", String.valueOf(handle.pid()), "stat")); + int commEnd = stat.lastIndexOf(')'); // the state follows "(comm) " + return commEnd < 0 || commEnd + 2 >= stat.length() || stat.charAt(commEnd + 2) != 'Z'; + } catch (IOException | RuntimeException e) { + return handle.isAlive(); + } } /** The Haskell node of this devnet, as a {@link NodeControl} a catch-up can restart as relay or producer. */ diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java index 69b39375..72365d48 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdog.java @@ -50,35 +50,38 @@ public class DevnetStallWatchdog { private ScheduledFuture check; private volatile String clusterName; private volatile boolean hintShown; + long intervalMillis = TimeUnit.SECONDS.toMillis(CHECK_INTERVAL_SECONDS); @EventListener public synchronized void handleClusterStarted(ClusterStarted event) { - stopChecking(); + stopChecking(false); clusterName = event.getClusterName(); hintShown = false; - check = scheduler.scheduleWithFixedDelay(this::checkSafely, CHECK_INTERVAL_SECONDS, CHECK_INTERVAL_SECONDS, - TimeUnit.SECONDS); + check = scheduler.scheduleWithFixedDelay(this::checkSafely, intervalMillis, intervalMillis, + TimeUnit.MILLISECONDS); } + // The node is gone: interrupt a check still reading its tip. ClusterStopped comes after the stop, and a check + // that started meanwhile finds the node's stale socket; the node client retries that connect until interrupted. @EventListener public void handleClusterStopped(ClusterStopped event) { - stopChecking(); + stopChecking(true); } @EventListener public void handleClusterDeleted(ClusterDeleted event) { - stopChecking(); + stopChecking(true); } @PreDestroy public void shutdown() { - stopChecking(); + stopChecking(true); scheduler.shutdownNow(); } - private synchronized void stopChecking() { + private synchronized void stopChecking(boolean interrupt) { if (check != null) { - check.cancel(false); + check.cancel(interrupt); check = null; } } diff --git a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/LocalNodeService.java b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/LocalNodeService.java index 18936fa5..603bb246 100644 --- a/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/LocalNodeService.java +++ b/applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/service/LocalNodeService.java @@ -104,7 +104,13 @@ public void txAccepted(TxSubmissionRequest txSubmissionRequest, MsgAcceptTx msgA public void txRejected(TxSubmissionRequest txSubmissionRequest, MsgRejectTx msgRejectTx) { } }); - this.localClientProvider.start(); + try { + this.localClientProvider.start(); + } catch (Exception e) { + // The caller never gets this service to shut down: stop the client's reconnects here + localClientProvider.shutdown(); + throw e; + } this.utxoSupplier = new LocalUtxoSupplier(localClientProvider.getLocalStateQueryClient(), era); this.protocolParamsSupplier = new LocalProtocolParamSupplier(localClientProvider.getLocalStateQueryClient(), era); diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java index 0d81ca6f..c76cf79f 100644 --- a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/ClusterStartServiceStopTest.java @@ -181,6 +181,33 @@ void catchUpCommandIsANoOpForALiveYanoOnlyDevnet() throws Exception { verify(catchUp, never()).catchUpYanoOnly(any(), any(), any()); } + @Test + void stopProcessTreeStopsTheNodeWhileItsShellStillRunsToReapIt() throws Exception { + // Like node.sh: a shell running the node. The marker is written only if the shell outlives the node + Path marker = tmp.resolve("reaped"); + Process shell = new ProcessBuilder("bash", "-c", "sleep 600; echo reaped > '" + marker + "'").start(); + long deadline = System.currentTimeMillis() + 5_000; + while (shell.descendants().findAny().isEmpty() && System.currentTimeMillis() < deadline) + Thread.sleep(20); + ProcessHandle node = shell.descendants().findFirst().orElseThrow(); + + assertThat(ClusterStartService.stopProcessTree(shell)).isTrue(); + + // Had the shell been stopped with the node, the node would be orphaned: in Docker, PID 1 ('sleep infinity') + // never reaps it, and the zombie still counts as alive + assertThat(node.isAlive()).isFalse(); + assertThat(shell.isAlive()).isFalse(); + assertThat(marker).exists(); + } + + @Test + void aProcessThatHasExitedIsNotRunning() throws Exception { + Process process = new ProcessBuilder("true").start(); + process.waitFor(5, TimeUnit.SECONDS); + assertThat(ClusterStartService.isRunning(process.toHandle())).isFalse(); + assertThat(ClusterStartService.isRunning(ProcessHandle.current())).isTrue(); + } + /** A process that runs until it is destroyed. */ private static final class FakeProcess extends Process { private final String name; diff --git a/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdogTest.java b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdogTest.java new file mode 100644 index 00000000..76493caf --- /dev/null +++ b/applications/cli/src/test/java/com/bloxbean/cardano/yacicli/localcluster/catchup/DevnetStallWatchdogTest.java @@ -0,0 +1,65 @@ +package com.bloxbean.cardano.yacicli.localcluster.catchup; + +import com.bloxbean.cardano.yacicli.localcluster.ClusterInfo; +import com.bloxbean.cardano.yacicli.localcluster.ClusterService; +import com.bloxbean.cardano.yacicli.localcluster.NodeMode; +import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStarted; +import com.bloxbean.cardano.yacicli.localcluster.events.ClusterStopped; +import org.jline.reader.LineReader; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; + +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DevnetStallWatchdogTest { + private DevnetStallWatchdog watchdog; + + @AfterEach + void tearDown() { + if (watchdog != null) + watchdog.shutdown(); + } + + @Test + @SuppressWarnings("unchecked") + void aStopInterruptsACheckStillReadingTheStoppedNodesTip() throws Exception { + ClusterService clusterService = mock(ClusterService.class); + ObjectProvider clusterServiceProvider = mock(ObjectProvider.class); + when(clusterServiceProvider.getObject()).thenReturn(clusterService); + DevnetCatchUpService catchUp = mock(DevnetCatchUpService.class); + when(catchUp.unsupportedReason(any())).thenReturn(Optional.empty()); + when(clusterService.getClusterInfo("default")) + .thenReturn(ClusterInfo.builder().nodeMode(NodeMode.COMPANION).build()); + + // The node client retries a refused connect (a stale socket after the stop) until interrupted + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + when(clusterService.chainLag("default")).thenAnswer(invocation -> { + reading.countDown(); + try { + Thread.sleep(60_000); + } catch (InterruptedException e) { + interrupted.countDown(); + throw e; + } + return null; + }); + + watchdog = new DevnetStallWatchdog(clusterServiceProvider, catchUp, mock(ObjectProvider.class)); + watchdog.intervalMillis = 50; + watchdog.handleClusterStarted(new ClusterStarted("default")); + assertThat(reading.await(5, TimeUnit.SECONDS)).isTrue(); + + watchdog.handleClusterStopped(new ClusterStopped("default")); + + assertThat(interrupted.await(5, TimeUnit.SECONDS)).isTrue(); + } +}