Keep a devnet across long pauses: catch up to wall clock with Yano (companion, yano-only), with step progress in the CLI

applications/cli/config/application.properties

@@ -48,6 +48,12 @@ bp.create.enabled=true
 #yano.relay.sync.stall.timeout.seconds=30
 #yano.relay.sync.max.wait.seconds=0
 
+## Catch-up after a long pause (companion and yano-only). A devnet paused for longer than its stability window
+## (laptop lid closed, stopped for days) cannot produce blocks again on its own. With auto catch-up, DevKit
+## backfills the gap with Yano (sparse empty blocks, one per epoch boundary) and keeps the same chain:
+## on 'start', and every 30s while the devnet runs. false: only report it; run 'catch-up' by hand.
+#devnet.auto.catch.up=true
+
 
 ######################################################
 #To configure an external database for Yaci Store (Indexer),

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/commands/common/DownloadService.java

@@ -1,6 +1,8 @@
 package com.bloxbean.cardano.yacicli.commands.common;
 
 import com.bloxbean.cardano.yacicli.localcluster.ClusterConfig;
+import com.bloxbean.cardano.yacicli.util.progress.ConsoleProgress;
+import com.bloxbean.cardano.yacicli.util.progress.Step;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
@@ -490,29 +492,30 @@ private Path download(String component, String downloadUrl, String targetDir, St
             Path targetPath = Paths.get(targetDir, targetFileName);
             Files.createDirectories(targetPath.getParent());
 
+            Step step = ConsoleProgress.step("Downloading " + component, console());
             try (InputStream inputStream = httpConn.getInputStream();
                  FileOutputStream outputStream = new FileOutputStream(targetPath.toFile())) {
 
-                byte[] buffer = new byte[4096];
+                byte[] buffer = new byte[65536];
                 int bytesRead;
                 long totalBytesRead = 0;
-                int percentCompleted;
 
                 while ((bytesRead = inputStream.read(buffer)) != -1) {
                     if (Thread.currentThread().isInterrupted()) {
-                        writeLn(warnLabel("Download", "Download interrupted by user."));
+                        step.fail("interrupted by user");
                         Files.deleteIfExists(targetPath);
                         return null;
                     }
 
                     outputStream.write(buffer, 0, bytesRead);
                     totalBytesRead += bytesRead;
-                    percentCompleted = (int) ((totalBytesRead * 100) / fileSize);
-                    write("\rDownloading: " + percentCompleted + "%");
+                    step.progress(totalBytesRead, fileSize, fileSize > 0
+                            ? megabytes(totalBytesRead) + " / " + megabytes(fileSize)
+                            : megabytes(totalBytesRead));
                 }
-                System.out.println();
-                writeLn(success("Download complete for %s!", component));
+                step.done(megabytes(totalBytesRead));
             } catch (Exception e) {
+                step.fail(e.getMessage());
                 e.printStackTrace();
             }
 
@@ -524,6 +527,10 @@ private Path download(String component, String downloadUrl, String targetDir, St
         return null;
     }
 
+    private static String megabytes(long bytes) {
+        return String.format("%.1f MB", bytes / (1024.0 * 1024.0));
+    }
+
     public void extractTarGz(String filePath, String extractDir) throws IOException {
         try (FileInputStream fis = new FileInputStream(filePath);
              BufferedInputStream bis = new BufferedInputStream(fis);

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterCommands.java

@@ -278,6 +278,23 @@ public void startLocalCluster() {
 
     }
 
+    @ShellMethod(value = "Catch the devnet up to wall clock after the machine slept or the devnet was paused longer than "
+            + "the stability window. Runs automatically on start and while the devnet runs (devnet.auto.catch.up).", key = "catch-up")
+    @ShellMethodAvailability("localClusterCmdAvailability")
+    public void catchUp(@ShellOption(value = {"--force"}, defaultValue = "false",
+            help = "Catch up even when the chain is not stalled") boolean force) {
+        String clusterName = CommandContext.INSTANCE.getProperty(ClusterConfig.CLUSTER_NAME);
+        try {
+            var result = localClusterService.catchUp(clusterName, force, console());
+            if (!result.success())
+                writeLn(error(result.message()));
+            else if (result.blocksProduced() == 0)
+                writeLn(info(result.message()));
+        } catch (IOException e) {
+            writeLn(error("Catch-up failed: " + e.getMessage()));
+        }
+    }
+
     @ShellMethod(value = "Stop the running local devnet", key = "stop")
     @ShellMethodAvailability("localClusterCmdAvailability")
     public void stopLocalCluster() {

applications/cli/src/main/java/com/bloxbean/cardano/yacicli/localcluster/ClusterService.java

@@ -3,6 +3,8 @@
 import com.bloxbean.cardano.yaci.core.protocol.localstate.api.Era;
 import com.bloxbean.cardano.yaci.core.util.OSUtil;
 import com.bloxbean.cardano.yacicli.common.CommandContext;
+import com.bloxbean.cardano.yacicli.localcluster.catchup.ChainLag;
+import com.bloxbean.cardano.yacicli.localcluster.catchup.DevnetCatchUpService;
 import com.bloxbean.cardano.yacicli.localcluster.config.CustomGenesisConfig;
 import com.bloxbean.cardano.yacicli.localcluster.model.RunStatus;
 import com.bloxbean.cardano.yacicli.localcluster.config.ApplicationConfig;
@@ -93,6 +95,9 @@ public class ClusterService {
     @Autowired
     private LocalPeerService localPeerService;
 
+    @Autowired
+    private DevnetCatchUpService devnetCatchUpService;
+
     public ClusterService(ClusterConfig config,
                           ClusterStartService clusterStartService,
                           BlockStreamerService blockStreamerService,
@@ -119,7 +124,7 @@ public void startClusterContext(String clusterName, Consumer<String> writer) {
 
     public RunStatus startCluster(String clusterName) {
         try {
-            RunStatus startedSuccessfully = clusterStartService.startCluster(getClusterInfo(clusterName), getClusterFolder(clusterName), msg -> writeLn(msg));
+            RunStatus startedSuccessfully = clusterStartService.startCluster(getClusterInfo(clusterName), getClusterFolder(clusterName), console());
             if (startedSuccessfully.stared())
                 writeLn(info("Swagger Url to interact with the cluster's node : " + "http://localhost:" + server.getWebServer().getPort() +"/swagger-ui.html"));
 
@@ -136,6 +141,30 @@ public void stopCluster(Consumer<String> writer) {
         publisher.publishEvent(new ClusterStopped(clusterName));
     }
 
+    /**
+     * How far the running devnet's tip is behind wall clock.
+     *
+     * @return the lag, or null when the devnet is not running or its tip cannot be read
+     */
+    public ChainLag chainLag(String clusterName) throws IOException {
+        if (!clusterStartService.isClusterRunning())
+            return null;
+        ClusterInfo clusterInfo = getClusterInfo(clusterName);
+        return clusterInfo != null ? devnetCatchUpService.probeLag(clusterInfo) : null;
+    }
+
+    /**
+     * Catch the running devnet up to wall clock (see ADR-0017).
+     *
+     * @param force catch up even when the chain is not stalled
+     */
+    public DevnetCatchUpService.Result catchUp(String clusterName, boolean force, Consumer<String> writer) throws IOException {
+        ClusterInfo clusterInfo = getClusterInfo(clusterName);
+        if (clusterInfo == null)
+            return new DevnetCatchUpService.Result(false, "Devnet not found: " + clusterName, -1, -1, 0, 0);
+        return clusterStartService.catchUp(clusterInfo, getClusterFolder(clusterName), force, writer);
+    }
+
     public void stopClusterNode(Consumer<String> writer) {
         clusterStartService.stopCluster(writer);
     }