ADD: Pluto as an Obol distributed validator client

controls/roles/manage-service/templates/prometheus.yml.j2

@@ -336,13 +336,13 @@ scrape_configs:
 {% endif %}
         ]
 
-# Charon
+# Charon / Pluto - same metric names on the same port, so one job
   - job_name: charon
     static_configs:
       - targets: [
 {% set targets = [] %}
 {%- for service_config in service_configs.results %}
-  {%- if (service_config.content | b64decode | from_yaml).service == "CharonService" %}
+  {%- if (service_config.content | b64decode | from_yaml).service in ["CharonService", "PlutoService"] %}
     {%- set _ = targets.append("stereum-" ~ (service_config.content | b64decode | from_yaml).id ~ ":3620") %}
   {%- endif %}
 {% endfor %}

launcher/src/backend/Monitoring.js

@@ -1,6 +1,7 @@
 import { ServiceManager } from "./ServiceManager";
 import { ValidatorAccountManager } from "./ValidatorAccountManager";
 import { StringUtils } from "./StringUtils.js";
+import { isObolDVTService, OBOL_DVT_SERVICES } from "@/share/ObolDVTServices";
 import { SSHService } from "./SSHService.js";
 import * as log from "electron-log";
 import * as crypto from "crypto";
@@ -3268,9 +3269,9 @@ export class Monitoring {
     }
 
     try {
-      //Check if connected to CharonService
+      //Check if connected to an Obol DV middleware service
       const vc = await this.nodeConnection.readServiceConfiguration(serviceID);
-      const connectedCharon = vc?.dependencies.consensusClients.find((client) => client.service === "CharonService");
+      const connectedCharon = vc?.dependencies.consensusClients.find((client) => isObolDVTService(client.service));
 
       const beaconAPIPort = connectedCharon ? 3600 : beaconStatus.data[0].beacon.servicePort;
       const serviceId = connectedCharon ? connectedCharon.id : beaconStatus.data[0].sid;
@@ -3396,7 +3397,7 @@ export class Monitoring {
    */
   async fetchObolCharonAlerts() {
     try {
-      const serviceInfos = await this.getServiceInfos("CharonService");
+      const serviceInfos = await this.getServiceInfos(...OBOL_DVT_SERVICES);
       if (serviceInfos.length < 1) {
         return [];
       }
@@ -3422,11 +3423,18 @@ export class Monitoring {
           if (metric.result.status != "success") {
             return;
           }
+          // a labelled counter has no series until first incremented, so a
+          // healthy cluster legitimately returns nothing for some of these
+          const series = metric.result.data?.result ?? [];
+          if (!series.length) {
+            log.debug(`No data for Obol alert metric "${metric.key}", skipping it`);
+            return;
+          }
           if (metric.key === "peer_ping_latency") {
-            let value = Math.max(...metric.result.data.result.map((r) => r.value[1]));
+            let value = Math.max(...series.map((r) => r.value[1]));
             return this.parseObolCharonAlerts(metric.key, value);
           }
-          let value = metric.result.data.result[0].value[1];
+          let value = series[0].value[1];
           return this.parseObolCharonAlerts(metric.key, value);
         })
         .filter((alert) => alert);
@@ -3827,15 +3835,20 @@ export class Monitoring {
   }
 
   async getObolClusterInformation(serviceID) {
-    const serviceInfos = await this.getServiceInfos("CharonService");
+    const serviceInfos = await this.getServiceInfos(...OBOL_DVT_SERVICES);
     const charon = serviceInfos.find((service) => service.config?.serviceID === serviceID);
     if (!charon) {
       log.info("No such Charon found!");
       return {};
     }
 
+    // charon deleted this check in v1.10.0 (ObolNetwork/charon#4359) as a false
+    // positive; pluto still ports it and reports 6 while doing its duties fine
+    const readyzCodes = charon.service === "PlutoService" ? ["1", "6"] : ["1"];
+
     const queries = {
       app_monitoring_readyz: `app_monitoring_readyz{instance=~".*${serviceID}.*"}`, // for Cluster Peer
+      app_peer_name: `app_peer_name{instance=~".*${serviceID}.*"}`,
       cluster_attestation_performance: `(sum(increase(core_tracker_success_duties_total{instance=~".*${serviceID}.*",duty="attester"}[1h])) / sum(increase(core_tracker_expect_duties_total{instance=~".*${serviceID}.*",duty="attester"}[1h])) > 0) * 100`,
       cluster_attestation_participation: `core_tracker_participation{instance=~".*${serviceID}.*",duty="attester"}`,
       cluster_operators: `cluster_operators{instance=~".*${serviceID}.*"}`,
@@ -3855,25 +3868,34 @@ export class Monitoring {
       if (metric.result.status != "success") {
         return;
       }
+      const series = metric.result.data?.result ?? [];
+      if (!series.length) {
+        log.debug(`No data for Obol cluster metric "${metric.key}", skipping it`);
+        return;
+      }
       switch (metric.key) {
         case "app_monitoring_readyz":
-          stats.nodeStatus = metric.result.data.result[0].value[1] === "1" ? "ACTIVE" : "INACTIVE";
-          stats.peerName = metric.result.data.result[0].metric.cluster_peer;
+          stats.nodeStatus = readyzCodes.includes(series[0].value[1]) ? "ACTIVE" : "INACTIVE";
+          stats.peerName = series[0].metric.cluster_peer;
+          break;
+        case "app_peer_name":
+          // charon carries the name on readyz, pluto only here
+          if (!stats.peerName) stats.peerName = series[0].metric.peer_name;
           break;
         case "cluster_attestation_performance":
-          stats.attestationPerformance = metric.result.data.result[0].value[1];
+          stats.attestationPerformance = series[0].value[1];
           break;
         case "cluster_attestation_participation":
-          stats.attestationParticipation = metric.result.data.result.reduce((acc, curr) => acc + parseInt(curr.value[1]), 0);
+          stats.attestationParticipation = series.reduce((acc, curr) => acc + parseInt(curr.value[1]), 0);
           break;
         case "cluster_operators":
-          stats.operators = parseInt(metric.result.data.result[0].value[1]);
+          stats.operators = parseInt(series[0].value[1]);
           break;
         case "cluster_threshold":
-          stats.threshold = parseInt(metric.result.data.result[0].value[1]);
+          stats.threshold = parseInt(series[0].value[1]);
           break;
         case "cluster_validators":
-          stats.validators = parseInt(metric.result.data.result[0].value[1]);
+          stats.validators = parseInt(series[0].value[1]);
           break;
       }
     });

launcher/src/backend/OneClickInstall.js

@@ -2,6 +2,7 @@ import { ServiceManager } from "./ServiceManager";
 import YAML from "yaml";
 import { StringUtils } from "./StringUtils";
 import { ConfigManager } from "./ConfigManager";
+import { isObolDVTService } from "@/share/ObolDVTServices";
 
 const log = require("electron-log");
 
@@ -227,9 +228,9 @@ export class OneClickInstall {
     }
 
     let charon = undefined;
-    if (constellation.includes("CharonService")) {
-      //CharonService
-      charon = this.serviceManager.getService("CharonService", {
+    const obolDVTService = constellation.find((service) => isObolDVTService(service));
+    if (obolDVTService) {
+      charon = this.serviceManager.getService(obolDVTService, {
         ...args,
         consensusClients: this.beaconService.filter((service) => service.service !== "OpNodeBeaconService"),
       });
@@ -351,7 +352,7 @@ export class OneClickInstall {
           ...args,
           consensusClients: this.beaconService
             .filter((service) => service.service !== "OpNodeBeaconService")
-            .concat(this.extraServices.filter((s) => s.service === "CharonService")),
+            .concat(this.extraServices.filter((s) => isObolDVTService(s.service))),
           otherServices: this.extraServices.filter((s) => s.service === "ValidatorEjectorService"),
         })
       );

launcher/src/backend/SSHService.js

@@ -243,9 +243,10 @@ export class SSHService {
 
   async execCommand(command) {
     if (this.loggingOut) return { rc: -1, stdout: "", stderr: "Logging Out!" };
+    const conn = this.getConnectionFromPool();
+    // an empty pool would otherwise surface as a TypeError on conn.exec
+    if (!conn) return { rc: -1, stdout: "", stderr: "Not connected!" };
     return new Promise((resolve, reject) => {
-      let conn = this.getConnectionFromPool();
-
       const data = {
         rc: -1,
         stdout: "",