diff --git a/node/rustchain_v2_integrated_v2.2.1_rip200.py b/node/rustchain_v2_integrated_v2.2.1_rip200.py index 7ea61bef0..1161f162e 100644 --- a/node/rustchain_v2_integrated_v2.2.1_rip200.py +++ b/node/rustchain_v2_integrated_v2.2.1_rip200.py @@ -5134,6 +5134,35 @@ def current_slot(): """Get current slot number""" return (int(time.time()) - GENESIS_TIMESTAMP) // BLOCK_TIME +def _record_unsettled_epoch(cursor, conn, epoch, reason): + """Leave a trace when an epoch is processed but pays nobody. + + Both no-payout paths below used to `return` silently, writing nothing. + That is how epochs 91-174 came to have no `epoch_state` row *at all* - + not `settled = 0`, absent - and why an 84-day settlement outage + (2026-03-03 to 2026-05-27) was invisible to every dashboard while 183 + miners kept enrolling for nothing. An epoch that pays nobody is a fact + worth recording, not a no-op. + + Writing `settled = 0` is safe: the authoritative replay guard inserts the + same row and then atomically claims `0 -> 1`, so a pre-existing unsettled + row does not block a later real settlement, and nothing was credited here + to be credited twice. + """ + try: + cursor.execute( + "INSERT INTO epoch_state (epoch, settled) VALUES (?, 0) " + "ON CONFLICT(epoch) DO NOTHING", + (epoch,) + ) + conn.commit() + except Exception as exc: + # Never let bookkeeping abort settlement, but do not swallow it either. + print(f"[SETTLE] epoch {epoch}: could not record unsettled state: {exc}") + print(f"[SETTLE] Epoch {epoch} paid nobody (reason={reason}). " + f"Recorded as unsettled; settlement_lag_epochs will report this.") + + def finalize_epoch(epoch, per_block_rtc, prev_block_hash: bytes = b""): """Finalize epoch and distribute rewards with security hardening""" from contextlib import closing @@ -5159,6 +5188,7 @@ def finalize_epoch(epoch, per_block_rtc, prev_block_hash: bytes = b""): miners = [(pk, normalize_epoch_weight_units(weight)) for pk, weight in raw_miners] if not miners: + _record_unsettled_epoch(c, conn, epoch, "no_enrolled_miners") return # Calculate total weight @@ -5167,6 +5197,7 @@ def finalize_epoch(epoch, per_block_rtc, prev_block_hash: bytes = b""): # DIVISION BY ZERO PROTECTION if total_weight == 0: print(f"[SECURITY] Total weight is 0 for epoch {epoch}, skipping reward distribution") + _record_unsettled_epoch(c, conn, epoch, "total_weight_zero") return # PRECISION: Use Decimal for exact financial calculations @@ -6406,6 +6437,12 @@ def get_epoch(): "SELECT COUNT(*) FROM epoch_enroll WHERE epoch = ?", (epoch,) ).fetchone()[0] + # Reuse this connection - `enrolled_miners` alone cannot distinguish a + # healthy chain from one that has been enrolling miners and paying + # none of them for 84 days, which is what happened in 2026. + settlement_lag, last_settled = _settlement_lag_epochs( + conn=c, now_epoch=epoch + ) return jsonify({ "epoch": epoch, @@ -6413,7 +6450,9 @@ def get_epoch(): "epoch_pot": PER_EPOCH_RTC, "enrolled_miners": enrolled, "blocks_per_epoch": EPOCH_SLOTS, - "total_supply_rtc": TOTAL_SUPPLY_RTC + "total_supply_rtc": TOTAL_SUPPLY_RTC, + "settlement_lag_epochs": settlement_lag, + "last_settled_epoch": last_settled }) @app.route('/epoch/proposer-duty-calendar', methods=['GET']) @@ -10610,6 +10649,47 @@ def _tip_age_slots(): except Exception: return None +def _settlement_lag_epochs(conn=None, now_epoch=None): + """Epochs elapsed since the newest *settled* epoch. + + `conn` / `now_epoch` let a caller that already has a connection and the + current epoch reuse them (`/epoch` is a hot path), without duplicating the + bounding rule below - which is the part that is easy to get wrong. + + Block production and epoch settlement are independent subsystems, and + nothing was watching the second one. Epochs 91-174 (2026-03-03 to + 2026-05-27) were never settled: `settle_epoch` returned early on an empty + miner set without writing an `epoch_state` row or logging, so 84 + consecutive epochs left no trace at all. Blocks kept being produced + throughout, so `tip_age_slots` stayed at 0 and `/health` reported + `ok: true` for the entire 84 days. 183 miners enrolled and were paid + nothing. This is the signal that would have caught it on day one. + + Returns `(lag_epochs, last_settled_epoch)`; either may be None when the + state is genuinely unknown (nothing settled on record, or the query + failed). Never raises - health reporting must not take the node down. + """ + # Bound by the current epoch on purpose. `epoch_state` still carries rows + # from the pre-2025-12 numbering scheme (values in the 20000s, plus a stray + # 424 settled in Dec 2025). An unbounded MAX() picks one of those, yields + # a *negative* lag, and reports perfect health during exactly the stall + # this exists to detect. + _SQL = "SELECT MAX(epoch) FROM epoch_state WHERE settled = 1 AND epoch <= ?" + try: + if now_epoch is None: + now_epoch = slot_to_epoch(current_slot()) + if conn is not None: + row = conn.execute(_SQL, (now_epoch,)).fetchone() + else: + with sqlite3.connect(DB_PATH, timeout=3) as db: + row = db.execute(_SQL, (now_epoch,)).fetchone() + last_settled = row[0] if row else None + if last_settled is None: + return None, None + return max(0, now_epoch - int(last_settled)), int(last_settled) + except Exception: + return None, None + # ============= READINESS AGGREGATOR (RIP-0143) ============= # Global metrics snapshot for lightweight readiness checks @@ -10684,6 +10764,12 @@ def api_health(): ok_db = _db_rw_ok() age_h = _backup_age_hours() tip_age = _tip_age_slots() + settlement_lag, last_settled = _settlement_lag_epochs() + # Settlement lag is reported but deliberately does NOT flip `ok`. A 503 + # pulls the node out of load-balancer rotation, which would stop miners + # enrolling - the opposite of what a settlement stall needs. Enrollment + # data is what makes a stalled epoch recoverable later. Expose the number + # and let alerting act on it. ok = ok_db and (age_h is None or age_h < 36) return jsonify({ "ok": bool(ok), @@ -10691,7 +10777,9 @@ def api_health(): "uptime_s": int(time.time() - APP_START_TS), "db_rw": bool(ok_db), "backup_age_hours": age_h, - "tip_age_slots": tip_age + "tip_age_slots": tip_age, + "settlement_lag_epochs": settlement_lag, + "last_settled_epoch": last_settled }), (200 if ok else 503) @app.route('/ready', methods=['GET']) diff --git a/tests/test_settlement_lag_visibility.py b/tests/test_settlement_lag_visibility.py new file mode 100644 index 000000000..ff540e182 --- /dev/null +++ b/tests/test_settlement_lag_visibility.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""An advancing epoch number is not evidence that anyone is being paid. + +RustChain never settled epochs 91-174 — 2026-03-03 to 2026-05-27, 84 +consecutive days. `epoch_state` has no row for any of them: not `settled = 0`, +absent. `finalize_epoch` returned early on an empty miner set without writing +a row or logging, so the outage left no trace anywhere. + +Nothing noticed, because nothing was looking at settlement: + + * the epoch number is derived from wall-clock, so it kept counting up; + * blocks kept being produced, so `tip_age_slots` stayed 0; + * `/health` therefore reported `ok: true` for the entire 84 days; + * the node health monitor read only `epoch` and `miners`, both healthy. + +183 miners enrolled across those epochs and were paid nothing. 83% of them +never attested again. 126 RTC was never emitted. + +These tests pin the two halves of the fix: the node must expose the lag, and +the monitor must alert on it. The nastiest case has its own test — the stale +epoch numbering that makes an unbounded query report a *negative* lag, i.e. +perfect health, during exactly the stall being looked for. +""" + +import io +import json +import os +import sqlite3 +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(ROOT, "tools")) + +import importlib.util + +_spec = importlib.util.spec_from_file_location( + "node_health_monitor", os.path.join(ROOT, "tools", "node_health_monitor.py") +) +MON = importlib.util.module_from_spec(_spec) +sys.modules["node_health_monitor"] = MON +_spec.loader.exec_module(MON) + + +# ── monitor side ────────────────────────────────────────────────────────────── + +class _Resp(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _probe(payload: dict): + mon = MON.NodeHealthMonitor(nodes=["http://example.invalid"]) + with mock.patch.object(MON.urllib.request, "urlopen", + return_value=_Resp(json.dumps(payload).encode())): + return mon.check_node("http://example.invalid") + + +class MonitorSeesTheStallTest(unittest.TestCase): + + def test_healthy_lag_raises_nothing(self): + st = _probe({"epoch": 254, "enrolled_miners": 16, + "settlement_lag_epochs": 1}) + self.assertEqual(st.status, "online") + self.assertEqual(st.settlement_lag_epochs, 1) + self.assertIsNone(st.error) + + def test_the_outage_shape_is_reported(self): + """Epoch advancing, miners enrolling, nothing settled for 84 epochs.""" + st = _probe({"epoch": 174, "enrolled_miners": 21, + "settlement_lag_epochs": 84}) + self.assertEqual(st.settlement_lag_epochs, 84) + self.assertIsNotNone(st.error, "an 84-epoch stall must not be silent") + self.assertIn("settlement stalled", st.error) + + def test_a_stalled_node_is_still_online(self): + """It is answering, and miners must keep enrolling so the stalled + epochs stay reconstructible from epoch_enroll.""" + st = _probe({"epoch": 174, "miners": 21, "settlement_lag_epochs": 84}) + self.assertEqual(st.status, "online") + + def test_an_older_node_that_omits_the_field_is_not_accused(self): + st = _probe({"epoch": 254, "miners": 16}) + self.assertIsNone(st.settlement_lag_epochs) + self.assertIsNone(st.error) + + def test_a_malformed_lag_does_not_kill_the_probe(self): + st = _probe({"epoch": 254, "miners": 16, + "settlement_lag_epochs": "ages"}) + self.assertEqual(st.status, "online") + self.assertIsNone(st.settlement_lag_epochs) + + +class NetworkAlertTest(unittest.TestCase): + + def _status(self, lag): + return MON.NodeStatus(url="http://n", status="online", + response_time_ms=10.0, epoch=254, miners=16, + error=None, settlement_lag_epochs=lag) + + def test_network_health_flags_the_stall(self): + mon = MON.NodeHealthMonitor(nodes=["http://n"]) + health = mon.get_network_health([self._status(84)]) + self.assertTrue(health.settlement_stalled) + self.assertTrue(any("SETTLEMENT STALLED" in a for a in health.alerts)) + + def test_consensus_can_be_perfect_while_settlement_is_dead(self): + """The exact 2026 blind spot: every node agreeing on an epoch number + was read as health. Agreement says nothing about payment.""" + mon = MON.NodeHealthMonitor(nodes=["http://a", "http://b"]) + health = mon.get_network_health([self._status(84), self._status(84)]) + self.assertTrue(health.consensus_ok) + self.assertFalse(health.split_brain) + self.assertTrue(health.settlement_stalled, + "consensus_ok must not mask a settlement stall") + + def test_healthy_network_is_not_flagged(self): + mon = MON.NodeHealthMonitor(nodes=["http://n"]) + health = mon.get_network_health([self._status(1)]) + self.assertFalse(health.settlement_stalled) + + +# ── node side: the bounding rule ────────────────────────────────────────────── + +class LegacyEpochNumberingTest(unittest.TestCase): + """`epoch_state` still holds rows from the pre-2025-12 numbering scheme. + + An unbounded `MAX(epoch) WHERE settled = 1` returns 424 (settled Dec 2025) + or a 20000-series row, so the lag computes negative and the node reports + flawless health during the stall. The query must be bounded by the current + epoch. This is the trap the fix exists to avoid, so it is pinned directly + against SQLite rather than mocked. + """ + + SQL_BOUNDED = ("SELECT MAX(epoch) FROM epoch_state " + "WHERE settled = 1 AND epoch <= ?") + SQL_UNBOUNDED = "SELECT MAX(epoch) FROM epoch_state WHERE settled = 1" + + def setUp(self): + self.db = sqlite3.connect(":memory:") + self.db.execute("CREATE TABLE epoch_state (epoch INTEGER PRIMARY KEY, " + "settled INTEGER DEFAULT 0, settled_ts INTEGER)") + # production shape: legacy rows, then the modern series stalled at 90 + self.db.executemany( + "INSERT INTO epoch_state (epoch, settled) VALUES (?, 1)", + [(90,), (424,), (20424,)] + ) + + def tearDown(self): + self.db.close() + + def test_unbounded_query_hides_the_outage(self): + """Documents why the naive version is wrong — it must NOT be used.""" + last = self.db.execute(self.SQL_UNBOUNDED).fetchone()[0] + self.assertEqual(last, 20424) + self.assertLess(174 - last, 0, + "the naive query yields a negative lag = 'all is well'") + + def test_bounded_query_finds_the_real_stall(self): + last = self.db.execute(self.SQL_BOUNDED, (174,)).fetchone()[0] + self.assertEqual(last, 90) + self.assertEqual(174 - last, 84, "the actual outage length") + + def test_no_settled_epoch_at_all_is_unknown_not_zero(self): + empty = sqlite3.connect(":memory:") + empty.execute("CREATE TABLE epoch_state (epoch INTEGER PRIMARY KEY, " + "settled INTEGER DEFAULT 0)") + last = empty.execute(self.SQL_BOUNDED, (254,)).fetchone()[0] + self.assertIsNone(last, "absent state must read as unknown, not lag 0") + empty.close() + + +class NodeHelperTest(unittest.TestCase): + """Exercise the node's real `_settlement_lag_epochs`, not a copy of its SQL. + + The class above documents the bounding principle against raw SQLite; this + one guards the actual shipped function, which is what regresses. + """ + + @classmethod + def setUpClass(cls): + os.environ.setdefault("RC_ADMIN_KEY", "t" * 64) + node_dir = os.path.join(ROOT, "node") + sys.path.insert(0, node_dir) + spec = importlib.util.spec_from_file_location( + "rc_node_settle", + os.path.join(node_dir, "rustchain_v2_integrated_v2.2.1_rip200.py")) + cls.NODE = importlib.util.module_from_spec(spec) + sys.modules["rc_node_settle"] = cls.NODE + spec.loader.exec_module(cls.NODE) + + def _db(self, settled_epochs): + db = sqlite3.connect(":memory:") + db.execute("CREATE TABLE epoch_state (epoch INTEGER PRIMARY KEY, " + "settled INTEGER DEFAULT 0, settled_ts INTEGER)") + db.executemany("INSERT INTO epoch_state (epoch, settled) VALUES (?, 1)", + [(e,) for e in settled_epochs]) + return db + + def test_legacy_rows_do_not_mask_the_stall(self): + """90 settled, legacy 424 and 20424 present, current epoch 174.""" + db = self._db([90, 424, 20424]) + lag, last = self.NODE._settlement_lag_epochs(conn=db, now_epoch=174) + db.close() + self.assertEqual(last, 90, "must ignore the pre-2025-12 numbering") + self.assertEqual(lag, 84, "the real outage length") + + def test_steady_state_is_lag_one(self): + db = self._db([253]) + lag, last = self.NODE._settlement_lag_epochs(conn=db, now_epoch=254) + db.close() + self.assertEqual((lag, last), (1, 253)) + + def test_unknown_when_nothing_is_settled(self): + db = self._db([]) + lag, last = self.NODE._settlement_lag_epochs(conn=db, now_epoch=254) + db.close() + self.assertEqual((lag, last), (None, None)) + + def test_unsettled_rows_do_not_count_as_settled(self): + db = self._db([90]) + db.execute("INSERT INTO epoch_state (epoch, settled) VALUES (150, 0)") + lag, last = self.NODE._settlement_lag_epochs(conn=db, now_epoch=174) + db.close() + self.assertEqual(last, 90, + "a recorded-but-unsettled epoch is not a settlement") + + def test_a_broken_table_reports_unknown_and_does_not_raise(self): + db = sqlite3.connect(":memory:") + lag, last = self.NODE._settlement_lag_epochs(conn=db, now_epoch=254) + db.close() + self.assertEqual((lag, last), (None, None)) + + +class UnsettledEpochLeavesATraceTest(unittest.TestCase): + """A no-payout epoch must be recorded, not silently skipped. + + Writing `settled = 0` is safe: the authoritative replay guard inserts the + same row then atomically claims `0 -> 1`, so an existing unsettled row does + not block a later real settlement. + """ + + def setUp(self): + self.db = sqlite3.connect(":memory:") + self.db.execute("CREATE TABLE epoch_state (epoch INTEGER PRIMARY KEY, " + "settled INTEGER DEFAULT 0, settled_ts INTEGER)") + + def tearDown(self): + self.db.close() + + def _record(self, epoch): + self.db.execute("INSERT INTO epoch_state (epoch, settled) VALUES (?, 0) " + "ON CONFLICT(epoch) DO NOTHING", (epoch,)) + + def test_the_epoch_becomes_visible_instead_of_absent(self): + self._record(91) + row = self.db.execute( + "SELECT settled FROM epoch_state WHERE epoch = 91").fetchone() + self.assertIsNotNone(row, "epochs 91-174 had no row at all") + self.assertEqual(row[0], 0) + + def test_a_later_real_settlement_can_still_claim_it(self): + self._record(91) + claim = self.db.execute( + "UPDATE epoch_state SET settled = 1, settled_ts = 1 " + "WHERE epoch = ? AND settled = 0", (91,)) + self.assertEqual(claim.rowcount, 1, + "recording must not block recovery of the epoch") + + def test_it_cannot_overwrite_an_already_settled_epoch(self): + self.db.execute("INSERT INTO epoch_state (epoch, settled, settled_ts) " + "VALUES (91, 1, 12345)") + self._record(91) + settled, ts = self.db.execute( + "SELECT settled, settled_ts FROM epoch_state WHERE epoch = 91" + ).fetchone() + self.assertEqual((settled, ts), (1, 12345), + "must never demote a settled epoch") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/node_health_monitor.py b/tools/node_health_monitor.py index d7df74a3f..458951c5a 100644 --- a/tools/node_health_monitor.py +++ b/tools/node_health_monitor.py @@ -40,6 +40,12 @@ STATUS_ENDPOINTS = ("/epoch", "/status", "/health") REQUEST_TIMEOUT = 5 # seconds per HTTP request +# Steady state is a lag of 1: the current epoch is in progress and the previous +# one is settled. 3 tolerates one late settlement plus clock skew without +# crying wolf, while still catching a real stall on its second day rather than +# its eighty-fourth. +SETTLEMENT_LAG_THRESHOLD = 3 + # ── Known attestation nodes ─────────────────────────────────────────────────── DEFAULT_NODES = [ "http://50.28.86.131:8088", @@ -56,6 +62,11 @@ class NodeStatus: epoch: Optional[int] miners: Optional[int] error: Optional[str] + # Epochs since this node last settled one. The epoch number advances off + # wall-clock whether or not anybody is being paid, so `epoch` alone stayed + # perfectly healthy through the 84-epoch settlement outage of 2026-03-03 to + # 2026-05-27. None means the node does not report it (older build). + settlement_lag_epochs: Optional[int] = None def to_dict(self) -> Dict[str, Any]: return asdict(self) @@ -69,6 +80,11 @@ class NetworkHealth: consensus_ok: bool split_brain: bool alerts: List[str] + # True when any online node has not settled an epoch in a while. Kept + # separate from consensus_ok/split_brain because nodes can agree perfectly + # on the epoch number and still be paying nobody — which is exactly what + # happened for 84 consecutive epochs in 2026. + settlement_stalled: bool = False def to_dict(self) -> Dict[str, Any]: return asdict(self) @@ -158,14 +174,29 @@ def check_node(self, url: str) -> NodeStatus: error="/status carried no epoch (not a RustChain node?)", ) + lag = data.get("settlement_lag_epochs") + try: + lag = int(lag) if lag is not None else None + except (TypeError, ValueError): + lag = None + status = "slow" if elapsed_ms > SLOW_THRESHOLD_MS else "online" + # A settlement stall does not make the node offline — it is + # answering, and miners must keep enrolling so the stalled + # epochs stay reconstructible. Report it, do not hide it in + # the status word. + err = None + if lag is not None and lag >= SETTLEMENT_LAG_THRESHOLD: + err = (f"settlement stalled: {lag} epochs since last settled " + f"(miners are enrolling and being paid nothing)") return NodeStatus( url=url, status=status, response_time_ms=round(elapsed_ms, 1), epoch=epoch, miners=miners, - error=None, + error=err, + settlement_lag_epochs=lag, ) except urllib.error.HTTPError as exc: @@ -243,6 +274,17 @@ def get_network_health(self, statuses: Optional[List[NodeStatus]] = None) -> Net if nodes_online == 0: alerts.append("ALL NODES OFFLINE — network unreachable") + stalled = [s for s in online + if s.settlement_lag_epochs is not None + and s.settlement_lag_epochs >= SETTLEMENT_LAG_THRESHOLD] + if stalled: + detail = ", ".join(f"{s.url} ({s.settlement_lag_epochs} epochs)" + for s in stalled) + alerts.append( + f"SETTLEMENT STALLED — no epoch settled recently: {detail}. " + f"Miners are still enrolling and earning nothing." + ) + return NetworkHealth( nodes_online=nodes_online, total_nodes=len(statuses), @@ -250,6 +292,7 @@ def get_network_health(self, statuses: Optional[List[NodeStatus]] = None) -> Net consensus_ok=consensus_ok, split_brain=split_brain, alerts=alerts, + settlement_stalled=bool(stalled), ) def detect_split_brain(self, statuses: Optional[List[NodeStatus]] = None) -> bool: