From 7bf90c3fe7febb65abe0b0fb5c41c4565450303c Mon Sep 17 00:00:00 2001 From: Ayush Bansal Date: Sun, 11 Oct 2026 11:27:39 +0530 Subject: [PATCH] perf(gateways): serve the gateway details page from precomputed overviews The gateway page made two requests per load and the 90D window took up to 55 s. A background task now precomputes each federation's gateway list and uptime trend per window and serves both from /federations/:id/gateways/overview, so the page loads with one request in about a millisecond. - 24h/7D refresh every 5 minutes, 30D every 30, 90D hourly - activity CTEs are NOT MATERIALIZED (90D: 55 s to 4 s, same results) - gateways missing from the registry for 7 days count as retired and no longer drag uptime down; the page lists them separately - gateway polls are stored again (the window start parameter was inferred as an interval) - remove the old /gateways and /gateways/uptime-trend endpoints and the 1h window - /config/:invite/gateways reuses the observer's connector registry instead of building two per request, which left their guardian connections open - the page keeps its view in the URL, refreshes every minute, looks up the live registry in the background and counts gateways online for 15 minutes after they were seen Co-Authored-By: Claude Opus 5.5 --- fmo_api_types/src/lib.rs | 15 + .../src/components/ErrorBoundary.tsx | 42 + .../src/components/GatewayWarningPage.tsx | 13 +- .../src/components/UptimeTrendChart.tsx | 13 + .../src/pages/FederationGateways.tsx | 809 +++++++++++------- fmo_frontend_react/src/services/api.ts | 26 +- fmo_frontend_react/src/types/api.ts | 11 +- fmo_frontend_react/src/utils/storage.ts | 22 + fmo_server/src/config/mod.rs | 11 +- fmo_server/src/federation/gateways.rs | 267 ++++-- fmo_server/src/federation/mod.rs | 7 +- fmo_server/src/federation/observer.rs | 11 + 12 files changed, 865 insertions(+), 382 deletions(-) create mode 100644 fmo_frontend_react/src/components/ErrorBoundary.tsx create mode 100644 fmo_frontend_react/src/components/UptimeTrendChart.tsx create mode 100644 fmo_frontend_react/src/utils/storage.ts diff --git a/fmo_api_types/src/lib.rs b/fmo_api_types/src/lib.rs index 416d080..328607d 100644 --- a/fmo_api_types/src/lib.rs +++ b/fmo_api_types/src/lib.rs @@ -138,3 +138,18 @@ pub struct GatewayUptimeTrendPoint { pub total_samples: u64, pub uptime_pct: f64, } + +/// A federation's gateways and their availability trend for one window, +/// computed at the same moment so both always agree +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GatewayOverview { + /// The window label, e.g. `7d` + pub window: String, + pub computed_at: DateTime, + /// Left out when `include` doesn't ask for `gateways` + #[serde(skip_serializing_if = "Option::is_none")] + pub gateways: Option>, + /// Left out when `include` doesn't ask for `trend` + #[serde(skip_serializing_if = "Option::is_none")] + pub uptime_trend: Option>, +} diff --git a/fmo_frontend_react/src/components/ErrorBoundary.tsx b/fmo_frontend_react/src/components/ErrorBoundary.tsx new file mode 100644 index 0000000..f4e938c --- /dev/null +++ b/fmo_frontend_react/src/components/ErrorBoundary.tsx @@ -0,0 +1,42 @@ +import { Component, type ErrorInfo, type ReactNode } from 'react'; + +interface ErrorBoundaryProps { + fallback: ReactNode; + children: ReactNode; +} + +interface ErrorBoundaryState { + failed: boolean; +} + +// Without a boundary, React unmounts the whole app on a render error, leaving a blank page +export class ErrorBoundary extends Component { + state: ErrorBoundaryState = { failed: false }; + + static getDerivedStateFromError(): ErrorBoundaryState { + return { failed: true }; + } + + componentDidCatch(error: Error, info: ErrorInfo) { + console.error('Render error:', error, info.componentStack); + } + + render() { + return this.state.failed ? this.props.fallback : this.props.children; + } +} + +export function ReloadMessage({ message }: { message: string }) { + return ( +
+

{message}

+ +
+ ); +} diff --git a/fmo_frontend_react/src/components/GatewayWarningPage.tsx b/fmo_frontend_react/src/components/GatewayWarningPage.tsx index 54115ea..1ee4bf7 100644 --- a/fmo_frontend_react/src/components/GatewayWarningPage.tsx +++ b/fmo_frontend_react/src/components/GatewayWarningPage.tsx @@ -10,6 +10,7 @@ export interface GatewayWarningState { interface GatewayWarningPageProps { warning: GatewayWarningState; className?: string; + action?: { label: string; onClick: () => void; busy?: boolean }; } function levelClasses(level: GatewayWarningLevel): string { @@ -23,7 +24,7 @@ function levelClasses(level: GatewayWarningLevel): string { } } -export function GatewayWarningPage({ warning, className = '' }: GatewayWarningPageProps) { +export function GatewayWarningPage({ warning, className = '', action }: GatewayWarningPageProps) { return (
)} + {action && ( + + )}
); } diff --git a/fmo_frontend_react/src/components/UptimeTrendChart.tsx b/fmo_frontend_react/src/components/UptimeTrendChart.tsx new file mode 100644 index 0000000..c6c6581 --- /dev/null +++ b/fmo_frontend_react/src/components/UptimeTrendChart.tsx @@ -0,0 +1,13 @@ +import ReactEChartsCore from 'echarts-for-react/lib/core'; +import * as echarts from 'echarts/core'; +import { LineChart } from 'echarts/charts'; +import { GridComponent, TooltipComponent } from 'echarts/components'; +import { CanvasRenderer } from 'echarts/renderers'; +import type { EChartsOption } from 'echarts'; + +// Register only what the chart uses instead of loading all of echarts +echarts.use([LineChart, GridComponent, TooltipComponent, CanvasRenderer]); + +export function UptimeTrendChart({ option }: { option: EChartsOption }) { + return ; +} diff --git a/fmo_frontend_react/src/pages/FederationGateways.tsx b/fmo_frontend_react/src/pages/FederationGateways.tsx index 3f380fe..3b9576e 100644 --- a/fmo_frontend_react/src/pages/FederationGateways.tsx +++ b/fmo_frontend_react/src/pages/FederationGateways.tsx @@ -1,27 +1,51 @@ -import { lazy, Suspense, useEffect, useMemo, useRef, useState } from 'react'; -import { Link, useParams } from 'react-router-dom'; +import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from 'react'; +import { Link, useParams, useSearchParams } from 'react-router-dom'; import { api } from '../services/api'; import type { FederationSummary, GatewayInfo, GatewayUptimeTrendPoint, - GatewayWindow, } from '../types/api'; import { GatewayWarningPage, type GatewayWarningState } from '../components/GatewayWarningPage'; +import { ErrorBoundary, ReloadMessage } from '../components/ErrorBoundary'; +import { readStorage, writeStorage } from '../utils/storage'; +import { useTheme } from '../hooks/useTheme'; +import type { EChartsOption } from 'echarts'; -type GatewayStatus = 'online' | 'degraded' | 'offline' | 'unknown'; +type GatewayStatus = 'online' | 'degraded' | 'offline' | 'unknown' | 'retired'; type UptimeStripStatus = 'online' | 'degraded' | 'offline' | 'unknown'; -type GatewayFilter = 'all' | GatewayStatus; -type GatewaySort = 'freshness' | 'status' | 'uptime' | 'activity'; -type SortDirection = 'asc' | 'desc'; -const UptimeTrendChart = lazy(() => import('echarts-for-react')); +const GATEWAY_WINDOWS = ['24h', '7d', '30d', '90d'] as const; +type SelectableWindow = (typeof GATEWAY_WINDOWS)[number]; +// 'all' means every active gateway; retired ones have their own filter +const GATEWAY_FILTERS = ['all', 'online', 'degraded', 'offline', 'unknown', 'retired'] as const; +const GATEWAY_SORTS = ['freshness', 'status', 'uptime', 'activity'] as const; +type GatewaySort = (typeof GATEWAY_SORTS)[number]; +const SORT_DIRECTIONS = ['asc', 'desc'] as const; +type SortDirection = (typeof SORT_DIRECTIONS)[number]; + +const UptimeTrendChart = lazy(() => import('../components/UptimeTrendChart').then((module) => ({ default: module.UptimeTrendChart }))); const INITIAL_RENDER_COUNT = 50; +const DEFAULT_WINDOW: SelectableWindow = '7d'; +const WINDOW_STORAGE_KEY = 'gatewayWindow'; +const WINDOW_MINUTES: Record = { + '24h': 24 * 60, + '7d': 7 * 24 * 60, + '30d': 30 * 24 * 60, + '90d': 90 * 24 * 60, +}; +// Gateways are polled and their overview rebuilt every 5 minutes on the server +const REFRESH_INTERVAL_MS = 60_000; +const REFRESH_CHECK_MS = 10_000; +const LIVE_LOOKUP_TIMEOUT_MS = 10_000; +const EMPTY_TREND: GatewayUptimeTrendPoint[] = []; +const RETIRED_AFTER_MINUTES = 7 * 24 * 60; const STATUS_RANK: Record = { offline: 0, degraded: 1, unknown: 2, online: 3, + retired: 4, }; interface GatewayWithStatus extends GatewayInfo { @@ -48,11 +72,15 @@ function parseTimestamp(value?: string): Date | null { return Number.isNaN(parsed.getTime()) ? null : parsed; } -function getGatewayStatus(lastSeen: Date | null): GatewayStatus { +function getGatewayStatus(lastSeen: Date | null, now: number): GatewayStatus { if (!lastSeen) return 'unknown'; - const minutes = (Date.now() - lastSeen.getTime()) / (1000 * 60); - if (minutes <= 10) return 'online'; + const minutes = (now - lastSeen.getTime()) / (1000 * 60); + // Gateways are polled every 5 minutes and the overview is rebuilt every 5, so allow + // up to three polls before a healthy gateway stops counting as online + if (minutes <= 15) return 'online'; if (minutes <= 30) return 'degraded'; + // Gone from the registry for a week: it has left the federation (same rule as the backend) + if (minutes > RETIRED_AFTER_MINUTES) return 'retired'; return 'offline'; } @@ -67,10 +95,10 @@ function formatDateTime(date: Date | null): string { }); } -function formatRelative(date: Date | null): string { +function formatRelative(date: Date | null, now: number): string { if (!date) return 'Never seen'; - const diffMs = Date.now() - date.getTime(); + const diffMs = now - date.getTime(); const minutes = Math.floor(diffMs / (1000 * 60)); if (minutes < 1) return 'just now'; if (minutes < 60) return `${minutes}m ago`; @@ -130,7 +158,7 @@ function getUptimeStripClass(status: UptimeStripStatus): string { function buildUptimeStrip(gateway: GatewayWithStatus, windowMinutes: number): UptimeStripStatus[] { const segments = 30; - if (gateway.status === 'unknown' || windowMinutes <= 0) { + if (gateway.status === 'unknown' || gateway.status === 'retired' || windowMinutes <= 0) { return Array.from({ length: segments }, () => 'unknown'); } @@ -204,16 +232,6 @@ function getUptimeBucketTooltip( )} · Estimated ${status}`; } -function formatEndpointLabel(endpoint: string): string { - try { - const url = new URL(endpoint); - if (url.protocol === 'iroh:') return `iroh://${shortId(url.host)}`; - return `${url.host}${url.pathname === '/' ? '' : url.pathname}`; - } catch { - return shortId(endpoint); - } -} - function mergeGatewayData(observedGateways: GatewayInfo[], liveGateways: GatewayInfo[]): GatewayInfo[] { if (observedGateways.length === 0) return liveGateways; if (liveGateways.length === 0) return observedGateways; @@ -253,6 +271,7 @@ function mergeGatewayData(observedGateways: GatewayInfo[], liveGateways: Gateway interface GatewaySelection { gateways: GatewayInfo[]; warning: GatewayWarningState | null; + retry: boolean; } interface RawAnnouncementDialog { @@ -260,257 +279,372 @@ interface RawAnnouncementDialog { raw: Record; } -function selectGatewayData( - observedGateways: GatewayInfo[], - liveGateways: GatewayInfo[], - observedError: string | null, - liveError: string | null, - hasInvite: boolean, -): GatewaySelection { - if (liveGateways.length > 0) { - if (observedGateways.length === 0) { +type LiveLookup = + | { status: 'pending' } + | { status: 'skipped' } + | { status: 'done'; gateways: GatewayInfo[]; error: string | null }; + +interface LoadError { + window: SelectableWindow; + message: string; + network: boolean; // fetch itself failed, as opposed to an error answer from the API +} + +interface GatewaySources { + observed: GatewayInfo[] | null; // recorded history, null until it has loaded once + observedError: LoadError | null; // the latest request for the selected window failed + live: LiveLookup; // the federation's own registry, which only adds details + federationFailed: boolean; + federationOffline: boolean; + dataWindow: SelectableWindow; +} + +// Banners are only for problems or empty states; normal pages show none +function selectGatewayData({ + observed, + observedError, + live, + federationFailed, + federationOffline, + dataWindow, +}: GatewaySources): GatewaySelection { + const liveGateways = live.status === 'done' ? live.gateways : []; + const liveError = live.status === 'done' ? live.error : null; + + if (observed !== null) { + const gateways = liveGateways.length > 0 ? mergeGatewayData(observed, liveGateways) : observed; + if (observedError) { return { - gateways: liveGateways, + gateways, + retry: true, warning: { - level: 'info', - title: 'Live Gateway Data Only', - message: 'Showing gateways from invite-based live discovery.', - detail: 'Observed gateway history is unavailable on this backend.', + level: 'warning', + title: `Could not load ${observedError.window.toUpperCase()} gateway data`, + message: dataWindow === observedError.window + ? 'Showing the last data that loaded.' + : `Showing ${dataWindow.toUpperCase()} data instead.`, + detail: observedError.message, }, }; } - - return { - gateways: mergeGatewayData(observedGateways, liveGateways), - warning: { - level: 'info', - title: 'Merged Gateway Sources', - message: 'Combined observed history with live invite-based gateway metadata.', - detail: 'Live data provides the latest registry details, while observed data keeps status and activity context.', - }, - }; - } - - if (observedGateways.length > 0) { - if (liveError) { + if (federationFailed) { return { - gateways: observedGateways, + gateways, + retry: true, warning: { level: 'warning', - title: 'Live Lookup Failed', - message: 'Showing observed gateway data from the backend.', - detail: `Live invite lookup error: ${liveError}`, + title: "Could not load the federation's details", + message: 'Its name and the live gateway check are missing until they load.', }, }; } - - if (hasInvite) { + if (observed.length > 0) return { gateways, warning: null, retry: false }; + if (liveGateways.length > 0) { + return { + gateways, + retry: false, + warning: { + level: 'info', + title: 'No recorded history yet', + message: "These gateways come from the federation's live registry. Their status history has not been recorded yet.", + }, + }; + } + if (live.status === 'pending') { + return { + gateways, + retry: false, + warning: { level: 'info', title: 'Checking for gateways', message: "Looking up the federation's gateway registry…" }, + }; + } + if (liveError) { return { - gateways: observedGateways, + gateways, + retry: true, warning: { level: 'warning', - title: 'Live Lookup Returned No Gateways', - message: 'Showing observed gateway data from the backend.', - detail: 'Invite-based lookup returned an empty gateway list.', + title: "Could not check the federation's gateway registry", + message: 'No gateways have been recorded for this federation, and its own registry could not be reached.', + detail: liveError, }, }; } - - return { gateways: observedGateways, warning: null }; - } - - const reason = observedError ?? 'No gateway data available on the configured API backend.'; - if (liveError) { return { - gateways: [], - warning: { - level: 'error', - title: 'Gateway Data Unavailable', - message: 'Could not load gateway data from backend or invite-based live lookup.', - detail: `Backend: ${reason}. Live: ${liveError}`, - }, + gateways, + retry: false, + warning: federationOffline + ? { level: 'info', title: 'Federation offline', message: 'This federation is offline and no gateways have been recorded for it.' } + : { level: 'info', title: 'No gateways', message: 'No gateways have been recorded for this federation.' }, }; } - if (hasInvite) { + // Registry-only rows stand in for the history only once loading it has failed + if (observedError && liveGateways.length > 0) { return { - gateways: [], + gateways: liveGateways, + retry: true, warning: { level: 'warning', - title: 'No Gateways Returned', - message: 'Both backend and invite-based lookup returned no gateway records.', - detail: 'This can happen for new federations or backends with incomplete gateway ingestion.', + title: 'Gateway history unavailable', + message: "Showing gateways from the federation's live registry only.", + detail: observedError?.message, }, }; } return { gateways: [], + retry: true, warning: { - level: 'warning', - title: 'Gateway Data Unavailable', - message: 'The configured backend did not return any gateway data.', - detail: reason, + level: 'error', + title: 'Could not load gateway data', + message: observedError?.network + ? 'Check your connection and try again.' + : 'The observer API returned an error. Try again in a moment.', + detail: observedError?.message, }, }; } +function parseOption(options: readonly T[], value: string | null): T | null { + const lower = value?.toLowerCase(); + return options.find((option) => option === lower) ?? null; +} + +function defaultDirection(sort: GatewaySort): SortDirection { + return sort === 'status' ? 'asc' : 'desc'; +} + +function errorMessage(err: unknown, fallback: string): string { + return err instanceof Error ? err.message : fallback; +} + +const DARK_QUERY = '(prefers-color-scheme: dark)'; +function subscribeToColorScheme(onChange: () => void) { + const media = window.matchMedia(DARK_QUERY); + media.addEventListener('change', onChange); + return () => media.removeEventListener('change', onChange); +} + +// Re-renders when the system switches between light and dark, for the 'auto' theme +function usePrefersDark(): boolean { + return useSyncExternalStore(subscribeToColorScheme, () => window.matchMedia(DARK_QUERY).matches); +} + export function FederationGateways() { - const { id } = useParams<{ id: string }>(); - const [federation, setFederation] = useState(null); - const [gateways, setGateways] = useState([]); - const [loading, setLoading] = useState(true); - const [windowLoading, setWindowLoading] = useState(false); - const [error, setError] = useState(null); - const [gatewayWarning, setGatewayWarning] = useState(null); - const [timeWindow, setTimeWindow] = useState('7d'); - const [uptimeTrend, setUptimeTrend] = useState([]); - const [uptimeTrendLoading, setUptimeTrendLoading] = useState(true); - const [gatewayFilter, setGatewayFilter] = useState('all'); - const [gatewaySort, setGatewaySort] = useState('freshness'); - const [sortDirection, setSortDirection] = useState('desc'); + // The router remounts this page per federation (App.tsx), so id never changes here + const { id = '' } = useParams<{ id: string }>(); + const [searchParams, setSearchParams] = useSearchParams(); + // The view lives in the URL so a refresh, Back or a shared link keeps it. Without a + // window in the URL, e.g. coming from "Gateway Details", the last one picked applies; + // it is read once, so picking a window in another tab does not change this one. + const [storedWindow] = useState(() => parseOption(GATEWAY_WINDOWS, readStorage(WINDOW_STORAGE_KEY))); + const timeWindow = parseOption(GATEWAY_WINDOWS, searchParams.get('window')) ?? storedWindow ?? DEFAULT_WINDOW; + const gatewayFilter = parseOption(GATEWAY_FILTERS, searchParams.get('status')) ?? 'all'; + const gatewaySort = parseOption(GATEWAY_SORTS, searchParams.get('sort')) ?? 'freshness'; + const sortDirection = parseOption(SORT_DIRECTIONS, searchParams.get('dir')) ?? defaultDirection(gatewaySort); + + const [federation, setFederation] = useState(undefined); + const [federationFailed, setFederationFailed] = useState(false); + const [federationKey, setFederationKey] = useState(0); + // Each response is stored with the window it belongs to, so a window switch never + // recomputes the previous window's numbers against the new window's length + const [observed, setObserved] = useState<{ window: SelectableWindow; gateways: GatewayInfo[] } | null>(null); + const [observedError, setObservedError] = useState(null); + const [observedLoading, setObservedLoading] = useState(true); + const [live, setLive] = useState({ status: 'pending' }); + const [liveKey, setLiveKey] = useState(0); + const [trend, setTrend] = useState<{ window: SelectableWindow; points: GatewayUptimeTrendPoint[] } | null>(null); + const [updatedAt, setUpdatedAt] = useState(null); + const [now, setNow] = useState(() => Date.now()); + const [reloadKey, setReloadKey] = useState(0); // Try again + const [refreshKey, setRefreshKey] = useState(0); // background refresh const [gatewaySearch, setGatewaySearch] = useState(''); const [visibleGatewayCount, setVisibleGatewayCount] = useState(INITIAL_RENDER_COUNT); const [rawAnnouncement, setRawAnnouncement] = useState(null); - const hasLoadedOnce = useRef(false); - const requestSeq = useRef(0); - const federationCache = useRef>(new Map()); - const liveGatewayCache = useRef>(new Map()); + const loadInFlight = useRef(false); + const lastLoadAt = useRef(0); // when the last load finished + const focusAfterRetry = useRef(false); + const headingRef = useRef(null); + + // One navigation per action; replace keeps Back working. It starts from the address + // bar because the router applies URL updates in a transition, so the hook's copy can + // lag behind a quick second click and drop the first change. + const updateView = useCallback((changes: Partial>) => { + const next = new URLSearchParams(window.location.search); + for (const [key, value] of Object.entries(changes)) { + if (value == null) next.delete(key); + else next.set(key, value); + } + setSearchParams(next, { replace: true }); + }, [setSearchParams]); + + const selectWindow = (nextWindow: SelectableWindow) => { + writeStorage(WINDOW_STORAGE_KEY, nextWindow); + updateView({ window: nextWindow }); + }; useEffect(() => { - if (!id) return; let cancelled = false; - const currentRequest = ++requestSeq.current; + api.getFederations() + .then((federations) => { + if (cancelled) return; + setFederation(federations.find((item) => item.id === id) ?? null); + setFederationFailed(false); + }) + .catch(() => { + // Only the name and the live lookup need it; the recorded data still shows + if (!cancelled) setFederationFailed(true); + }); + return () => { + cancelled = true; + }; + }, [id, federationKey]); - if (!hasLoadedOnce.current) { - setLoading(true); - } else { - setWindowLoading(true); - } - setError(null); - setGatewayWarning(null); - setUptimeTrend([]); - setUptimeTrendLoading(true); - - (async () => { - try { - void api.getFederationGatewayUptimeTrend(id, timeWindow) - .then((trend) => { - if (!cancelled && currentRequest === requestSeq.current) setUptimeTrend(trend); - }) - .catch(() => { - if (!cancelled && currentRequest === requestSeq.current) setUptimeTrend([]); - }) - .finally(() => { - if (!cancelled && currentRequest === requestSeq.current) setUptimeTrendLoading(false); - }); - let fed: FederationSummary | null | undefined = federationCache.current.get(id); - if (fed === undefined) { - const federations = await api.getFederations(); - fed = federations.find((item) => item.id === id) || null; - federationCache.current.set(id, fed); - } - if (cancelled || currentRequest !== requestSeq.current) return; - - setFederation(fed ?? null); - - let observedGateways: GatewayInfo[] = []; - let observedError: string | null = null; - try { - observedGateways = await api.getFederationGateways(id, timeWindow); - } catch (observedErr: unknown) { - observedError = - observedErr instanceof Error - ? observedErr.message - : `Failed to fetch gateways for federation ${id}`; - } - if (cancelled || currentRequest !== requestSeq.current) return; - - let liveGateways: GatewayInfo[] = []; - let liveError: string | null = null; - if (fed?.invite) { - const cachedLive = liveGatewayCache.current.get(fed.invite); - if (cachedLive) { - liveGateways = cachedLive.gateways; - liveError = cachedLive.error; - } else { - try { - liveGateways = await api.getFederationGatewaysByInvite(fed.invite); - } catch (liveErr: unknown) { - liveError = - liveErr instanceof Error - ? liveErr.message - : 'Invite-based gateway lookup failed.'; - } - liveGatewayCache.current.set(fed.invite, { - gateways: liveGateways, - error: liveError, - }); - } - } - if (cancelled || currentRequest !== requestSeq.current) return; - - const selection = selectGatewayData( - observedGateways, - liveGateways, - observedError, - liveError, - Boolean(fed?.invite), - ); - setGateways(selection.gateways); - setGatewayWarning(selection.warning); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Failed to load gateways'; - if (!hasLoadedOnce.current) { - setError(message); - } else { - setGatewayWarning({ - level: 'warning', - title: 'Refresh Failed', - message: 'Failed to refresh the selected time window.', - detail: message, - }); - } - } finally { - if (!cancelled && currentRequest === requestSeq.current) { - if (!hasLoadedOnce.current) { - setLoading(false); - hasLoadedOnce.current = true; - } - setWindowLoading(false); - } - } - })(); + // Keep retrying the federation list while it is missing + useEffect(() => { + if (!federationFailed) return; + const timer = setTimeout(() => setFederationKey((key) => key + 1), REFRESH_INTERVAL_MS); + return () => clearTimeout(timer); + }, [federationFailed, federationKey]); + useEffect(() => { + const controller = new AbortController(); + const { signal } = controller; + loadInFlight.current = true; + setObservedLoading(true); + + // The table and the trend come in one answer, so they always describe the same moment + api.getFederationGatewayOverview(id, timeWindow, signal) + .then((overview) => { + if (signal.aborted) return; + setObserved({ window: timeWindow, gateways: overview.gateways ?? [] }); + setTrend({ window: timeWindow, points: overview.uptime_trend ?? [] }); + setObservedError(null); + setUpdatedAt(Date.parse(overview.computed_at)); + }) + .catch((err: unknown) => { + if (signal.aborted) return; + setObservedError({ + window: timeWindow, + message: errorMessage(err, 'Failed to fetch gateways'), + network: err instanceof TypeError, + }); + }) + .finally(() => { + if (signal.aborted) return; + setObservedLoading(false); + loadInFlight.current = false; + lastLoadAt.current = Date.now(); + }); + + return () => { + controller.abort(); + loadInFlight.current = false; + }; + }, [id, timeWindow, reloadKey, refreshKey]); + + // The federation's own registry only adds details, so it never holds up the page + const invite = federation?.invite; + const federationOffline = federation?.health === 'offline'; + useEffect(() => { + // An offline federation's guardians can't answer; the lookup would hang for a minute + if (!invite || federationOffline) return; + const controller = new AbortController(); + let cancelled = false; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, LIVE_LOOKUP_TIMEOUT_MS); + setLive({ status: 'pending' }); + api.getFederationGatewaysByInvite(invite, controller.signal) + .then((gateways) => { + if (!cancelled) setLive({ status: 'done', gateways, error: null }); + }) + .catch((err: unknown) => { + if (cancelled) return; + setLive({ + status: 'done', + gateways: [], + error: timedOut ? 'The live registry lookup timed out' : errorMessage(err, 'The live registry lookup failed'), + }); + }) + .finally(() => clearTimeout(timeout)); return () => { cancelled = true; + clearTimeout(timeout); + controller.abort(); + }; + }, [invite, federationOffline, liveKey]); + + // Keep statuses and "seen" times current, and refresh a minute after the last load + // finished. A load in flight is never interrupted, so slow windows can complete. + useEffect(() => { + const clock = setInterval(() => setNow(Date.now()), 30_000); + const refresh = setInterval(() => { + if (document.visibilityState !== 'visible' || loadInFlight.current) return; + if (Date.now() - lastLoadAt.current >= REFRESH_INTERVAL_MS) setRefreshKey((key) => key + 1); + }, REFRESH_CHECK_MS); + const onVisibilityChange = () => { + if (document.visibilityState !== 'visible') return; + setNow(Date.now()); + if (!loadInFlight.current && Date.now() - lastLoadAt.current > 30_000) setRefreshKey((key) => key + 1); + }; + document.addEventListener('visibilitychange', onVisibilityChange); + return () => { + clearInterval(clock); + clearInterval(refresh); + document.removeEventListener('visibilitychange', onVisibilityChange); }; - }, [id, timeWindow]); - - const windowMinutes = useMemo(() => { - switch (timeWindow) { - case '1h': - return 60; - case '24h': - return 24 * 60; - case '7d': - return 7 * 24 * 60; - case '30d': - return 30 * 24 * 60; - case '90d': - default: - return 90 * 24 * 60; + }, []); + + const liveLookup = useMemo(() => { + if (invite && !federationOffline) return live; + return { status: federation === undefined && !federationFailed ? 'pending' : 'skipped' }; + }, [federation, federationFailed, federationOffline, invite, live]); + + // An error only describes the window it happened in, never one that is still loading + const currentError = observedError?.window === timeWindow ? observedError : null; + const dataWindow = observed?.window ?? timeWindow; + const windowMinutes = WINDOW_MINUTES[dataWindow]; + const selection = useMemo(() => selectGatewayData({ + observed: observed?.gateways ?? null, + observedError: currentError, + live: liveLookup, + federationFailed, + federationOffline, + dataWindow, + }), [currentError, dataWindow, federationFailed, federationOffline, liveLookup, observed]); + const gateways = selection.gateways; + + // Try again repeats only what failed, and is ignored while an attempt is running + const retryBusy = observedLoading; + const retry = () => { + if (retryBusy) return; + focusAfterRetry.current = true; + setReloadKey((key) => key + 1); + if (federationFailed) setFederationKey((key) => key + 1); + if (live.status === 'done' && live.error) setLiveKey((key) => key + 1); + }; + + // When a retry succeeds, its banner disappears together with the focused Try again + // button; put keyboard focus on the page heading instead of losing it + useEffect(() => { + if (focusAfterRetry.current && !observedLoading && document.activeElement === document.body) { + focusAfterRetry.current = false; + headingRef.current?.focus(); } - }, [timeWindow]); + }); const rows = useMemo(() => { return gateways .map((gateway) => { const firstSeenDate = parseTimestamp(gateway.first_seen); const lastSeenDate = parseTimestamp(gateway.last_seen); - const status = getGatewayStatus(lastSeenDate); + const status = getGatewayStatus(lastSeenDate, now); const minutesSinceLastSeen = lastSeenDate - ? (Date.now() - lastSeenDate.getTime()) / (1000 * 60) + ? (now - lastSeenDate.getTime()) / (1000 * 60) : null; const observedUptime = gateway.uptime_window; const hasObservedSamples = Boolean(observedUptime && observedUptime.sample_count > 0); @@ -578,36 +712,40 @@ export function FederationGateways() { searchText, }; }); - }, [gateways, windowMinutes]); + }, [gateways, now, windowMinutes]); + + // Retired gateways are listed separately and left out of every figure + const activeRows = useMemo(() => rows.filter((row) => row.status !== 'retired'), [rows]); const totals = useMemo(() => { - const total = rows.length; - const online = rows.filter((row) => row.status === 'online').length; - const degraded = rows.filter((row) => row.status === 'degraded').length; - const offline = rows.filter((row) => row.status === 'offline').length; + const total = activeRows.length; + const online = activeRows.filter((row) => row.status === 'online').length; + const degraded = activeRows.filter((row) => row.status === 'degraded').length; + const offline = activeRows.filter((row) => row.status === 'offline').length; const unknown = total - online - degraded - offline; - const vetted = rows.filter((row) => row.vetted).length; + const vetted = activeRows.filter((row) => row.vetted).length; + const retired = rows.length - total; - return { total, online, degraded, offline, unknown, vetted }; - }, [rows]); + return { total, online, degraded, offline, unknown, vetted, retired }; + }, [activeRows, rows]); const avgUptime = useMemo(() => { - const observedRows = rows.filter((row) => row.coveragePct > 0); + const observedRows = activeRows.filter((row) => row.coveragePct > 0); if (observedRows.length === 0) return 0; const total = observedRows.reduce((sum, row) => sum + row.estimatedUptimePct, 0); return total / observedRows.length; - }, [rows]); + }, [activeRows]); const avgCoverage = useMemo(() => { - if (rows.length === 0) return 0; - const total = rows.reduce((sum, row) => sum + row.coveragePct, 0); - return total / rows.length; - }, [rows]); + if (activeRows.length === 0) return 0; + const total = activeRows.reduce((sum, row) => sum + row.coveragePct, 0); + return total / activeRows.length; + }, [activeRows]); const filteredRows = useMemo(() => { const query = gatewaySearch.trim().toLowerCase(); const filtered = rows.filter((row) => ( - (gatewayFilter === 'all' || row.status === gatewayFilter) + (gatewayFilter === 'all' ? row.status !== 'retired' : row.status === gatewayFilter) && (!query || row.searchText.includes(query)) )); @@ -634,7 +772,7 @@ export function FederationGateways() { useEffect(() => { setVisibleGatewayCount(INITIAL_RENDER_COUNT); - }, [gatewayFilter, gatewaySearch, gatewaySort, id, sortDirection]); + }, [gatewayFilter, gatewaySearch, gatewaySort, sortDirection]); useEffect(() => { if (!rawAnnouncement) return undefined; @@ -658,70 +796,135 @@ export function FederationGateways() { [filteredRows, visibleGatewayCount], ); - const uptimeTrendOption = useMemo(() => ({ + const { theme } = useTheme(); + const prefersDark = usePrefersDark(); + const darkMode = theme === 'dark' || (theme === 'auto' && prefersDark); + const chartColors = useMemo(() => (darkMode + ? { label: '#9ca3af', axis: '#4b5563', grid: '#374151', line: '#60a5fa', area: 'rgba(96, 165, 250, 0.15)' } + : { label: '#6b7280', axis: '#d1d5db', grid: '#e5e7eb', line: '#2563eb', area: 'rgba(37, 99, 235, 0.12)' }), [darkMode]); + + // A window switch shows the loading state rather than the previous window's trend + const uptimeTrend = trend?.window === timeWindow ? trend.points : EMPTY_TREND; + const uptimeTrendOption = useMemo(() => ({ animationDuration: 200, grid: { top: 18, right: 18, bottom: 30, left: 42 }, - tooltip: { trigger: 'axis', valueFormatter: (value: number | string) => `${Number(value).toFixed(1)}%` }, + tooltip: { trigger: 'axis', valueFormatter: (value) => `${Number(value).toFixed(1)}%` }, xAxis: { type: 'category', boundaryGap: false, - data: uptimeTrend.map((point) => new Date(point.day).toLocaleDateString('en-US', { month: 'short', day: 'numeric' })), - axisLabel: { color: '#6b7280', fontSize: 11 }, axisLine: { lineStyle: { color: '#d1d5db' } }, + // Days are bucketed in the server's time zone (midnight there is e.g. 22:00Z). Half a + // day later, formatted in UTC, names that same calendar day for every viewer. + data: uptimeTrend.map((point) => new Date(Date.parse(point.day) + 12 * 60 * 60 * 1000).toLocaleDateString('en-US', { month: 'short', day: 'numeric', timeZone: 'UTC' })), + axisLabel: { color: chartColors.label, fontSize: 11 }, axisLine: { lineStyle: { color: chartColors.axis } }, }, - yAxis: { type: 'value', min: 0, max: 100, axisLabel: { formatter: '{value}%', color: '#6b7280', fontSize: 11 }, splitLine: { lineStyle: { color: '#e5e7eb' } } }, - series: [{ name: 'Gateway availability', type: 'line', smooth: true, symbol: uptimeTrend.length > 45 ? 'none' : 'circle', symbolSize: 5, data: uptimeTrend.map((point) => Number(point.uptime_pct.toFixed(2))), lineStyle: { color: '#2563eb', width: 2.5 }, itemStyle: { color: '#2563eb' }, areaStyle: { color: 'rgba(37, 99, 235, 0.12)' } }], - }), [uptimeTrend]); - - if (loading) { - return ( -
-
Loading gateways...
-
- ); - } + yAxis: { type: 'value', min: 0, max: 100, axisLabel: { formatter: '{value}%', color: chartColors.label, fontSize: 11 }, splitLine: { lineStyle: { color: chartColors.grid } } }, + series: [{ name: 'Gateway availability', type: 'line', smooth: true, symbol: uptimeTrend.length > 45 ? 'none' : 'circle', symbolSize: 5, data: uptimeTrend.map((point) => Number(point.uptime_pct.toFixed(2))), lineStyle: { color: chartColors.line, width: 2.5 }, itemStyle: { color: chartColors.line }, areaStyle: { color: chartColors.area } }], + }), [chartColors, uptimeTrend]); - if (error) { + if (federation === null && observed !== null && observed.gateways.length === 0) { return ( -
-
Error: {error}
+
+

This federation is not tracked by the observer.

+ ← All federations
); } - const federationName = federation?.name || 'Federation'; - const metricsWindow = (rows.find((row) => row.metrics_window)?.metrics_window ?? timeWindow) + const metricsWindow = (rows.find((row) => row.metrics_window)?.metrics_window ?? dataWindow) .toUpperCase(); + // Until the recorded gateways arrive there are no numbers to show, only the controls + const hasData = observed !== null || gateways.length > 0; + const switchingWindow = observed !== null && dataWindow !== timeWindow && observedLoading; + const trendPlaceholder = trend?.window === timeWindow + ? 'No gateway poll history is available for this window.' + : observedLoading + ? 'Loading availability trend…' + : currentError ? 'The availability trend could not be loaded.' : ''; + const windowStatus = observedLoading && (observed === null || dataWindow !== timeWindow) + ? `Loading ${timeWindow.toUpperCase()}…` + : observed === null + ? ' ' + : dataWindow !== timeWindow + ? `Showing ${dataWindow.toUpperCase()}` + : updatedAt ? `Updated ${formatRelative(new Date(updatedAt), now)}` : 'Refreshing…'; return (
← Federation details
-
Lightning gateway observatory

{federationName}

{(['24h', '7d', '30d', '90d'] as GatewayWindow[]).map((window) => )}
{windowLoading ? 'Refreshing…' : `Observed over ${timeWindow.toUpperCase()}`}
-
{totals.total} gateways● {totals.online} online● {totals.degraded} degraded● {totals.offline} offline{totals.vetted} vetted{avgUptime.toFixed(1)}% uptime ({avgCoverage.toFixed(0)}% observed)
+
+
+
Lightning gateway observatory
+

+ {federation?.name || (federation === undefined && !federationFailed + ?

+
+
+
+ {GATEWAY_WINDOWS.map((option) => ( + + ))} +
+
{windowStatus}
+
+
+ {hasData && ( +
{totals.total} gateways● {totals.online} online● {totals.degraded} degraded● {totals.offline} offline{totals.unknown > 0 && ● {totals.unknown} unknown}{totals.retired > 0 && {totals.retired} retired}{totals.vetted} vetted{avgCoverage > 0 ? <>{avgUptime.toFixed(1)}% uptime ({avgCoverage.toFixed(0)}% observed) : 'No uptime data'}
+ )}
- {gatewayWarning && } + {(hasData || currentError) && selection.warning && ( + + )} + {!hasData && !currentError && ( +
Loading gateways…
+ )} + + {hasData && (<>
-

Gateway availability trend ({timeWindow.toUpperCase()})

Each point combines every gateway poll snapshot recorded on that calendar day.

{uptimeTrend.length > 0 && {uptimeTrend.length} calendar day{uptimeTrend.length === 1 ? '' : 's'} with data}
- {uptimeTrendLoading ?
Loading availability trend…
: uptimeTrend.length > 0 ? Loading chart…
}> :
No gateway poll history is available for this window.
} +

Gateway availability trend ({timeWindow.toUpperCase()})

Each point combines every gateway poll snapshot recorded on that calendar day.

{uptimeTrend.length > 0 && {uptimeTrend.length} calendar day{uptimeTrend.length === 1 ? '' : 's'} with data}
+ {uptimeTrend.length > 0 ? ( +
}> + Loading chart…}> + + + + ) : ( +
{trendPlaceholder}
+ )} -
+
-

Gateway directory

{filteredRows.length} of {rows.length} gateways · every row includes its 30-bucket availability strip, ordered oldest to newest.

setGatewaySearch(event.target.value)} aria-label="Search gateways" placeholder="Search gateway, node, endpoint" className="rounded-lg border border-gray-300 bg-white px-3 py-2 text-sm text-gray-900 outline-none focus:ring-2 focus:ring-blue-500 dark:border-gray-600 dark:bg-gray-900 dark:text-white" />
-
{(['all', 'online', 'degraded', 'offline', 'unknown'] as GatewayFilter[]).map((filter) => )}
+

Gateway directory

{filteredRows.length} of {rows.length} gateways · every row includes its 30-bucket availability strip, ordered oldest to newest.

setGatewaySearch(event.target.value)} aria-label="Search gateways" placeholder="Search gateway, node, endpoint" className="rounded-lg border border-gray-300 bg-white px-3 py-2 text-sm text-gray-900 outline-none focus:ring-2 focus:ring-blue-500 dark:border-gray-600 dark:bg-gray-900 dark:text-white" />
+
{GATEWAY_FILTERS.map((filter) => )}
OnlineDegradedOfflineUnknown
-
- - + {/* Phones get one stacked card per gateway instead of a 960px table behind a sideways scroll */} +
+
+ - + @@ -730,11 +933,15 @@ export function FederationGateways() { - + {filteredRows.length === 0 && ( - - + )} @@ -742,9 +949,9 @@ export function FederationGateways() { {visibleRows.map((gateway) => ( - - - - -
Gateway AvailabilityEndpoint
- No gateways match these filters. +
+ {gatewayFilter !== 'all' || gatewaySearch.trim() + ? 'No gateways match these filters.' + : totals.retired > 0 + ? `No active gateways. ${totals.retired} retired ${totals.retired === 1 ? 'one is' : 'ones are'} under the Retired filter.` + : 'No gateways to show.'}
+
{gateway.lightning_alias || 'Unnamed Gateway'}
@@ -761,36 +968,44 @@ export function FederationGateways() { Node: {shortId(gateway.node_pub_key)} 		+
{gateway.status} - seen {formatRelative(gateway.lastSeenDate)} + {gateway.lastSeenDate ? `seen ${formatRelative(gateway.lastSeenDate, now)}` : 'never seen'}
{gateway.coveragePct > 0 ? `${gateway.estimatedUptimePct.toFixed(1)}% uptime` : 'No samples'} {gateway.coveragePct.toFixed(0)}% observed
-
+ {/* One stop for keyboards and screen readers instead of 30 per row */} +
0 ? `${gateway.estimatedUptimePct.toFixed(1)}% uptime` : 'no samples'}, ${gateway.coveragePct.toFixed(0)}% observed`} + > {(uptimeStripByGatewayId.get(gateway.gateway_id) ?? []).map((status, index) => { const tooltip = getUptimeBucketTooltip(status, index, 30, windowMinutes); + // On phones the strip spans the whole card, so more tooltips anchor to an edge const tooltipPosition = index < 4 ? 'left-0' : index > 25 ? 'right-0' - : 'left-1/2 -translate-x-1/2'; + : index < 10 + ? 'left-0 sm:left-1/2 sm:-translate-x-1/2' + : index > 19 + ? 'right-0 sm:right-auto sm:left-1/2 sm:-translate-x-1/2' + : 'left-1/2 -translate-x-1/2'; return ( - -
On {formatCompactDuration(gateway.estimatedOnlineMinutes)}Off {formatCompactDuration(gateway.estimatedOfflineMinutes)}Unknown {formatCompactDuration(gateway.estimatedUnknownMinutes)}
 + +
Activity
{gateway.realActivityScore !== null ? ( <>
- {gateway.realActivityScore.toLocaleString()} score + {gateway.realActivityScore.toLocaleString()} score
{metricsWindow} · {gateway.fundCountWindow} funds · {gateway.settleCountWindow} settles · {gateway.cancelCountWindow} cancels
@@ -812,27 +1028,27 @@ export function FederationGateways() { ) : (
- N/A (no real {metricsWindow} data) + No activity recorded in {metricsWindow}
)}
+ +
Trust & history
{gateway.vetted ? '✓ Vetted' : 'Not vetted'}
First seen
{formatDateTime(gateway.firstSeenDate)}
Last seen
{formatDateTime(gateway.lastSeenDate)}
 - +
Endpoint
+ {/* Gateway APIs (iroh:// or /v1 endpoints) are not pages, so show them as text to copy */} + - {formatEndpointLabel(gateway.api_endpoint)} - + {gateway.api_endpoint} + {gateway.raw && ( )} + )} {rawAnnouncement && (
{ - const query = window ? `?window=${encodeURIComponent(window)}` : ''; - const response = await fetch(`${BASE_URL}/federations/${id}/gateways${query}`); + async getFederationGatewayOverview(id: string, window: GatewayWindow, signal?: AbortSignal): Promise { + const response = await fetch(`${BASE_URL}/federations/${id}/gateways/overview?window=${encodeURIComponent(window)}`, { signal }); if (!response.ok) { - throw new Error(`Failed to fetch gateways for federation ${id} (${response.status})`); + throw new Error(`Failed to fetch gateways (${response.status})`); } return response.json(); }, - async getFederationGatewayUptimeTrend( - id: string, - window: GatewayWindow, - ): Promise { - const response = await fetch( - `${BASE_URL}/federations/${id}/gateways/uptime-trend?window=${encodeURIComponent(window)}`, - ); - if (!response.ok) { - throw new Error(`Failed to fetch gateway uptime trend (${response.status})`); - } - return response.json(); - }, - - async getFederationGatewaysByInvite(inviteCode: string): Promise { + async getFederationGatewaysByInvite(inviteCode: string, signal?: AbortSignal): Promise { const encodedInvite = encodeURIComponent(inviteCode); - const response = await fetch(`${BASE_URL}/config/${encodedInvite}/gateways`); + const response = await fetch(`${BASE_URL}/config/${encodedInvite}/gateways`, { signal }); if (!response.ok) { throw new Error(`Failed to fetch gateways by invite (${response.status})`); } diff --git a/fmo_frontend_react/src/types/api.ts b/fmo_frontend_react/src/types/api.ts index 25f0391..95bbe6a 100644 --- a/fmo_frontend_react/src/types/api.ts +++ b/fmo_frontend_react/src/types/api.ts @@ -61,6 +61,15 @@ export interface GatewayUptimeTrendPoint { uptime_pct: number; } -export type GatewayWindow = '1h' | '24h' | '7d' | '30d' | '90d'; +export type GatewayWindow = '24h' | '7d' | '30d' | '90d'; + +// Gateways and their trend for one window, computed together on the server +export interface GatewayOverview { + window: GatewayWindow; + computed_at: string; + // Both are present unless the request narrows them with ?include= + gateways?: GatewayInfo[]; + uptime_trend?: GatewayUptimeTrendPoint[]; +} export type FederationHealth = 'online' | 'degraded' | 'offline'; diff --git a/fmo_frontend_react/src/utils/storage.ts b/fmo_frontend_react/src/utils/storage.ts new file mode 100644 index 0000000..d7602d2 --- /dev/null +++ b/fmo_frontend_react/src/utils/storage.ts @@ -0,0 +1,22 @@ +// Browsers throw on any localStorage access when site data is blocked, so +// preferences must never be able to break a page. + +export function readStorage(key: string): string | null { + try { + return window.localStorage.getItem(key); + } catch { + return null; + } +} + +export function writeStorage(key: string, value: string | null): void { + try { + if (value === null) { + window.localStorage.removeItem(key); + } else { + window.localStorage.setItem(key, value); + } + } catch { + // The preference is not remembered; nothing else depends on it + } +} diff --git a/fmo_server/src/config/mod.rs b/fmo_server/src/config/mod.rs index 7d72d83..9684b9e 100644 --- a/fmo_server/src/config/mod.rs +++ b/fmo_server/src/config/mod.rs @@ -61,13 +61,14 @@ pub async fn fetch_federation_config( pub async fn fetch_federation_gateways( Path(invite): Path, + State(state): State, ) -> Result>> { - let connectors = fedimint_connectors::ConnectorRegistry::build_from_client_env()? - .bind() - .await?; + // Reuse the observer's registry: a registry built per request keeps its + // guardian connections open after the request ends + let connectors = state.federation_observer.connectors(); let (config, _api) = - fedimint_api_client::download_from_invite_code(&connectors, &invite).await?; - let gateways = fetch_gateways_for_config(&config).await?; + fedimint_api_client::download_from_invite_code(connectors, &invite).await?; + let gateways = fetch_gateways_for_config(connectors, &config).await?; Ok(gateways.into()) } diff --git a/fmo_server/src/federation/gateways.rs b/fmo_server/src/federation/gateways.rs index 03e989f..65fbea4 100644 --- a/fmo_server/src/federation/gateways.rs +++ b/fmo_server/src/federation/gateways.rs @@ -1,11 +1,13 @@ use std::collections::HashMap; -use std::time::Duration; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; use anyhow::{bail, Context}; use axum::extract::{Path, Query, State}; use axum::Json; use chrono::{DateTime, Utc}; use fedimint_api_client::api::{DynGlobalApi, FederationApiExt}; +use fedimint_connectors::ConnectorRegistry; use fedimint_core::config::{ClientConfig, FederationId}; use fedimint_core::core::ModuleInstanceId; use fedimint_core::encoding::Encodable; @@ -13,7 +15,8 @@ use fedimint_core::module::ApiRequestErased; use fedimint_ln_common::federation_endpoint_constants::LIST_GATEWAYS_ENDPOINT; use fedimint_ln_common::LightningGatewayAnnouncement; use fmo_api_types::{ - GatewayActivityMetrics, GatewayInfo, GatewayUptimeMetrics, GatewayUptimeTrendPoint, + GatewayActivityMetrics, GatewayInfo, GatewayOverview, GatewayUptimeMetrics, + GatewayUptimeTrendPoint, }; use futures::future::join_all; use serde::Deserialize; @@ -25,10 +28,21 @@ use crate::util::query; const GATEWAY_POLL_INTERVAL_MINUTES: u64 = 5; const GATEWAY_SNAPSHOT_RETENTION_DAYS: i64 = 90; const GATEWAY_PRUNE_INTERVAL_HOURS: i64 = 6; - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum GatewayMetricsWindow { - H1, +// A gateway missing from the registry this long has left the federation: it +// gets no more "not seen" samples, and later ones don't count against uptime or +// the trend +const GATEWAY_RETIRED_AFTER_DAYS: i32 = 7; +// Matches the poll interval, so short windows are never more than one poll +// behind; longer windows refresh every few ticks (see refresh_every_ticks) +const GATEWAY_OVERVIEW_TICK_MINUTES: u64 = 5; + +/// Overviews for every federation and window, rebuilt in the background so +/// requests never wait for the queries (90D takes seconds) +pub(crate) type GatewayOverviewCache = + Arc>>>; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum GatewayMetricsWindow { H24, D7, D30, @@ -36,22 +50,23 @@ enum GatewayMetricsWindow { } impl GatewayMetricsWindow { + // The page's default window first, so it is ready soonest after a restart + const ALL: [Self; 4] = [Self::D7, Self::H24, Self::D30, Self::D90]; + fn parse(value: Option<&str>) -> anyhow::Result { match value.unwrap_or("7d") { - "1h" => Ok(Self::H1), "24h" => Ok(Self::H24), "7d" => Ok(Self::D7), "30d" => Ok(Self::D30), "90d" => Ok(Self::D90), - invalid => bail!( - "Invalid gateways window '{invalid}'. Supported values: 1h, 24h, 7d, 30d, 90d" - ), + invalid => { + bail!("Invalid gateways window '{invalid}'. Supported values: 24h, 7d, 30d, 90d") + } } } fn label(self) -> &'static str { match self { - Self::H1 => "1h", Self::H24 => "24h", Self::D7 => "7d", Self::D30 => "30d", @@ -61,33 +76,43 @@ impl GatewayMetricsWindow { fn duration(self) -> chrono::Duration { match self { - Self::H1 => chrono::Duration::hours(1), Self::H24 => chrono::Duration::hours(24), Self::D7 => chrono::Duration::days(7), Self::D30 => chrono::Duration::days(30), Self::D90 => chrono::Duration::days(90), } } + + /// How many overview ticks pass between refreshes. Five more minutes + /// barely move a 30- or 90-day figure, and those windows run the heaviest + /// queries. + fn refresh_every_ticks(self) -> u64 { + match self { + Self::H24 | Self::D7 => 1, + Self::D30 => 6, + Self::D90 => 12, + } + } } #[derive(Debug, Deserialize)] pub(super) struct GetFederationGatewaysParams { window: Option, + /// Comma-separated parts to return: `gateways`, `trend`. Both by default. + include: Option, } pub(crate) async fn fetch_gateways_for_config( + connectors: &ConnectorRegistry, config: &ClientConfig, ) -> anyhow::Result> { - let connectors = fedimint_connectors::ConnectorRegistry::build_from_client_env()? - .bind() - .await?; let peers = config .global .api_endpoints .iter() .map(|(&peer_id, peer_url)| (peer_id, peer_url.url.clone())) .collect(); - let api = DynGlobalApi::new(connectors, peers, None)?; + let api = DynGlobalApi::new(connectors.clone(), peers, None)?; let ln_instance_id = config .modules @@ -324,6 +349,7 @@ impl FederationObserver { SELECT g.gateway_id, FALSE AS is_seen FROM gateways g WHERE g.federation_id = $1 + AND g.last_seen >= $2::timestamptz - make_interval(days => $4) AND NOT EXISTS ( SELECT 1 FROM current_gateway_ids c @@ -339,7 +365,12 @@ impl FederationObserver { a.is_seen FROM all_gateway_ids a ON CONFLICT DO NOTHING", - &[&federation_id_bytes, &now, &gateway_ids], + &[ + &federation_id_bytes, + &now, + &gateway_ids, + &GATEWAY_RETIRED_AFTER_DAYS, + ], ) .await?; @@ -367,10 +398,107 @@ impl FederationObserver { Ok(()) } + /// Background task: rebuild every federation's gateway overviews. Runs in + /// a loop until cancelled. + pub(crate) async fn refresh_gateway_overviews(self) { + let mut interval = + tokio::time::interval(Duration::from_secs(GATEWAY_OVERVIEW_TICK_MINUTES * 60)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // Tick 0 builds every window, so all are ready right after a restart + let mut tick: u64 = 0; + loop { + interval.tick().await; + let start = Instant::now(); + let windows: Vec = GatewayMetricsWindow::ALL + .into_iter() + .filter(|window| tick.is_multiple_of(window.refresh_every_ticks())) + .collect(); + tick += 1; + let federations = match self.list_federations().await { + Ok(federations) => federations, + Err(e) => { + warn!("Failed to list federations for gateway overviews: {e:?}"); + continue; + } + }; + // One query at a time keeps the load on the database steady + for &window in &windows { + for federation in &federations { + let federation_id = federation.federation_id; + match self.compute_gateway_overview(federation_id, window).await { + Ok(overview) => { + self.gateway_overviews() + .write() + .expect("gateway overview lock poisoned") + .insert((federation_id, window), Arc::new(overview)); + } + Err(e) => warn!( + "Failed to compute {} gateway overview for {}: {e:?}", + window.label(), + federation_id + ), + } + } + } + info!( + "Gateway overviews refreshed in {:.2}s ({})", + start.elapsed().as_secs_f64(), + windows + .iter() + .map(|window| window.label()) + .collect::>() + .join(", ") + ); + } + } + + async fn gateway_overview( + &self, + federation_id: FederationId, + window: GatewayMetricsWindow, + ) -> anyhow::Result> { + let cached = self + .gateway_overviews() + .read() + .expect("gateway overview lock poisoned") + .get(&(federation_id, window)) + .cloned(); + // Only right after a restart, or for a federation added since the last + // refresh. Not cached here, so unknown IDs can't grow the cache. + match cached { + Some(overview) => Ok(overview), + None => Ok(Arc::new( + self.compute_gateway_overview(federation_id, window).await?, + )), + } + } + + async fn compute_gateway_overview( + &self, + federation_id: FederationId, + window: GatewayMetricsWindow, + ) -> anyhow::Result { + let computed_at = Utc::now(); + let window_start = computed_at - window.duration(); + Ok(GatewayOverview { + window: window.label().to_owned(), + computed_at, + gateways: Some( + self.list_federation_gateways(federation_id, window, window_start) + .await?, + ), + uptime_trend: Some( + self.federation_gateway_uptime_trend(federation_id, window_start) + .await?, + ), + }) + } + async fn list_federation_gateways( &self, federation_id: FederationId, window: GatewayMetricsWindow, + window_start_utc: DateTime, ) -> anyhow::Result> { #[derive(postgres_from_row::FromRow)] struct GatewayRow { @@ -402,7 +530,6 @@ impl FederationObserver { let conn = self.connection().await?; let federation_id_bytes = federation_id.consensus_encode_to_vec(); - let window_start_utc: DateTime = Utc::now() - window.duration(); let window_start_naive = window_start_utc.naive_utc(); let metrics_window = window.label().to_owned(); @@ -416,9 +543,12 @@ impl FederationObserver { ) .await?; + // NOT MATERIALIZED lets the planner see real row counts through the CTEs; + // materialized, it estimated one row each and nested-looped ~10k x 10k + // rows (90D: 55 s instead of 4 s) let activity_rows = query::( &conn, - "WITH tx_window AS ( + "WITH tx_window AS NOT MATERIALIZED ( SELECT t.federation_id, t.txid FROM transactions t JOIN session_times st @@ -427,7 +557,7 @@ impl FederationObserver { WHERE t.federation_id = $1 AND st.estimated_session_timestamp >= $2 ), - window_ln_outputs AS ( + window_ln_outputs AS NOT MATERIALIZED ( SELECT o.federation_id, o.txid, @@ -443,7 +573,7 @@ impl FederationObserver { AND o.kind = 'ln' AND o.ln_contract_id IS NOT NULL ), - window_ln_inputs AS ( + window_ln_inputs AS NOT MATERIALIZED ( SELECT i.federation_id, i.txid, @@ -456,7 +586,7 @@ impl FederationObserver { AND i.kind = 'ln' AND i.ln_contract_id IS NOT NULL ), - contract_map AS ( + contract_map AS NOT MATERIALIZED ( SELECT DISTINCT ON (wlo.federation_id, wlo.ln_contract_id) wlo.federation_id, wlo.ln_contract_id, @@ -539,14 +669,20 @@ impl FederationObserver { let uptime_rows = query::( &conn, "SELECT - gateway_id, - COUNT(*) FILTER (WHERE is_seen)::bigint AS seen_samples, + s.gateway_id, + COUNT(*) FILTER (WHERE s.is_seen)::bigint AS seen_samples, COUNT(*)::bigint AS total_samples - FROM gateway_poll_snapshots - WHERE federation_id = $1 - AND poll_time >= $2 - GROUP BY gateway_id", - &[&federation_id_bytes, &window_start_utc], + FROM gateway_poll_snapshots s + JOIN gateways g ON g.federation_id = s.federation_id AND g.gateway_id = s.gateway_id + WHERE s.federation_id = $1 + AND s.poll_time >= $2 + AND s.poll_time <= g.last_seen + make_interval(days => $3) + GROUP BY s.gateway_id", + &[ + &federation_id_bytes, + &window_start_utc, + &GATEWAY_RETIRED_AFTER_DAYS, + ], ) .await?; @@ -612,7 +748,7 @@ impl FederationObserver { async fn federation_gateway_uptime_trend( &self, federation_id: FederationId, - window: GatewayMetricsWindow, + window_start: DateTime, ) -> anyhow::Result> { #[derive(postgres_from_row::FromRow)] struct TrendRow { @@ -623,19 +759,24 @@ impl FederationObserver { let conn = self.connection().await?; let federation_id_bytes = federation_id.consensus_encode_to_vec(); - let window_start = Utc::now() - window.duration(); let rows = query::( &conn, "SELECT - date_trunc('day', poll_time) AS day, - COUNT(*) FILTER (WHERE is_seen)::bigint AS seen_samples, + date_trunc('day', s.poll_time) AS day, + COUNT(*) FILTER (WHERE s.is_seen)::bigint AS seen_samples, COUNT(*)::bigint AS total_samples - FROM gateway_poll_snapshots - WHERE federation_id = $1 - AND poll_time >= $2 - GROUP BY date_trunc('day', poll_time) + FROM gateway_poll_snapshots s + JOIN gateways g ON g.federation_id = s.federation_id AND g.gateway_id = s.gateway_id + WHERE s.federation_id = $1 + AND s.poll_time >= $2 + AND s.poll_time <= g.last_seen + make_interval(days => $3) + GROUP BY date_trunc('day', s.poll_time) ORDER BY day ASC", - &[&federation_id_bytes, &window_start], + &[ + &federation_id_bytes, + &window_start, + &GATEWAY_RETIRED_AFTER_DAYS, + ], ) .await?; @@ -660,28 +801,44 @@ impl FederationObserver { } } -pub(super) async fn get_federation_gateways( - Path(federation_id): Path, - Query(params): Query, - State(state): State, -) -> crate::error::Result>> { - let window = GatewayMetricsWindow::parse(params.window.as_deref())?; - Ok(state - .federation_observer - .list_federation_gateways(federation_id, window) - .await? - .into()) +/// Which parts of the overview to return, as (gateways, trend) +fn parse_overview_include(include: Option<&str>) -> anyhow::Result<(bool, bool)> { + let Some(include) = include else { + return Ok((true, true)); + }; + let (mut gateways, mut trend) = (false, false); + for part in include.split(',').map(str::trim) { + match part { + "gateways" => gateways = true, + "trend" => trend = true, + invalid => { + bail!("Invalid include '{invalid}'. Supported values: gateways, trend") + } + } + } + Ok((gateways, trend)) } -pub(super) async fn get_federation_gateway_uptime_trend( +pub(super) async fn get_federation_gateway_overview( Path(federation_id): Path, Query(params): Query, State(state): State, -) -> crate::error::Result>> { +) -> crate::error::Result> { let window = GatewayMetricsWindow::parse(params.window.as_deref())?; - Ok(state + let (include_gateways, include_trend) = parse_overview_include(params.include.as_deref())?; + let overview = state .federation_observer - .federation_gateway_uptime_trend(federation_id, window) - .await? - .into()) + .gateway_overview(federation_id, window) + .await?; + // Only the requested parts are copied out of the shared cached overview + Ok(Json(GatewayOverview { + window: overview.window.clone(), + computed_at: overview.computed_at, + gateways: include_gateways + .then(|| overview.gateways.clone()) + .flatten(), + uptime_trend: include_trend + .then(|| overview.uptime_trend.clone()) + .flatten(), + })) } diff --git a/fmo_server/src/federation/mod.rs b/fmo_server/src/federation/mod.rs index c728eb4..44b6215 100644 --- a/fmo_server/src/federation/mod.rs +++ b/fmo_server/src/federation/mod.rs @@ -20,7 +20,7 @@ use fmo_api_types::{FederationSummary, FedimintTotals, NonceSpendInfo, NoncesReq use serde::Deserialize; use serde_json::json; -use crate::federation::gateways::{get_federation_gateway_uptime_trend, get_federation_gateways}; +use crate::federation::gateways::get_federation_gateway_overview; use crate::federation::guardians::get_federation_health; use crate::federation::meta::get_federation_meta; use crate::federation::session::{count_sessions, list_sessions}; @@ -57,10 +57,9 @@ pub fn get_federations_routes() -> Router { "/:federation_id/transactions/histogram", get(transaction_histogram), ) - .route("/:federation_id/gateways", get(get_federation_gateways)) .route( - "/:federation_id/gateways/uptime-trend", - get(get_federation_gateway_uptime_trend), + "/:federation_id/gateways/overview", + get(get_federation_gateway_overview), ) .route("/:federation_id/utxos", get(get_federation_utxos)) .route("/:federation_id/sessions", get(list_sessions)) diff --git a/fmo_server/src/federation/observer.rs b/fmo_server/src/federation/observer.rs index a38978d..0addb55 100644 --- a/fmo_server/src/federation/observer.rs +++ b/fmo_server/src/federation/observer.rs @@ -41,6 +41,7 @@ use tracing::{debug, error, info_span, warn, Instrument}; use crate::config::meta::{ConsensusMetaCache, MetaFieldsExt}; use crate::db::DbMigration; use crate::federation::db::Federation; +use crate::federation::gateways::GatewayOverviewCache; use crate::federation::{db, decoders_from_config, instance_to_kind}; use crate::util::{execute, query, query_one, query_opt, query_value}; use crate::{migration, migration_backfill, schema_setup}; @@ -53,6 +54,7 @@ pub struct FederationObserver { task_group: TaskGroup, consensus_meta_cache: ConsensusMetaCache, connectors: ConnectorRegistry, + gateway_overviews: GatewayOverviewCache, } impl FederationObserver { @@ -78,6 +80,7 @@ impl FederationObserver { task_group: Default::default(), consensus_meta_cache: Default::default(), connectors, + gateway_overviews: Default::default(), }; slf.setup_schema().await?; @@ -92,6 +95,10 @@ impl FederationObserver { .spawn_cancellable("sync nostr events", Self::sync_nostr_events(slf.clone())); slf.task_group .spawn_cancellable("refresh views", Self::refresh_views(slf.clone())); + slf.task_group.spawn_cancellable( + "refresh gateway overviews", + Self::refresh_gateway_overviews(slf.clone()), + ); Ok(slf) } @@ -100,6 +107,10 @@ impl FederationObserver { &self.connectors } + pub(crate) fn gateway_overviews(&self) -> &GatewayOverviewCache { + &self.gateway_overviews + } + async fn spawn_observer(&self, federation: Federation) { let slf = self.clone();