From e252495332783ff373d5f5d531accd259f81b4ce Mon Sep 17 00:00:00 2001 From: Jesper van den Munckhof Date: Sat, 18 Jul 2026 17:29:12 -0700 Subject: [PATCH 1/2] Add server hibernation --- app/(app)/[id]/actions/hibernate.ts | 55 ++++++++ app/(app)/[id]/actions/wake.ts | 49 +++++++ app/(app)/[id]/components/shell.tsx | 72 ++++++++++ lib/providers/hetzner/servers.ts | 10 ++ lib/providers/types.ts | 1 + lib/workflows/hibernate-server.ts | 81 +++++++++++ lib/workflows/steps.ts | 206 +++++++++++++++++++++++++++- lib/workflows/teardown-server.ts | 2 + lib/workflows/wake-server.ts | 94 +++++++++++++ prisma/schema.prisma | 54 ++++---- protocol/constants.ts | 13 +- test/protocol-constants.test.ts | 7 +- 12 files changed, 617 insertions(+), 27 deletions(-) create mode 100644 app/(app)/[id]/actions/hibernate.ts create mode 100644 app/(app)/[id]/actions/wake.ts create mode 100644 lib/workflows/hibernate-server.ts create mode 100644 lib/workflows/wake-server.ts diff --git a/app/(app)/[id]/actions/hibernate.ts b/app/(app)/[id]/actions/hibernate.ts new file mode 100644 index 00000000..2eb95833 --- /dev/null +++ b/app/(app)/[id]/actions/hibernate.ts @@ -0,0 +1,55 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { start } from "workflow/api"; +import { z } from "zod"; + +import { prisma } from "@/lib/db"; +import { requireUser } from "@/lib/session"; +import { hibernateServer } from "@/lib/workflows/hibernate-server"; + +const inputSchema = z.object({ serverId: z.string().min(1) }); + +export type HibernateServerInput = z.infer; + +export type HibernateServerResult = { ok: true } | { error: string; ok: false }; + +export const hibernate = async ( + input: HibernateServerInput +): Promise => { + await requireUser(); + + const parsed = inputSchema.safeParse(input); + if (!parsed.success) { + return { error: "Invalid input", ok: false }; + } + + const server = await prisma.server.findFirst({ + where: { deletedAt: null, id: parsed.data.serverId }, + }); + if (!server) { + return { error: "Not found", ok: false }; + } + if (!server.providerServerId) { + return { error: "Server is not provisioned yet", ok: false }; + } + if ( + server.observedState !== "running" && + server.observedState !== "stopped" + ) { + return { + error: "Server must be running or stopped to hibernate", + ok: false, + }; + } + + await prisma.server.update({ + data: { desiredState: "hibernated", observedState: "hibernating" }, + where: { id: parsed.data.serverId }, + }); + + await start(hibernateServer, [{ serverId: parsed.data.serverId }]); + + revalidatePath("/", "layout"); + return { ok: true }; +}; diff --git a/app/(app)/[id]/actions/wake.ts b/app/(app)/[id]/actions/wake.ts new file mode 100644 index 00000000..191d2036 --- /dev/null +++ b/app/(app)/[id]/actions/wake.ts @@ -0,0 +1,49 @@ +"use server"; + +import { revalidatePath } from "next/cache"; +import { start } from "workflow/api"; +import { z } from "zod"; + +import { prisma } from "@/lib/db"; +import { requireUser } from "@/lib/session"; +import { wakeServer } from "@/lib/workflows/wake-server"; + +const inputSchema = z.object({ serverId: z.string().min(1) }); + +export type WakeServerInput = z.infer; + +export type WakeServerResult = { ok: true } | { error: string; ok: false }; + +export const wake = async ( + input: WakeServerInput +): Promise => { + await requireUser(); + + const parsed = inputSchema.safeParse(input); + if (!parsed.success) { + return { error: "Invalid input", ok: false }; + } + + const server = await prisma.server.findFirst({ + where: { deletedAt: null, id: parsed.data.serverId }, + }); + if (!server) { + return { error: "Not found", ok: false }; + } + if (server.observedState !== "hibernated") { + return { error: "Server is not hibernated", ok: false }; + } + if (!server.hibernationImageId) { + return { error: "No hibernation snapshot found", ok: false }; + } + + await prisma.server.update({ + data: { desiredState: "running", observedState: "waking" }, + where: { id: parsed.data.serverId }, + }); + + await start(wakeServer, [{ serverId: parsed.data.serverId }]); + + revalidatePath("/", "layout"); + return { ok: true }; +}; diff --git a/app/(app)/[id]/components/shell.tsx b/app/(app)/[id]/components/shell.tsx index a063c8e5..9a003956 100644 --- a/app/(app)/[id]/components/shell.tsx +++ b/app/(app)/[id]/components/shell.tsx @@ -1,9 +1,11 @@ "use client"; import { + MoonIcon, MoreHorizontalIcon, PlayIcon, RotateCcwIcon, SquareIcon, + SunIcon, Trash2Icon, } from "lucide-react"; import Image from "next/image"; @@ -40,6 +42,8 @@ import { cn } from "@/lib/utils"; import { PageBody, PageHeader } from "../../components/page-header"; import { runServerCommand } from "../actions/commands"; import { deleteServer } from "../actions/delete"; +import { hibernate } from "../actions/hibernate"; +import { wake } from "../actions/wake"; import { ProvisioningStatus } from "./provisioning-status"; import { ServerProvider } from "./server-context"; import type { ServerView } from "./server-context"; @@ -53,6 +57,8 @@ const PROVISIONING_PHASES = new Set([ "starting", "healthy", "errored", + "hibernating", + "waking", ]); const TABS = [ @@ -109,6 +115,7 @@ export const ServerShell = ({ const [server, setServer] = useState(initial); const [pending, setPending] = useState(null); const [deleteOpen, setDeleteOpen] = useState(false); + const [hibernateOpen, setHibernateOpen] = useState(false); useEffect(() => { const t = setInterval(async () => { @@ -159,6 +166,29 @@ export const ServerShell = ({ } }; + const runHibernate = async () => { + setPending("HIBERNATE"); + const result = await hibernate({ serverId: server.id }); + setPending(null); + if (result.ok) { + toast.success("Hibernating server"); + setHibernateOpen(false); + } else { + toast.error(result.error); + } + }; + + const runWake = async () => { + setPending("WAKE"); + const result = await wake({ serverId: server.id }); + setPending(null); + if (result.ok) { + toast.success("Waking server"); + } else { + toast.error(result.error); + } + }; + const updateServer = (patch: Partial) => setServer((prev) => ({ ...prev, ...patch })); @@ -170,6 +200,7 @@ export const ServerShell = ({ } const deleting = server.desiredState === "deleted"; + const hibernated = server.observedState === "hibernated"; const statusLabel = deleting ? "deleting" : server.observedState; const meta = ( @@ -214,6 +245,25 @@ export const ServerShell = ({ Restart + {hibernated ? ( + + + Wake + + ) : ( + setHibernateOpen(true)} + > + + Hibernate + + )} + setDeleteOpen(true)} @@ -244,6 +294,27 @@ export const ServerShell = ({ ); + const hibernateDialog = ( + + + + Hibernate this server? + + We'll snapshot the disk and release the VM to stop Hetzner + charges. Restoring takes a few minutes, and the public IP will + change on wake. + + + + Cancel + + Hibernate + + + + + ); + const gameIcon = ( {game.name} {deleteDialog} + {hibernateDialog} ); } diff --git a/lib/providers/hetzner/servers.ts b/lib/providers/hetzner/servers.ts index 4ddc248c..c48cfc23 100644 --- a/lib/providers/hetzner/servers.ts +++ b/lib/providers/hetzner/servers.ts @@ -140,4 +140,14 @@ export const createServerOps = (client: HetznerClient) => ({ throwIfHetznerError(error, response); } }, + + shutdownServer: async (id: ServerResourceId): Promise => { + const { error, response } = await client.POST( + "/servers/{id}/actions/shutdown", + { params: { path: { id: Number(id) } } } + ); + if (!response.ok && response.status !== 422) { + throwIfHetznerError(error, response); + } + }, }); diff --git a/lib/providers/types.ts b/lib/providers/types.ts index 34e36359..fa42be44 100644 --- a/lib/providers/types.ts +++ b/lib/providers/types.ts @@ -100,6 +100,7 @@ export interface Provider { createServer: (input: CreateServerInput) => Promise; getServer: (id: ServerResourceId) => Promise; deleteServer: (id: ServerResourceId) => Promise<{ deleted: boolean }>; + shutdownServer: (id: ServerResourceId) => Promise; rescaleServer: (id: ServerResourceId, serverType: string) => Promise; getMetrics: ( id: ServerResourceId, diff --git a/lib/workflows/hibernate-server.ts b/lib/workflows/hibernate-server.ts new file mode 100644 index 00000000..ef7044dc --- /dev/null +++ b/lib/workflows/hibernate-server.ts @@ -0,0 +1,81 @@ +import { FatalError, sleep } from "workflow"; + +import { + stepCreateHibernationSnapshot, + stepDeleteHibernationSnapshot, + stepDeleteProviderServer, + stepDeleteProviderServerForHibernation, + stepGetSnapshotStatus, + stepMarkFailed, + stepReadDesiredState, + stepReadPhase, + stepSendStopCommand, + stepShutdownProviderServer, +} from "./steps"; + +const MAX_STOP_DRAIN_SECONDS = 60; +const STOP_DRAIN_POLL_SECONDS = 3; +const MAX_SNAPSHOT_WAIT_SECONDS = 1800; +const SNAPSHOT_POLL_SECONDS = 10; + +const isCancelled = async (serverId: string): Promise => + (await stepReadDesiredState(serverId)) !== "hibernated"; + +export const hibernateServer = async (input: { serverId: string }) => { + "use workflow"; + + const { serverId } = input; + + try { + if (await isCancelled(serverId)) { + return; + } + + const { hadAgent } = await stepSendStopCommand(serverId); + if (hadAgent) { + const deadline = Date.now() + MAX_STOP_DRAIN_SECONDS * 1000; + while (Date.now() < deadline) { + const phase = await stepReadPhase(serverId); + if (phase === "stopped" || phase === "errored") { + break; + } + await sleep(`${STOP_DRAIN_POLL_SECONDS}s`); + } + } + + if (await isCancelled(serverId)) { + return; + } + + await stepShutdownProviderServer(serverId); + const { imageId } = await stepCreateHibernationSnapshot(serverId); + + const deadline = Date.now() + MAX_SNAPSHOT_WAIT_SECONDS * 1000; + let ready = false; + while (Date.now() < deadline) { + const { status } = await stepGetSnapshotStatus({ imageId, serverId }); + if (status === "available") { + ready = true; + break; + } + if (status === "unavailable" || status === "unknown") { + throw new FatalError(`Snapshot entered ${status} state`); + } + await sleep(`${SNAPSHOT_POLL_SECONDS}s`); + } + if (!ready) { + await stepDeleteHibernationSnapshot(serverId); + await stepDeleteProviderServer(serverId); + throw new FatalError("Snapshot did not become available in time"); + } + + await stepDeleteProviderServerForHibernation(serverId); + } catch (error) { + const reason = error instanceof Error ? error.message : "Unknown error"; + await stepMarkFailed({ reason, serverId }); + if (error instanceof FatalError) { + return; + } + throw error; + } +}; diff --git a/lib/workflows/steps.ts b/lib/workflows/steps.ts index 270406ad..24e45576 100644 --- a/lib/workflows/steps.ts +++ b/lib/workflows/steps.ts @@ -409,7 +409,211 @@ export const stepReadDesiredState = async (serverId: string) => { if (!server) { return "deleted" as const; } - return server.desiredState as "running" | "stopped" | "deleted"; + return server.desiredState as + | "running" + | "stopped" + | "hibernated" + | "deleted"; }; export type WaitPhaseTarget = Phase | Phase[]; + +export const stepSendStopCommand = async (serverId: string) => { + "use step"; + const { stepId } = getStepMetadata(); + const agent = await prisma.agent.findUnique({ where: { serverId } }); + if (!agent) { + return { hadAgent: false }; + } + const lastHeartbeat = agent.lastHeartbeatAt?.getTime() ?? 0; + if (Date.now() - lastHeartbeat > AGENT_LIVENESS_WINDOW_MS) { + return { hadAgent: false }; + } + await enqueueCommand({ + idempotencyKey: stepId, + payload: {}, + serverId, + type: "STOP", + }); + await emitActivity({ + message: "Stopping game for hibernation", + phase: "hibernating", + serverId, + }); + return { hadAgent: true }; +}; + +export const stepShutdownProviderServer = async (serverId: string) => { + "use step"; + const server = await prisma.server.findUnique({ where: { id: serverId } }); + if (!server?.providerServerId) { + return { ok: false as const }; + } + await getProvider().shutdownServer(server.providerServerId); + return { ok: true as const }; +}; + +export const stepCreateHibernationSnapshot = async (serverId: string) => { + "use step"; + const server = await prisma.server.findUnique({ where: { id: serverId } }); + if (!server?.providerServerId) { + throw new FatalError("Cannot snapshot a server with no provider VM"); + } + if (server.hibernationImageId) { + return { imageId: server.hibernationImageId }; + } + const imageId = await getProvider().createSnapshot(server.providerServerId, { + description: `ghost-hibernation-${serverId}`, + }); + await prisma.server.update({ + data: { hibernationImageId: imageId }, + where: { id: serverId }, + }); + await emitActivity({ + message: "Creating snapshot", + metadata: { imageId }, + phase: "hibernating", + serverId, + }); + return { imageId }; +}; + +export const stepGetSnapshotStatus = async (input: { + serverId: string; + imageId: string; +}) => { + "use step"; + const image = await getProvider().getImage(input.imageId); + if (!image) { + return { status: "unknown" as const }; + } + return { status: image.status }; +}; + +export const stepDeleteProviderServerForHibernation = async ( + serverId: string +) => { + "use step"; + const server = await prisma.server.findUnique({ where: { id: serverId } }); + if (!server?.providerServerId) { + return { deleted: false }; + } + await getProvider().deleteServer(server.providerServerId); + await prisma.server.update({ + data: { + hibernatedAt: new Date(), + ipv4: null, + observedState: "hibernated", + phase: "hibernated", + providerServerId: null, + }, + where: { id: serverId }, + }); + await emitActivity({ + message: "VM released; snapshot retained", + phase: "hibernated", + serverId, + }); + return { deleted: true }; +}; + +export const stepCreateProviderServerFromSnapshot = async ( + serverId: string +) => { + "use step"; + const server = await prisma.server.findUnique({ where: { id: serverId } }); + if (!server || server.desiredState !== "running") { + return { cancelled: true as const }; + } + if (server.providerServerId) { + return { + cancelled: false as const, + providerServerId: server.providerServerId, + }; + } + if (!server.hibernationImageId) { + throw new FatalError("Cannot wake server without a hibernation snapshot"); + } + const provider = getProvider(); + + const name = `ghost-${serverId.toLowerCase().slice(-12)}-${crypto + .randomBytes(2) + .toString("hex")}`; + + let created: Awaited>; + try { + created = await provider.createServer({ + imageId: server.hibernationImageId, + location: server.location, + name, + serverType: server.serverType, + userData: "", + }); + } catch (error) { + if (error instanceof ProviderApiError && error.isClientError) { + throw new FatalError(error.message); + } + throw error; + } + + await prisma.server.update({ + data: { + ipv4: created.ipv4, + observedState: "waking", + phase: "waking", + providerServerId: created.id, + }, + where: { id: serverId }, + }); + await emitActivity({ + message: "Restoring VM from snapshot", + metadata: { providerServerId: created.id }, + phase: "waking", + serverId, + }); + return { cancelled: false as const, providerServerId: created.id }; +}; + +export const stepWaitAgentReconnected = async (serverId: string) => { + "use step"; + const agent = await prisma.agent.findUnique({ + select: { lastHeartbeatAt: true }, + where: { serverId }, + }); + if (!agent?.lastHeartbeatAt) { + return { reconnected: false }; + } + const reconnected = + Date.now() - agent.lastHeartbeatAt.getTime() < AGENT_LIVENESS_WINDOW_MS; + return { reconnected }; +}; + +export const stepDeleteHibernationSnapshot = async (serverId: string) => { + "use step"; + const server = await prisma.server.findUnique({ where: { id: serverId } }); + if (!server?.hibernationImageId) { + return { deleted: false }; + } + await getProvider().deleteImage(server.hibernationImageId); + await prisma.server.update({ + data: { hibernatedAt: null, hibernationImageId: null }, + where: { id: serverId }, + }); + return { deleted: true }; +}; + +export const stepMarkAwake = async (serverId: string) => { + "use step"; + const { count } = await prisma.server.updateMany({ + data: { observedState: "running", phase: "ready" }, + where: { id: serverId }, + }); + if (count === 0) { + return; + } + await emitActivity({ + message: "Server back online", + phase: "ready", + serverId, + }); +}; diff --git a/lib/workflows/teardown-server.ts b/lib/workflows/teardown-server.ts index 21eb013b..7d53437d 100644 --- a/lib/workflows/teardown-server.ts +++ b/lib/workflows/teardown-server.ts @@ -1,6 +1,7 @@ import { sleep } from "workflow"; import { + stepDeleteHibernationSnapshot, stepDeleteProviderServer, stepMarkDeleted, stepReadPhase, @@ -29,5 +30,6 @@ export const teardownServer = async (input: { serverId: string }) => { } await stepDeleteProviderServer(serverId); + await stepDeleteHibernationSnapshot(serverId); await stepMarkDeleted(serverId); }; diff --git a/lib/workflows/wake-server.ts b/lib/workflows/wake-server.ts new file mode 100644 index 00000000..7406a80a --- /dev/null +++ b/lib/workflows/wake-server.ts @@ -0,0 +1,94 @@ +import { FatalError, sleep } from "workflow"; + +import { + stepCreateProviderServerFromSnapshot, + stepDeleteHibernationSnapshot, + stepGetServerStatus, + stepMarkAwake, + stepMarkFailed, + stepMarkServerRunning, + stepReadDesiredState, + stepWaitAgentReconnected, +} from "./steps"; + +const MAX_PROVIDER_WAIT_SECONDS = 300; +const PROVIDER_POLL_SECONDS = 6; +const MAX_RECONNECT_WAIT_SECONDS = 300; +const RECONNECT_POLL_SECONDS = 6; + +const isCancelled = async (serverId: string): Promise => + (await stepReadDesiredState(serverId)) !== "running"; + +export const wakeServer = async (input: { serverId: string }) => { + "use workflow"; + + const { serverId } = input; + + try { + if (await isCancelled(serverId)) { + return; + } + + const created = await stepCreateProviderServerFromSnapshot(serverId); + if (created.cancelled || !("providerServerId" in created)) { + return; + } + const { providerServerId } = created; + + const providerDeadline = Date.now() + MAX_PROVIDER_WAIT_SECONDS * 1000; + let runningIp: string | null = null; + let booted = false; + while (Date.now() < providerDeadline) { + const status = await stepGetServerStatus({ providerServerId, serverId }); + if (status.status === "running") { + runningIp = status.ip; + booted = true; + break; + } + if (status.status === "unknown") { + throw new FatalError("Provider server vanished after wake"); + } + if (await isCancelled(serverId)) { + return; + } + await sleep(`${PROVIDER_POLL_SECONDS}s`); + } + if (!booted) { + await stepMarkFailed({ reason: "Wake boot timeout", serverId }); + return; + } + + await stepMarkServerRunning({ ipv4: runningIp, serverId }); + + const reconnectDeadline = Date.now() + MAX_RECONNECT_WAIT_SECONDS * 1000; + let reconnected = false; + while (Date.now() < reconnectDeadline) { + const result = await stepWaitAgentReconnected(serverId); + if (result.reconnected) { + reconnected = true; + break; + } + if (await isCancelled(serverId)) { + return; + } + await sleep(`${RECONNECT_POLL_SECONDS}s`); + } + if (!reconnected) { + await stepMarkFailed({ + reason: "Agent did not reconnect after wake", + serverId, + }); + return; + } + + await stepDeleteHibernationSnapshot(serverId); + await stepMarkAwake(serverId); + } catch (error) { + const reason = error instanceof Error ? error.message : "Unknown error"; + await stepMarkFailed({ reason, serverId }); + if (error instanceof FatalError) { + return; + } + throw error; + } +}; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 5aca97c0..6c5a595e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -26,6 +26,7 @@ model Snapshot { enum DesiredState { running stopped + hibernated deleted } @@ -36,35 +37,40 @@ enum ObservedState { unhealthy lost stopped + hibernating + hibernated + waking failed deleted } model Server { - id String @id - name String - game String - location String - serverType String - provider String @default("hetzner") - providerServerId String? @unique - ipv4 String? - desiredState DesiredState @default(running) - observedState ObservedState @default(pending) - phase String @default("queued") - errorReason String? - rconPassword String - joinPassword String? - settings Json @default("{}") - backupsEnabled Boolean @default(false) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - deletedAt DateTime? - agent Agent? - commands Command[] - activityEvents ActivityEvent[] - logChunks LogChunk[] - enrollments AgentEnrollment[] + id String @id + name String + game String + location String + serverType String + provider String @default("hetzner") + providerServerId String? @unique + ipv4 String? + desiredState DesiredState @default(running) + observedState ObservedState @default(pending) + phase String @default("queued") + errorReason String? + rconPassword String + joinPassword String? + settings Json @default("{}") + backupsEnabled Boolean @default(false) + hibernationImageId String? + hibernatedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + deletedAt DateTime? + agent Agent? + commands Command[] + activityEvents ActivityEvent[] + logChunks LogChunk[] + enrollments AgentEnrollment[] @@index([desiredState, observedState]) @@map("servers") diff --git a/protocol/constants.ts b/protocol/constants.ts index 24327634..e7586d7f 100644 --- a/protocol/constants.ts +++ b/protocol/constants.ts @@ -9,13 +9,21 @@ export const PHASES = [ "ready", "stopping", "stopped", + "hibernating", + "hibernated", + "waking", "errored", "deleting", "deleted", ] as const; export type Phase = (typeof PHASES)[number]; -export const DESIRED_STATES = ["running", "stopped", "deleted"] as const; +export const DESIRED_STATES = [ + "running", + "stopped", + "hibernated", + "deleted", +] as const; export type DesiredState = (typeof DESIRED_STATES)[number]; export const OBSERVED_STATES = [ @@ -25,6 +33,9 @@ export const OBSERVED_STATES = [ "unhealthy", "lost", "stopped", + "hibernating", + "hibernated", + "waking", "failed", "deleted", ] as const; diff --git a/test/protocol-constants.test.ts b/test/protocol-constants.test.ts index c9605d9b..9624df81 100644 --- a/test/protocol-constants.test.ts +++ b/test/protocol-constants.test.ts @@ -48,7 +48,12 @@ describe("protocol enums", () => { }); test("DESIRED_STATES is the small fixed set", () => { - expect([...DESIRED_STATES]).toEqual(["running", "stopped", "deleted"]); + expect([...DESIRED_STATES]).toEqual([ + "running", + "stopped", + "hibernated", + "deleted", + ]); }); test("OBSERVED_STATES, COMMAND_TYPES, COMMAND_STATUSES, LOG_STREAMS are non-empty", () => { From 07b5521da2865591987e9cc8d80749be3da6fe92 Mon Sep 17 00:00:00 2001 From: Hayden Bleasel Date: Sat, 18 Jul 2026 17:39:56 -0700 Subject: [PATCH 2/2] Harden hibernate/wake lifecycle - Never tear down the VM (or snapshot) when the snapshot polls time out; the VM holds the only copy of the data, so timeouts now just mark the server failed and leave everything for a retry to reuse - Wait for the VM to actually power off (ACPI, then hard poweroff) before snapshotting, instead of imaging a running disk - Atomically claim hibernate/wake state transitions with a guarded updateMany so double-clicks can't fan out duplicate workflows, and roll the row back if the workflow fails to start - Allow retrying from "failed": a failed hibernation can be re-hibernated and a failed wake re-woken (the workflow reuses a surviving VM), so timeouts are no longer dead-end states - Distinguish a snapshot that vanished ("missing") from a bad status, and clear the dangling image id so retries start fresh - Clear providerServerId when the provider VM 404s during wake so a retry recreates it from the snapshot - Reset the pending UI state in a finally block so a thrown action can't permanently disable the actions menu - Drop hibernating/waking from PROVISIONING_PHASES: the provisioning checklist can't represent them and would pin to "Create server" - Add the Prisma migration for the new enum values and columns --- app/(app)/[id]/actions/hibernate.ts | 52 +++++++++++---- app/(app)/[id]/actions/wake.ts | 46 ++++++++++--- app/(app)/[id]/components/server-context.tsx | 1 + app/(app)/[id]/components/shell.tsx | 54 +++++++++------- app/(app)/[id]/layout.tsx | 1 + lib/providers/hetzner/servers.ts | 10 +++ lib/providers/types.ts | 1 + lib/workflows/hibernate-server.ts | 64 +++++++++++++++++-- lib/workflows/steps.ts | 25 +++++++- lib/workflows/wake-server.ts | 2 + .../migration.sql | 11 ++++ 11 files changed, 220 insertions(+), 47 deletions(-) create mode 100644 prisma/migrations/20260718173919_add_hibernation/migration.sql diff --git a/app/(app)/[id]/actions/hibernate.ts b/app/(app)/[id]/actions/hibernate.ts index 2eb95833..c8186afe 100644 --- a/app/(app)/[id]/actions/hibernate.ts +++ b/app/(app)/[id]/actions/hibernate.ts @@ -23,9 +23,10 @@ export const hibernate = async ( if (!parsed.success) { return { error: "Invalid input", ok: false }; } + const { serverId } = parsed.data; const server = await prisma.server.findFirst({ - where: { deletedAt: null, id: parsed.data.serverId }, + where: { deletedAt: null, id: serverId }, }); if (!server) { return { error: "Not found", ok: false }; @@ -33,22 +34,51 @@ export const hibernate = async ( if (!server.providerServerId) { return { error: "Server is not provisioned yet", ok: false }; } - if ( - server.observedState !== "running" && - server.observedState !== "stopped" - ) { + + // Atomically claim the transition so a double-click can't start two + // workflows. "failed" is only claimable when the failure came from a + // hibernation attempt (desiredState is still "hibernated"), never from a + // failed provision. + const { count } = await prisma.server.updateMany({ + data: { + desiredState: "hibernated", + errorReason: null, + observedState: "hibernating", + phase: "hibernating", + }, + where: { + OR: [ + { observedState: { in: ["running", "stopped"] } }, + { desiredState: "hibernated", observedState: "failed" }, + ], + deletedAt: null, + id: serverId, + providerServerId: { not: null }, + }, + }); + if (count === 0) { return { error: "Server must be running or stopped to hibernate", ok: false, }; } - await prisma.server.update({ - data: { desiredState: "hibernated", observedState: "hibernating" }, - where: { id: parsed.data.serverId }, - }); - - await start(hibernateServer, [{ serverId: parsed.data.serverId }]); + try { + await start(hibernateServer, [{ serverId }]); + } catch { + // The workflow never started; put the row back so the server isn't + // stranded in "hibernating" with nothing driving it. + await prisma.server.updateMany({ + data: { + desiredState: server.desiredState, + errorReason: server.errorReason, + observedState: server.observedState, + phase: server.phase, + }, + where: { id: serverId, observedState: "hibernating" }, + }); + return { error: "Failed to start hibernation", ok: false }; + } revalidatePath("/", "layout"); return { ok: true }; diff --git a/app/(app)/[id]/actions/wake.ts b/app/(app)/[id]/actions/wake.ts index 191d2036..4421982c 100644 --- a/app/(app)/[id]/actions/wake.ts +++ b/app/(app)/[id]/actions/wake.ts @@ -23,26 +23,56 @@ export const wake = async ( if (!parsed.success) { return { error: "Invalid input", ok: false }; } + const { serverId } = parsed.data; const server = await prisma.server.findFirst({ - where: { deletedAt: null, id: parsed.data.serverId }, + where: { deletedAt: null, id: serverId }, }); if (!server) { return { error: "Not found", ok: false }; } - if (server.observedState !== "hibernated") { - return { error: "Server is not hibernated", ok: false }; - } if (!server.hibernationImageId) { return { error: "No hibernation snapshot found", ok: false }; } - await prisma.server.update({ - data: { desiredState: "running", observedState: "waking" }, - where: { id: parsed.data.serverId }, + // Atomically claim the transition so a double-click can't start two + // workflows (two concurrent wakes could each create a VM). "failed" is + // claimable so a wake that timed out can be retried; the workflow reuses + // the existing VM when one survived the failed attempt. + const { count } = await prisma.server.updateMany({ + data: { + desiredState: "running", + errorReason: null, + observedState: "waking", + phase: "waking", + }, + where: { + deletedAt: null, + hibernationImageId: { not: null }, + id: serverId, + observedState: { in: ["hibernated", "failed"] }, + }, }); + if (count === 0) { + return { error: "Server is not hibernated", ok: false }; + } - await start(wakeServer, [{ serverId: parsed.data.serverId }]); + try { + await start(wakeServer, [{ serverId }]); + } catch { + // The workflow never started; put the row back so the server isn't + // stranded in "waking" with nothing driving it. + await prisma.server.updateMany({ + data: { + desiredState: server.desiredState, + errorReason: server.errorReason, + observedState: server.observedState, + phase: server.phase, + }, + where: { id: serverId, observedState: "waking" }, + }); + return { error: "Failed to start wake", ok: false }; + } revalidatePath("/", "layout"); return { ok: true }; diff --git a/app/(app)/[id]/components/server-context.tsx b/app/(app)/[id]/components/server-context.tsx index 793225bc..74923021 100644 --- a/app/(app)/[id]/components/server-context.tsx +++ b/app/(app)/[id]/components/server-context.tsx @@ -33,6 +33,7 @@ export interface ServerView { lastHeartbeatAt: string | null; serverType: string; backupsEnabled: boolean; + hibernationImageId: string | null; specs: Specs | null; location: ServerLocation | null; settings: Record; diff --git a/app/(app)/[id]/components/shell.tsx b/app/(app)/[id]/components/shell.tsx index 9a003956..be8bdb97 100644 --- a/app/(app)/[id]/components/shell.tsx +++ b/app/(app)/[id]/components/shell.tsx @@ -57,8 +57,6 @@ const PROVISIONING_PHASES = new Set([ "starting", "healthy", "errored", - "hibernating", - "waking", ]); const TABS = [ @@ -131,6 +129,7 @@ export const ServerShell = ({ desiredState: fresh.desiredState, errorReason: fresh.errorReason ?? null, game: fresh.game, + hibernationImageId: fresh.hibernationImageId ?? null, id: fresh.id, ipv4: fresh.ipv4, lastHeartbeatAt: fresh.agent?.lastHeartbeatAt ?? null, @@ -168,24 +167,30 @@ export const ServerShell = ({ const runHibernate = async () => { setPending("HIBERNATE"); - const result = await hibernate({ serverId: server.id }); - setPending(null); - if (result.ok) { - toast.success("Hibernating server"); - setHibernateOpen(false); - } else { - toast.error(result.error); + try { + const result = await hibernate({ serverId: server.id }); + if (result.ok) { + toast.success("Hibernating server"); + setHibernateOpen(false); + } else { + toast.error(result.error); + } + } finally { + setPending(null); } }; const runWake = async () => { setPending("WAKE"); - const result = await wake({ serverId: server.id }); - setPending(null); - if (result.ok) { - toast.success("Waking server"); - } else { - toast.error(result.error); + try { + const result = await wake({ serverId: server.id }); + if (result.ok) { + toast.success("Waking server"); + } else { + toast.error(result.error); + } + } finally { + setPending(null); } }; @@ -200,7 +205,16 @@ export const ServerShell = ({ } const deleting = server.desiredState === "deleted"; - const hibernated = server.observedState === "hibernated"; + // "failed" is wake-able (or hibernate-able) again so a timed-out attempt + // isn't a dead end; which retry applies depends on which transition failed. + const canWake = + server.hibernationImageId !== null && + (server.observedState === "hibernated" || + (server.observedState === "failed" && server.desiredState === "running")); + const canHibernate = + server.observedState === "running" || + server.observedState === "stopped" || + (server.observedState === "failed" && server.desiredState === "hibernated"); const statusLabel = deleting ? "deleting" : server.observedState; const meta = ( @@ -245,18 +259,14 @@ export const ServerShell = ({ Restart - {hibernated ? ( + {canWake ? ( Wake ) : ( setHibernateOpen(true)} > diff --git a/app/(app)/[id]/layout.tsx b/app/(app)/[id]/layout.tsx index 075dd916..13917051 100644 --- a/app/(app)/[id]/layout.tsx +++ b/app/(app)/[id]/layout.tsx @@ -83,6 +83,7 @@ const ServerLayout = async ({ dockerImage: game?.dockerImage ?? null, errorReason: server.errorReason, game: server.game, + hibernationImageId: server.hibernationImageId, id: server.id, ipv4: server.ipv4, joinPassword: server.joinPassword, diff --git a/lib/providers/hetzner/servers.ts b/lib/providers/hetzner/servers.ts index c48cfc23..a22ab8e4 100644 --- a/lib/providers/hetzner/servers.ts +++ b/lib/providers/hetzner/servers.ts @@ -110,6 +110,16 @@ export const createServerOps = (client: HetznerClient) => ({ }; }, + poweroffServer: async (id: ServerResourceId): Promise => { + const { error, response } = await client.POST( + "/servers/{id}/actions/poweroff", + { params: { path: { id: Number(id) } } } + ); + if (!response.ok && response.status !== 422) { + throwIfHetznerError(error, response); + } + }, + rescaleServer: async ( id: ServerResourceId, serverType: string diff --git a/lib/providers/types.ts b/lib/providers/types.ts index fa42be44..1c188bbb 100644 --- a/lib/providers/types.ts +++ b/lib/providers/types.ts @@ -101,6 +101,7 @@ export interface Provider { getServer: (id: ServerResourceId) => Promise; deleteServer: (id: ServerResourceId) => Promise<{ deleted: boolean }>; shutdownServer: (id: ServerResourceId) => Promise; + poweroffServer: (id: ServerResourceId) => Promise; rescaleServer: (id: ServerResourceId, serverType: string) => Promise; getMetrics: ( id: ServerResourceId, diff --git a/lib/workflows/hibernate-server.ts b/lib/workflows/hibernate-server.ts index ef7044dc..dcfa12b8 100644 --- a/lib/workflows/hibernate-server.ts +++ b/lib/workflows/hibernate-server.ts @@ -3,10 +3,11 @@ import { FatalError, sleep } from "workflow"; import { stepCreateHibernationSnapshot, stepDeleteHibernationSnapshot, - stepDeleteProviderServer, stepDeleteProviderServerForHibernation, + stepGetServerStatus, stepGetSnapshotStatus, stepMarkFailed, + stepPoweroffProviderServer, stepReadDesiredState, stepReadPhase, stepSendStopCommand, @@ -15,12 +16,37 @@ import { const MAX_STOP_DRAIN_SECONDS = 60; const STOP_DRAIN_POLL_SECONDS = 3; +const MAX_SHUTDOWN_WAIT_SECONDS = 120; +const MAX_POWEROFF_WAIT_SECONDS = 60; +const POWER_POLL_SECONDS = 5; const MAX_SNAPSHOT_WAIT_SECONDS = 1800; const SNAPSHOT_POLL_SECONDS = 10; const isCancelled = async (serverId: string): Promise => (await stepReadDesiredState(serverId)) !== "hibernated"; +const waitForPowerOff = async ( + serverId: string, + providerServerId: string, + maxSeconds: number +): Promise => { + const deadline = Date.now() + maxSeconds * 1000; + while (Date.now() < deadline) { + const { status } = await stepGetServerStatus({ + providerServerId, + serverId, + }); + if (status === "off") { + return true; + } + if (status === "unknown") { + throw new FatalError("Provider VM vanished before snapshot"); + } + await sleep(`${POWER_POLL_SECONDS}s`); + } + return false; +}; + export const hibernateServer = async (input: { serverId: string }) => { "use workflow"; @@ -47,7 +73,31 @@ export const hibernateServer = async (input: { serverId: string }) => { return; } - await stepShutdownProviderServer(serverId); + const shutdown = await stepShutdownProviderServer(serverId); + if (!shutdown.ok) { + throw new FatalError("Server has no provider VM to hibernate"); + } + + // Snapshotting a running disk risks a corrupt world save: wait for the + // graceful shutdown to land, escalating to a hard poweroff if the guest + // ignores ACPI. + let off = await waitForPowerOff( + serverId, + shutdown.providerServerId, + MAX_SHUTDOWN_WAIT_SECONDS + ); + if (!off) { + await stepPoweroffProviderServer(serverId); + off = await waitForPowerOff( + serverId, + shutdown.providerServerId, + MAX_POWEROFF_WAIT_SECONDS + ); + } + if (!off) { + throw new FatalError("VM did not power off before snapshot"); + } + const { imageId } = await stepCreateHibernationSnapshot(serverId); const deadline = Date.now() + MAX_SNAPSHOT_WAIT_SECONDS * 1000; @@ -58,14 +108,18 @@ export const hibernateServer = async (input: { serverId: string }) => { ready = true; break; } - if (status === "unavailable" || status === "unknown") { + if (status === "unavailable" || status === "missing") { + // The snapshot is junk (or was deleted out from under us): clear it + // so a retry starts a fresh one. The VM stays untouched — it still + // holds the only copy of the data. + await stepDeleteHibernationSnapshot(serverId); throw new FatalError(`Snapshot entered ${status} state`); } await sleep(`${SNAPSHOT_POLL_SECONDS}s`); } if (!ready) { - await stepDeleteHibernationSnapshot(serverId); - await stepDeleteProviderServer(serverId); + // Never tear anything down on a timeout: the snapshot may yet complete + // and a retry will reuse it via the stored image id. throw new FatalError("Snapshot did not become available in time"); } diff --git a/lib/workflows/steps.ts b/lib/workflows/steps.ts index 24e45576..34a94c93 100644 --- a/lib/workflows/steps.ts +++ b/lib/workflows/steps.ts @@ -450,6 +450,16 @@ export const stepShutdownProviderServer = async (serverId: string) => { return { ok: false as const }; } await getProvider().shutdownServer(server.providerServerId); + return { ok: true as const, providerServerId: server.providerServerId }; +}; + +export const stepPoweroffProviderServer = async (serverId: string) => { + "use step"; + const server = await prisma.server.findUnique({ where: { id: serverId } }); + if (!server?.providerServerId) { + return { ok: false as const }; + } + await getProvider().poweroffServer(server.providerServerId); return { ok: true as const }; }; @@ -483,9 +493,12 @@ export const stepGetSnapshotStatus = async (input: { imageId: string; }) => { "use step"; + // "missing" (deleted out from under us) is distinct from a bad status so + // the workflow can clear the dangling image id; transient provider errors + // throw and retry the step instead of masquerading as either. const image = await getProvider().getImage(input.imageId); if (!image) { - return { status: "unknown" as const }; + return { status: "missing" as const }; } return { status: image.status }; }; @@ -574,6 +587,16 @@ export const stepCreateProviderServerFromSnapshot = async ( return { cancelled: false as const, providerServerId: created.id }; }; +export const stepClearVanishedProviderServer = async (serverId: string) => { + "use step"; + // The VM 404'd at the provider: drop the dangling reference so a wake + // retry recreates it from the snapshot instead of polling a ghost. + await prisma.server.updateMany({ + data: { ipv4: null, providerServerId: null }, + where: { id: serverId }, + }); +}; + export const stepWaitAgentReconnected = async (serverId: string) => { "use step"; const agent = await prisma.agent.findUnique({ diff --git a/lib/workflows/wake-server.ts b/lib/workflows/wake-server.ts index 7406a80a..d2f6a9f9 100644 --- a/lib/workflows/wake-server.ts +++ b/lib/workflows/wake-server.ts @@ -1,6 +1,7 @@ import { FatalError, sleep } from "workflow"; import { + stepClearVanishedProviderServer, stepCreateProviderServerFromSnapshot, stepDeleteHibernationSnapshot, stepGetServerStatus, @@ -46,6 +47,7 @@ export const wakeServer = async (input: { serverId: string }) => { break; } if (status.status === "unknown") { + await stepClearVanishedProviderServer(serverId); throw new FatalError("Provider server vanished after wake"); } if (await isCancelled(serverId)) { diff --git a/prisma/migrations/20260718173919_add_hibernation/migration.sql b/prisma/migrations/20260718173919_add_hibernation/migration.sql new file mode 100644 index 00000000..67430e57 --- /dev/null +++ b/prisma/migrations/20260718173919_add_hibernation/migration.sql @@ -0,0 +1,11 @@ +-- AlterEnum +ALTER TYPE "DesiredState" ADD VALUE 'hibernated' AFTER 'stopped'; + +-- AlterEnum +ALTER TYPE "ObservedState" ADD VALUE 'hibernating' AFTER 'stopped'; +ALTER TYPE "ObservedState" ADD VALUE 'hibernated' AFTER 'hibernating'; +ALTER TYPE "ObservedState" ADD VALUE 'waking' AFTER 'hibernated'; + +-- AlterTable +ALTER TABLE "servers" ADD COLUMN "hibernatedAt" TIMESTAMP(3), +ADD COLUMN "hibernationImageId" TEXT;