diff --git a/AGENTS.md b/AGENTS.md index f3fb859f..49b4feb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,7 @@ Prefer plain ASCII characters in Markdown unless a typographic character is genu - Reference is versioned; current version is v5 (`reference/`). Archived v4 content is in `reference_versioned_docs/version-v4/`. - Do not modify `reference_versioned_docs/` for current (v5) work; edit `reference/` instead. - The `` component is globally registered — no import needed in `.md`/`.mdx` files. +- The `` component (also global) tags storage-engine support inline: `` or ``. - See the complete repository organization in `CONTRIBUTING.md` ## Versioning Content diff --git a/reference/backups/operations.md b/reference/backups/operations.md new file mode 100644 index 00000000..7ac0bfdd --- /dev/null +++ b/reference/backups/operations.md @@ -0,0 +1,140 @@ +--- +id: operations +title: Backup Operations +--- + + + +Operations for backing up and restoring databases. For how the backup system works, its limitations, and worked examples, see the [Backups Overview](./overview.md). + +All backup operations require a `super_user` role when invoked through a running server; run offline (from the CLI with the server stopped), they are governed by filesystem permissions instead. All accept a `database` parameter that defaults to `data`. The engine badge on each operation indicates which storage engines support it: the managed-backup operations require RocksDB, while `get_backup` works with both RocksDB and LMDB. + +| Operation | Description | Role Required | +| ----------------------------------- | ------------------------------------------------------------------- | ------------- | +| [`create_backup`](#create_backup) | Creates a managed, incremental directory backup of a database (job) | super_user | +| [`list_backups`](#list_backups) | Lists the managed backups for a database | super_user | +| [`verify_backup`](#verify_backup) | Verifies a managed backup's integrity (job) | super_user | +| [`delete_backup`](#delete_backup) | Deletes a single managed backup | super_user | +| [`purge_backups`](#purge_backups) | Deletes all but the newest `keep_count` managed backups | super_user | +| [`restore_backup`](#restore_backup) | Restores a database from a managed backup (job) | super_user | +| [`get_backup`](#get_backup) | Streams a full snapshot of a database in the response for download | super_user | + +## Running backup operations from the CLI + +Every operation on this page can be run from the CLI under its operation name, for example `harper create_backup database=data`. When Harper is running, the CLI forwards the operation to the server; when it is stopped, the command operates directly on the database and backup files. `get_backup` is the exception — it streams from a running server and has no offline form. + +Like any [CLI operation](../cli/operations-api-commands.md), backup commands also accept a remote `target=` to run against another instance instead of the local one (see [Remote Operations](../cli/overview.md#remote-operations)). With a remote target the operation is always forwarded — there is no offline path. + +Offline invocation matters most for restore: an in-place `restore_backup` of the `system` database, or of any database a loaded component keeps open, must be run with the server stopped. See [when can a database be restored?](./overview.md#when-can-a-database-be-restored) + +## `create_backup` + + + +Creates an incremental directory backup of the database under the configured backup root ([`storage.backupPath`](../configuration/options.md#storage), default `/backup`). Through a running server this runs as a background [job](../operations-api/operations.md#jobs): the operation returns a `job_id` immediately, and [`get_job`](../operations-api/operations.md#get_job) reports the outcome including the new `backup_id`, `size`, and `timestamp`. + +Backups of the same database share unchanged files, so the second and subsequent backups only copy what changed. Use [`purge_backups`](#purge_backups) to manage retention. + +```json +{ "operation": "create_backup", "database": "data" } +``` + +```bash +harper create_backup database=data +``` + +## `list_backups` + + + +Returns the managed backups for a database, each with its `backup_id`, `timestamp`, `size`, and `file_count`. Returns an empty array if no backups have been created yet. + +```json +{ "operation": "list_backups", "database": "data" } +``` + +```bash +harper list_backups database=data +``` + +## `verify_backup` + + + +Verifies a managed backup's file sizes, and optionally their checksums when `verify_checksum` is `true` (slower). Through a running server this runs as a background [job](../operations-api/operations.md#jobs). `backup_id` is required. + +```json +{ "operation": "verify_backup", "database": "data", "backup_id": 1, "verify_checksum": true } +``` + +```bash +harper verify_backup database=data backup_id=1 verify_checksum=true +``` + +## `delete_backup` + + + +Deletes a single managed backup. Files shared with other backups are reference-counted and removed only when no remaining backup references them. `backup_id` is required. + +```json +{ "operation": "delete_backup", "database": "data", "backup_id": 1 } +``` + +```bash +harper delete_backup database=data backup_id=1 +``` + +## `purge_backups` + + + +Deletes all but the newest `keep_count` managed backups, returning the number `deleted` and the number `remaining`. + +```json +{ "operation": "purge_backups", "database": "data", "keep_count": 3 } +``` + +```bash +harper purge_backups database=data keep_count=3 +``` + +## `restore_backup` + + + +Restores a database in place from a managed backup. `backup_id` defaults to the latest backup. The audit/transaction log is restored alongside the data. + +Through a running server this runs as a background [job](../operations-api/operations.md#jobs): Harper closes the database across all worker threads, restores it, and reloads it. This works only when no loaded component is holding the database open — restoring the `system` database, or a database a component keeps open, requires running the command from the CLI with the server stopped. See [when can a database be restored?](./overview.md#when-can-a-database-be-restored) + +From the CLI with the server stopped, `target_database=` restores into a new database instead of overwriting the source. The target must not already exist; Harper picks the new database up on the next start. + +```json +{ "operation": "restore_backup", "database": "data", "backup_id": 1 } +``` + +```bash +harper restore_backup database=data backup_id=1 +# restore into a new database instead of overwriting the source (server stopped) +harper restore_backup database=data backup_id=1 target_database=data_restored +``` + +## `get_backup` + + + +Streams a full snapshot of the specified database in the HTTP response for download — there is no server-side artifact and nothing to clean up. The server must be running; this is the one backup operation with no offline form. Behavior depends on the storage engine: + +- **RocksDB** — streams a `tar` archive of the current database state (all tables plus the transaction log), gzipped by default (`gzip` is a RocksDB-only option and compresses the snapshot substantially). Pass `"gzip": false` for a plain `tar`. The snapshot is always the current state; downloading a specific historical `backup_id` is not supported — to move a retained backup off-host, copy its backup directory. +- **LMDB** — streams the `.mdb` file. Specify `"table"` for a single table or `"tables"` for a set, and `"include_audit": true` to include the audit store. These options are LMDB-only. + +```json +{ "operation": "get_backup", "database": "data" } +``` + +From the CLI, pass `out=` to choose the output file (defaults to a name derived from the response): + +```bash +harper get_backup database=data out=./data.tar.gz +harper get_backup database=data target=https://node-2.example.com:9925 out=./data.tar.gz +``` diff --git a/reference/backups/overview.md b/reference/backups/overview.md new file mode 100644 index 00000000..a0a94e39 --- /dev/null +++ b/reference/backups/overview.md @@ -0,0 +1,107 @@ +--- +id: overview +title: Backups +--- + + + +Harper can back up and restore its databases natively. A backup is a whole-database copy — all tables plus the audit/transaction log — so a restored database is a consistent point-in-time image of everything in it. (`get_backup` on an LMDB database is the exception: it can stream a subset of tables, and includes the audit store only when requested.) + +Two complementary mechanisms are available: + +- **Managed backups** — incremental, checksum-verified backups kept in a server-side repository. Harper creates, lists, verifies, deletes, purges, and restores them through the [backup operations](./operations.md). +- **Snapshot download** — [`get_backup`](./operations.md#get_backup) streams a full snapshot of a database over HTTP for storage off-host, with no server-side artifact. + +Every backup operation is available through the [Operations API](../operations-api/overview.md) and as a [CLI command](./operations.md#running-backup-operations-from-the-cli) under the same operation name. + +## How managed backups work + +Managed backups use the RocksDB backup engine. Each backup is a directory under the configured backup root ([`storage.backupPath`](../configuration/options.md#storage), default `/backup`), with one subdirectory per database. Because RocksDB data files are immutable once written, backups in the same location share unchanged files: the first backup copies the whole database, and each subsequent backup only copies what changed since the last one. Shared files are reference-counted, so deleting a backup only removes files no remaining backup references. + +Every managed backup includes the database's transaction log, so a restored database keeps its `read_audit_log` history as of the backup point. + +The long-running operations (`create_backup`, `verify_backup`, `restore_backup`) run as background [jobs](../operations-api/operations.md#jobs) when invoked through a running server: the operation returns a `job_id` immediately, and [`get_job`](../operations-api/operations.md#get_job) reports the outcome. + +## Online and offline + +Backup operations work whether or not Harper is running. Invoked from the CLI while the server is running, the operation is forwarded to the server; while the server is stopped, the command operates directly on the database and backup files. `get_backup` is the exception — it streams from a running server and has no offline form. + +The offline path matters for restore. RocksDB is single-writer, so an in-place restore requires the database to be fully closed first. A running Harper server can close its own worker threads' handles, but it cannot close a handle held by a loaded component, nor stop the `system` database it depends on to run — those databases can only be restored with the server stopped. See [when can a database be restored?](#when-can-a-database-be-restored) below. + +## Limitations + +- **Managed backups require the RocksDB storage engine.** For LMDB databases, use [`get_backup`](./operations.md#get_backup) or [volume snapshots](../cli/commands.md#how-backups-work). +- **Whole-database granularity.** There is no per-table backup or restore. (The one exception: `get_backup` on an LMDB database can stream individual tables, and includes the audit store only with `include_audit`.) +- **One storage root per database.** A database whose tables use per-table `path` storage configs spans multiple root stores and cannot be backed up with these operations. +- **Backups live on the node that created them.** The backup repository is a local directory. For disaster recovery, copy backup directories off-host, or use `get_backup` to pull snapshots from a running server. +- **`get_backup` always streams the current state.** It cannot download a historical managed backup; to move a retained backup off-host, copy its backup directory. +- **A restore is a point-in-time rollback.** In a replicated cluster, coordinate a restore with replication before bringing the node back. + +### When can a database be restored? + +An in-place restore purges and rewrites the database's files, which requires the database to be fully closed first. Whether a restore can run online depends on what is holding the database open: + +| Database | Online `restore_backup` (server running) | Offline `harper restore_backup` (server stopped) | +| -------------------------------------------------- | ---------------------------------------- | ------------------------------------------------ | +| A user database not opened by any loaded component | Yes — restored in place | Yes | +| A user database that a loaded component keeps open | No — the job fails with a `409` | Yes | +| The `system` database | No — rejected up front | Yes | + +- **Online** works only when no loaded component is holding the database open. If a component is using it, the job fails fast with a `409` telling you to restore offline — Harper cannot force a component's handles closed while the server is running, and restoring under an open instance would corrupt it. (Harper does not track which component uses which database, so it cannot selectively stop one; it can only detect that the database is still open.) +- **Offline** — the same [`restore_backup`](./operations.md#restore_backup) command run from the CLI with the server stopped — works for any database, because no components are loaded and nothing holds it open. This is the required path for the `system` database and for any database a component keeps open. + +Online restore always restores in place. To restore into a new database instead of overwriting the source, run `restore_backup` with `target_database` from the CLI with the server stopped. + +## Example: incremental backups and restore + +Create a managed backup of the `data` database (the default): + +```bash +harper create_backup database=data +``` + +The first backup copies the entire database; each one after that only copies files that changed, so frequent backups are cheap. Schedule `create_backup` as often as your recovery point requires, and manage retention with `purge_backups`: + +```bash +harper purge_backups database=data keep_count=7 +``` + +List the backups to find the one to restore: + +```bash +harper list_backups database=data +``` + +Restore the latest backup, or pass `backup_id=` for an earlier one: + +```bash +harper restore_backup database=data +``` + +With the server running, this restores the database in place — Harper closes the database across its worker threads, restores it, and reloads it — as long as nothing is holding the database open (see [when can a database be restored?](#when-can-a-database-be-restored)). With the server stopped, the same command restores the files directly and works for any database. + +## Example: download a snapshot and restore it manually + +`get_backup` streams a full snapshot of a database from a running server — local or remote — which makes it the simplest way to keep backups off-host. For a RocksDB database the stream is a `tar` archive, gzipped by default: + +```bash +harper get_backup database=data out=./data.tar.gz +# or pull from another instance +harper get_backup database=data target=https://node-2.example.com:9925 out=./data.tar.gz +``` + +The archive contains the database's current state — all tables plus the transaction log — and extracts into a directory that opens directly as a RocksDB database. + +To restore it, stop Harper, replace the database's directory under the storage path ([`storage.path`](../configuration/options.md#storage), default `/database`) with the extracted archive, and start Harper again: + +```bash +harper stop +mv ~/hdb/database/data ~/hdb/database/data.old +mkdir -p ~/hdb/database/data +tar -xzf data.tar.gz -C ~/hdb/database/data +harper start +``` + +Keeping the previous directory as `data.old` makes the swap easy to undo; delete it once the restored database checks out. + +For an LMDB database, `get_backup` streams the database's `.mdb` file instead; restore it by replacing the database's `.mdb` file under the same storage path (`/.mdb`) with the downloaded file while Harper is stopped. diff --git a/reference/cli/commands.md b/reference/cli/commands.md index 24cfbc57..7a108426 100644 --- a/reference/cli/commands.md +++ b/reference/cli/commands.md @@ -294,6 +294,8 @@ Database files are stored in the `hdb/database` directory. As long as the snapsh - Cloud provider volume snapshots (AWS EBS, Azure Disk, GCP Persistent Disk) - Enterprise backup solutions with snapshot capabilities +Harper also has a built-in backup system — managed incremental backups and snapshot downloads, runnable from the CLI under their operation names (e.g. `harper create_backup`). See [Backups](../backups/overview.md). + ## Remote Operations The CLI supports executing commands on remote Harper instances. For details, see [CLI Overview - Remote Operations](./overview.md#remote-operations). diff --git a/reference/cli/operations-api-commands.md b/reference/cli/operations-api-commands.md index e893f42e..23a7f5f7 100644 --- a/reference/cli/operations-api-commands.md +++ b/reference/cli/operations-api-commands.md @@ -97,6 +97,13 @@ This is just a brief overview of all operations available as CLI commands. Revie | `get_fingerprint` | Get instance fingerprint | [Licensing](../operations-api/operations.md#registration--licensing) | v4.3.0 | | `set_license` | Set license key | [Licensing](../operations-api/operations.md#registration--licensing) | v4.3.0 | | `get_usage_licenses` | Get usage and license info | [Licensing](../operations-api/operations.md#registration--licensing) | v4.7.3 | +| `create_backup` | Create a managed database backup | [Backups](../backups/operations.md) | v5.2.0 | +| `list_backups` | List managed backups | [Backups](../backups/operations.md) | v5.2.0 | +| `verify_backup` | Verify a managed backup's integrity | [Backups](../backups/operations.md) | v5.2.0 | +| `delete_backup` | Delete a managed backup | [Backups](../backups/operations.md) | v5.2.0 | +| `purge_backups` | Delete all but the newest backups | [Backups](../backups/operations.md) | v5.2.0 | +| `restore_backup` | Restore a database from a backup | [Backups](../backups/operations.md) | v5.2.0 | +| `get_backup` | Download a full database snapshot | [Backups](../backups/operations.md) | v5.2.0 | | `get_job` | Get job status | [Jobs](../operations-api/operations.md#jobs) | v4.3.0 | | `search_jobs_by_start_date` | Search jobs by start date | [Jobs](../operations-api/operations.md#jobs) | v4.3.0 | | `read_log` | Read application logs | [Logging](../operations-api/operations.md#logs) | v4.3.0 | diff --git a/reference/configuration/options.md b/reference/configuration/options.md index 482c7d9d..48b80fc6 100644 --- a/reference/configuration/options.md +++ b/reference/configuration/options.md @@ -257,6 +257,7 @@ storage: - `noReadAhead` — Advise OS against read-ahead; _Default_: `false` - `prefetchWrites` — Prefetch before write transactions; _Default_: `true` - `path` — Database files directory; _Default_: `/database` +- `backupPath` — Directory for managed database backups (created by [`create_backup`](../backups/operations.md#create_backup)), one subdirectory per database; _Default_: `/backup` (Added in: v5.2.0) - `blobPaths` — Blob storage directory or directories; _Default_: `/blobs` (Added in: v4.5.0) - `pageSize` — Database page size (bytes); _Default_: OS default - `reclamation.threshold` — Free-space ratio below which reclamation begins evicting from caching tables; _Default_: `0.4` (Added in: v4.5.0) diff --git a/reference/index.md b/reference/index.md index afb446d1..aff311c9 100644 --- a/reference/index.md +++ b/reference/index.md @@ -40,6 +40,7 @@ For concept introductions, tutorials, and guides, see the [Learn](/learn) sectio | Section | Description | | ------------------------------------------------------------ | ------------------------------------------------------------------ | +| [Backups](./backups/overview.md) | Managed incremental backups, snapshot downloads, and restore | | [Logging](./logging/overview.md) | Log configuration, the `logger` API, and log management operations | | [Analytics](./analytics/overview.md) | Resource and storage analytics, system tables | | [MQTT](./mqtt/overview.md) | MQTT broker configuration and usage | diff --git a/reference/operations-api/operations.md b/reference/operations-api/operations.md index e958e881..7f278ba1 100644 --- a/reference/operations-api/operations.md +++ b/reference/operations-api/operations.md @@ -44,7 +44,6 @@ Detailed documentation: [Database Overview](../database/overview.md) | `drop_table` | Drops a table and all its records | super_user | | `create_attribute` | Adds a new attribute to a table | super_user | | `drop_attribute` | Removes an attribute and all its values from a table | super_user | -| `get_backup` | Returns a binary snapshot of a database for backup purposes | super_user | ### `describe_all` @@ -133,9 +132,75 @@ Drops an attribute and all its values from the specified table. } ``` +--- + +## Backup & Restore + +Operations for backing up and restoring databases. Managed backups require the RocksDB storage engine; `get_backup` works with both RocksDB and LMDB. + +Detailed documentation: [Backup Operations](../backups/operations.md) + +| Operation | Description | Role Required | +| ---------------- | ------------------------------------------------------------------- | ------------- | +| `create_backup` | Creates a managed, incremental directory backup of a database (job) | super_user | +| `list_backups` | Lists the managed backups for a database | super_user | +| `verify_backup` | Verifies a managed backup's integrity (job) | super_user | +| `delete_backup` | Deletes a single managed backup | super_user | +| `purge_backups` | Deletes all but the newest `keep_count` managed backups | super_user | +| `restore_backup` | Restores a database from a managed backup (job) | super_user | +| `get_backup` | Streams a full snapshot of a database in the response for download | super_user | + +### `create_backup` + +Creates an incremental directory backup of the database under the configured backup root. Runs as a background [job](#jobs) that reports the new `backup_id`. + +```json +{ "operation": "create_backup", "database": "dev" } +``` + +### `list_backups` + +Returns the managed backups for a database, each with its `backup_id`, `timestamp`, `size`, and `file_count`. + +```json +{ "operation": "list_backups", "database": "dev" } +``` + +### `verify_backup` + +Verifies a managed backup's integrity, including checksums when `verify_checksum` is `true` (slower). Runs as a background [job](#jobs). + +```json +{ "operation": "verify_backup", "database": "dev", "backup_id": 1, "verify_checksum": true } +``` + +### `delete_backup` + +Deletes a single managed backup. + +```json +{ "operation": "delete_backup", "database": "dev", "backup_id": 1 } +``` + +### `purge_backups` + +Deletes all but the newest `keep_count` managed backups. + +```json +{ "operation": "purge_backups", "database": "dev", "keep_count": 3 } +``` + +### `restore_backup` + +Restores a database in place from a managed backup, as a background [job](#jobs). `backup_id` defaults to the latest backup. Restoring the `system` database, or a database a loaded component keeps open, requires the server to be stopped — see [when can a database be restored?](../backups/overview.md#when-can-a-database-be-restored) + +```json +{ "operation": "restore_backup", "database": "dev", "backup_id": 1 } +``` + ### `get_backup` -Returns a binary snapshot of the specified database (or individual table). Safe for backup while Harper is running. Specify `"table"` for a single table or `"tables"` for a set. +Streams a full snapshot of the specified database in the HTTP response for download. For RocksDB , a `tar` archive of the current state, gzipped by default; for LMDB, the `.mdb` file. ```json { "operation": "get_backup", "database": "dev" } diff --git a/release-notes/v5-lincoln/5.2.md b/release-notes/v5-lincoln/5.2.md index 8cea990a..dd89cb81 100644 --- a/release-notes/v5-lincoln/5.2.md +++ b/release-notes/v5-lincoln/5.2.md @@ -8,6 +8,12 @@ title: '5.2' All patch release notes for 5.2.x are available on the [releases page](https://github.com/HarperFast/harper/releases?q=v5.2&expanded=true). +## Backup & Restore + +New managed-backup operations for RocksDB databases: `create_backup`, `list_backups`, `verify_backup`, `delete_backup`, `purge_backups`, and `restore_backup`. Managed backups are a server-side, incremental, checksum-verified repository under the configured backup root (`storage.backupPath`, default `/backup`), and every backup captures the audit/transaction log alongside the data. `restore_backup` restores a user database in place without stopping Harper, as long as no loaded component is holding that database open; a database held open by a component — and always the `system` database — is restored offline by running `restore_backup` from the CLI with the server stopped. Each of these operations can be run from the CLI under its operation name (e.g. `harper create_backup database=data`): while Harper is running the operation is forwarded to the server, and while it is stopped the command operates directly on the files. See [Backups](/reference/v5/backups/overview) and [Backup Operations](/reference/v5/backups/operations). + +`get_backup` now understands RocksDB: it streams a full-snapshot `tar` of the database — gzipped by default (pass `gzip: false` for a plain `tar`) — instead of failing. The LMDB behavior (streaming the `.mdb`, with `table`/`tables`/`include_audit`) is unchanged. + ## Querying ### Filtered Vector Search (Predicate-Aware HNSW Traversal) diff --git a/sidebarsReference.ts b/sidebarsReference.ts index 63957800..c0cbba40 100644 --- a/sidebarsReference.ts +++ b/sidebarsReference.ts @@ -429,6 +429,24 @@ const sidebars: SidebarsConfig = { }, ], }, + { + type: 'category', + label: 'Backups', + collapsible: false, + className: 'reference-category-header', + items: [ + { + type: 'doc', + id: 'backups/overview', + label: 'Overview', + }, + { + type: 'doc', + id: 'backups/operations', + label: 'Operations', + }, + ], + }, { type: 'category', label: 'Logging', diff --git a/src/components/EngineBadge.module.css b/src/components/EngineBadge.module.css new file mode 100644 index 00000000..6f748ecf --- /dev/null +++ b/src/components/EngineBadge.module.css @@ -0,0 +1,7 @@ +.badge { + display: inline-block; + font-size: 0.8rem; + color: var(--ifm-color-content-secondary); + font-style: italic; + margin-bottom: 0.75rem; +} diff --git a/src/components/EngineBadge.tsx b/src/components/EngineBadge.tsx new file mode 100644 index 00000000..cf7e37cf --- /dev/null +++ b/src/components/EngineBadge.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import styles from './EngineBadge.module.css'; + +const CANONICAL_NAMES: Record = { + rocksdb: 'RocksDB', + lmdb: 'LMDB', +}; + +interface EngineBadgeProps { + /** Comma-separated storage engines, e.g. "RocksDB" or "RocksDB, LMDB" */ + engines: string; +} + +export default function EngineBadge({ engines }: EngineBadgeProps) { + const names = engines.split(',').map((engine) => { + const trimmed = engine.trim(); + return CANONICAL_NAMES[trimmed.toLowerCase()] ?? trimmed; + }); + return ( + + {names.length > 1 ? 'Engines' : 'Engine'}: {names.join(', ')} + + ); +} diff --git a/src/theme/MDXComponents.tsx b/src/theme/MDXComponents.tsx index 7c722ec4..ff8e3b6f 100644 --- a/src/theme/MDXComponents.tsx +++ b/src/theme/MDXComponents.tsx @@ -1,7 +1,9 @@ import MDXComponents from '@theme-original/MDXComponents'; import VersionBadge from '@site/src/components/VersionBadge'; +import EngineBadge from '@site/src/components/EngineBadge'; export default { ...MDXComponents, VersionBadge, + EngineBadge, };