diff --git a/cmd/seid/cmd/root.go b/cmd/seid/cmd/root.go index 2c9dee777e..82b09f20b2 100644 --- a/cmd/seid/cmd/root.go +++ b/cmd/seid/cmd/root.go @@ -15,6 +15,7 @@ import ( "github.com/sei-protocol/sei-chain/admin" "github.com/sei-protocol/sei-chain/app" "github.com/sei-protocol/sei-chain/app/params" + "github.com/sei-protocol/sei-chain/cmd/seid/cmd/statesync" "github.com/sei-protocol/sei-chain/cosmosmetrics" evmrpcconfig "github.com/sei-protocol/sei-chain/evmrpc/config" giganodeconfig "github.com/sei-protocol/sei-chain/giga/config" @@ -138,6 +139,7 @@ func initRootCmd( config.Cmd(), SnapshotCmd(), LogLevelCmd(), + statesync.Cmd(), ) tracingProviderOpts, err := tracing.GetTracerProviderOptions(tracing.DefaultTracingURL) diff --git a/cmd/seid/cmd/statesync/archive_test.go b/cmd/seid/cmd/statesync/archive_test.go new file mode 100644 index 0000000000..ba08d5ce8f --- /dev/null +++ b/cmd/seid/cmd/statesync/archive_test.go @@ -0,0 +1,255 @@ +package statesync + +import ( + "archive/tar" + "bytes" + "encoding/binary" + "encoding/json" + "math/rand" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + + sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" +) + +func validManifest() *manifest { + return &manifest{ + FormatVersion: formatVersion, + ChainID: testChainID, + CheckpointHeight: 7, + CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), + ProducerVersion: "test", + RowCount: 2, + StateBytes: int64(len("bank/a") + len("1") + len("bank/b") + len("2")), + StateHash: hexHash{0x01, 0x02, 0x03}, + } +} + +func TestDecodeManifestAcceptsAValidManifest(t *testing.T) { + data, err := json.Marshal(validManifest()) + require.NoError(t, err) + m, err := decodeManifest(data) + require.NoError(t, err) + require.Equal(t, validManifest(), m) +} + +func TestDecodeManifestRejects(t *testing.T) { + edit := func(f func(m *manifest)) []byte { + m := validManifest() + f(m) + data, err := json.Marshal(m) + require.NoError(t, err) + return data + } + valid, err := json.Marshal(validManifest()) + require.NoError(t, err) + + cases := map[string][]byte{ + "an unknown top field": bytes.Replace(valid, []byte(`{"format_version"`), []byte(`{"extra":1,"format_version"`), 1), + "trailing data": append(append([]byte{}, valid...), []byte(`{}`)...), + "a trailing brace": append(append([]byte{}, valid...), '}'), + "another format version": edit(func(m *manifest) { m.FormatVersion = 1 }), + "an empty chain id": edit(func(m *manifest) { m.ChainID = "" }), + "height 0": edit(func(m *manifest) { m.CheckpointHeight = 0 }), + "a negative row count": edit(func(m *manifest) { m.RowCount = -1 }), + "a negative byte count": edit(func(m *manifest) { m.StateBytes = -1 }), + "a missing state hash": edit(func(m *manifest) { m.StateHash = hexHash{} }), + "a short state hash": bytes.Replace(valid, []byte(`"state_hash":"01`), []byte(`"state_hash":"`), 1), + "a non-hex state hash": bytes.Replace(valid, []byte(`"state_hash":"01`), []byte(`"state_hash":"zz`), 1), + } + for name, data := range cases { + t.Run(name, func(t *testing.T) { + _, err := decodeManifest(data) + require.Error(t, err) + }) + } +} + +// tarEntry is one entry of a hand-built archive. +type tarEntry struct { + name string + content []byte + typeflag byte +} + +func buildRawArchive(t *testing.T, entries []tarEntry, opts ...zstd.EOption) []byte { + t.Helper() + var buf bytes.Buffer + zw, err := zstd.NewWriter(&buf, opts...) + require.NoError(t, err) + tw := tar.NewWriter(zw) + for _, e := range entries { + typeflag := e.typeflag + if typeflag == 0 { + typeflag = tar.TypeReg + } + hdr := &tar.Header{Name: e.name, Mode: 0o600, Typeflag: typeflag} + if typeflag == tar.TypeReg { + hdr.Size = int64(len(e.content)) + } else { + hdr.Linkname = string(e.content) + } + require.NoError(t, tw.WriteHeader(hdr)) + if typeflag == tar.TypeReg { + _, err := tw.Write(e.content) + require.NoError(t, err) + } + } + require.NoError(t, tw.Close()) + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +func manifestEntry(t *testing.T, m *manifest) tarEntry { + t.Helper() + data, err := json.Marshal(m) + require.NoError(t, err) + return tarEntry{name: manifestName, content: data} +} + +func proofEntry(height int64, data []byte) tarEntry { + return tarEntry{name: proofArchivePath(height), content: data} +} + +func rowChunkEntry(index int, rows ...rowKV) tarEntry { + var data []byte + for _, row := range rows { + data = appendRecord(data, []byte(row.key), []byte(row.value)) + } + return tarEntry{name: rowChunkPath(index), content: data} +} + +type rowKV struct { + key string + value string +} + +func appendRecord(dst, key, value []byte) []byte { + var buf [10]byte + n := binary.PutUvarint(buf[:], uint64(len(key))) + dst = append(dst, buf[:n]...) + dst = append(dst, key...) + n = binary.PutUvarint(buf[:], uint64(len(value))) + dst = append(dst, buf[:n]...) + dst = append(dst, value...) + return dst +} + +func readArchiveRows(t *testing.T, data []byte) ([]*sctypes.SnapshotNode, error) { + t.Helper() + a, m, err := openArchive(t.Context(), bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer a.Close() + if _, err := a.readProof(m.CheckpointHeight); err != nil { + return nil, err + } + importer := &recordingImporter{} + if err := a.importRows(m, importer); err != nil { + return nil, err + } + return importer.nodes, nil +} + +func TestOpenArchiveRejectsAWindowLargerThanTheArchiveWindow(t *testing.T) { + m := validManifest() + // An entry larger than one zstd block makes the encoder write a window size into the frame. + big := make([]byte, 1<<20) + _, _ = rand.New(rand.NewSource(1)).Read(big) //nolint:gosec // test content. + key := []byte("bank/big") + m.RowCount = 1 + m.StateBytes = int64(len(key) + len(big)) + entries := []tarEntry{ + manifestEntry(t, m), + proofEntry(m.CheckpointHeight, []byte("proof")), + {name: rowChunkPath(0), content: appendRecord(nil, key, big)}, + } + + _, err := readArchiveRows(t, buildRawArchive(t, entries, zstd.WithWindowSize(archiveWindowSize))) + require.NoError(t, err) + _, err = readArchiveRows(t, buildRawArchive(t, entries, zstd.WithWindowSize(2*archiveWindowSize))) + require.ErrorIs(t, err, zstd.ErrWindowSizeExceeded) +} + +func TestArchiveReadsRows(t *testing.T) { + m := validManifest() + nodes, err := readArchiveRows(t, buildRawArchive(t, []tarEntry{ + manifestEntry(t, m), + proofEntry(m.CheckpointHeight, []byte("proof")), + rowChunkEntry(0, rowKV{"bank/a", "1"}, rowKV{"bank/b", "2"}), + })) + require.NoError(t, err) + require.Len(t, nodes, 2) + require.Equal(t, []byte("bank/a"), nodes[0].Key) + require.Equal(t, []byte("1"), nodes[0].Value) + require.Equal(t, int64(7), nodes[0].Version) +} + +func TestArchiveRejects(t *testing.T) { + m := validManifest() + validRows := rowChunkEntry(0, rowKV{"bank/a", "1"}, rowKV{"bank/b", "2"}) + cases := map[string][]tarEntry{ + "a manifest that is not first": {proofEntry(m.CheckpointHeight, []byte("proof")), manifestEntry(t, m), validRows}, + "a raw pebble entry": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: "flatkv/snapshot-00000000000000000007/account/000001.sst", content: []byte("sst")}}, + "an unknown entry": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: "other/x", content: []byte("x")}}, + "a missing chunk zero": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: rowChunkPath(1), content: validRows.content}}, + "a symlink chunk": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: rowChunkPath(0), content: []byte("/etc/passwd"), typeflag: tar.TypeSymlink}}, + "a zero key length": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: rowChunkPath(0), content: appendRecord(nil, nil, []byte("1"))}}, + "a zero value length": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: rowChunkPath(0), content: appendRecord(nil, []byte("bank/a"), nil)}}, + "a row count mismatch": {manifestEntry(t, editManifest(t, m, func(m *manifest) { m.RowCount = 3 })), proofEntry(m.CheckpointHeight, []byte("proof")), validRows}, + "a state byte count mismatch": {manifestEntry(t, editManifest(t, m, func(m *manifest) { m.StateBytes++ })), proofEntry(m.CheckpointHeight, []byte("proof")), validRows}, + "a proof of another height": {manifestEntry(t, m), proofEntry(m.CheckpointHeight+1, []byte("proof")), validRows}, + "a backslash path": {manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), {name: "flatkv\\rows\\00000000", content: validRows.content}}, + "a chunk with an oversized key": {manifestEntry(t, editManifest(t, m, func(m *manifest) { m.RowCount = 1; m.StateBytes = int64(maxStateKeySize + 2) })), proofEntry(m.CheckpointHeight, []byte("proof")), {name: rowChunkPath(0), content: oversizedLength(maxStateKeySize + 1)}}, + "a chunk with an oversized value": {manifestEntry(t, editManifest(t, m, func(m *manifest) { m.RowCount = 1; m.StateBytes = int64(maxStateValueSize + 2) })), proofEntry(m.CheckpointHeight, []byte("proof")), {name: rowChunkPath(0), content: append(appendRecord(nil, []byte("bank/a"), []byte("1")), oversizedLength(maxStateValueSize+1)...)}}, + } + for name, entries := range cases { + t.Run(name, func(t *testing.T) { + _, err := readArchiveRows(t, buildRawArchive(t, entries)) + require.Error(t, err) + }) + } + + t.Run("a truncated stream", func(t *testing.T) { + data := buildRawArchive(t, []tarEntry{manifestEntry(t, m), proofEntry(m.CheckpointHeight, []byte("proof")), validRows}) + _, err := readArchiveRows(t, data[:len(data)/2]) + require.Error(t, err) + }) + t.Run("an oversized manifest", func(t *testing.T) { + _, err := readArchiveRows(t, buildRawArchive(t, []tarEntry{{name: manifestName, content: []byte(strings.Repeat(" ", maxManifestSize+1))}})) + require.ErrorContains(t, err, "more than") + }) +} + +func editManifest(t *testing.T, src *manifest, edit func(*manifest)) *manifest { + t.Helper() + cp := *src + edit(&cp) + return &cp +} + +func oversizedLength(n int) []byte { + var buf [10]byte + size := binary.PutUvarint(buf[:], uint64(n)) + return buf[:size] +} + +type recordingImporter struct { + nodes []*sctypes.SnapshotNode +} + +func (i *recordingImporter) AddModule(string) error { return nil } + +func (i *recordingImporter) AddNode(node *sctypes.SnapshotNode) error { + i.nodes = append(i.nodes, node) + return nil +} + +func (i *recordingImporter) Abort(reason error) error { return reason } + +func (i *recordingImporter) Close() error { return nil } diff --git a/cmd/seid/cmd/statesync/command.go b/cmd/seid/cmd/statesync/command.go new file mode 100644 index 0000000000..d8831b8a37 --- /dev/null +++ b/cmd/seid/cmd/statesync/command.go @@ -0,0 +1,222 @@ +// Package statesync implements `seid statesync`, which exports a node's FlatKV checkpoint as a +// Giga state sync archive, and imports one into a fresh Autobahn fullnode home. +package statesync + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "syscall" + + "github.com/spf13/cobra" + "golang.org/x/sys/unix" + + gigaconfig "github.com/sei-protocol/sei-chain/giga/config" + "github.com/sei-protocol/sei-chain/sei-cosmos/server" + tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" + "github.com/sei-protocol/sei-chain/sei-tendermint/node" + tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" +) + +// errDisabled is returned by both commands when [giga.state_sync] enabled is false. +var errDisabled = errors.New("giga state sync is disabled; set [giga.state_sync] enabled = true in app.toml") + +// Cmd returns the `statesync` command group. +func Cmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "statesync", + Short: "Export or import a Giga state sync archive", + } + cmd.AddCommand(exportCmd(), importCmd()) + return cmd +} + +func exportCmd() *cobra.Command { + var opts exportOptions + var archive *os.File + cmd := &cobra.Command{ + Use: "export", + Short: "Export the current FlatKV checkpoint as an archive", + Long: `Export the current FlatKV checkpoint of this home as an archive. It is +safe to run while the node runs. With --out -, it writes the archive to stdout and prints one +line {"height":H} on stderr before the first archive byte. Log lines also go to stderr, so a +reader must find that line among them.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if opts.out == "-" { + // The parent hooks load the config and may log to stdout, so stdout is taken first. + var err error + if archive, err = takeStdout(); err != nil { + return err + } + } + return runParentPreRun(cmd, args) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if archive != nil { + defer func() { _ = archive.Close() }() + } + if opts.out == "" { + return errors.New("--out is required") + } + if opts.height < 0 { + return fmt.Errorf("--height %d is negative", opts.height) + } + stdout := cmd.OutOrStdout() + if archive != nil { + stdout = archive + } + t, err := resolveTarget(cmd) + if err != nil { + return err + } + return withSignals(cmd.Context(), func(ctx context.Context) error { + return runExport(ctx, t, opts, stdout, cmd.ErrOrStderr()) + }) + }, + } + cmd.Flags().StringVar(&opts.out, "out", "", "archive path, or - for stdout") + cmd.Flags().Int64Var(&opts.height, "height", 0, "checkpoint height to pack (default: the current checkpoint)") + cmd.Flags().StringVar(&opts.stagingParent, "staging-dir", "", + "dir for the staging files (default: the persistent state dir); it must be on the same filesystem as the FlatKV root") + return cmd +} + +func importCmd() *cobra.Command { + var from string + cmd := &cobra.Command{ + Use: "import", + Short: "Import an archive into a fresh Autobahn fullnode home", + Long: `Import an archive into a fresh Autobahn fullnode home. If it fails or stops, run the same +command again. It never deletes Autobahn data that an earlier import did not write. It refuses +when the persistent state dir has less free space than the key and value bytes of the archive; +the database needs more than that for its WAL and compaction.`, + RunE: func(cmd *cobra.Command, _ []string) error { + if from == "" { + return errors.New("--from is required") + } + t, err := resolveTarget(cmd) + if err != nil { + return err + } + return withSignals(cmd.Context(), func(ctx context.Context) error { + return runImport(ctx, t, from, cmd.InOrStdin(), cmd.ErrOrStderr()) + }) + }, + } + cmd.Flags().StringVar(&from, "from", "", "archive path, or - for stdin") + return cmd +} + +// target is the node home a command acts on. +type target struct { + layout node.AutobahnLayout + epochGenesis node.EpochGenesis + chainID string + mode string +} + +func resolveTarget(cmd *cobra.Command) (target, error) { + sctx := server.GetServerContextFromCmd(cmd) + return resolveTargetFromConfig(sctx.Config, sctx.Viper) +} + +// resolveTargetFromConfig returns the home that cfg and the app options describe. It refuses before it +// creates any file when the home is not an Autobahn home or giga state sync is disabled. +func resolveTargetFromConfig(cfg *tmcfg.Config, opts gigaconfig.AppOptions) (target, error) { + if cfg.AutobahnConfigFile == "" { + return target{}, errors.New("autobahn-config-file is not set: state sync archives are for Autobahn nodes only") + } + gigaCfg, err := gigaconfig.ReadConfig(opts) + if err != nil { + return target{}, fmt.Errorf("read [giga] config: %w", err) + } + if !gigaCfg.StateSync.Enabled { + return target{}, errDisabled + } + genDoc, err := tmtypes.GenesisDocFromFile(cfg.GenesisFile()) + if err != nil { + return target{}, fmt.Errorf("read genesis: %w", err) + } + layout, err := node.ResolveAutobahnLayout(cfg, gigaCfg.Storage, genDoc) + if err != nil { + return target{}, fmt.Errorf("resolve the Autobahn layout: %w", err) + } + epochGenesis, err := node.ResolveEpochGenesis(cfg, genDoc) + if err != nil { + return target{}, fmt.Errorf("resolve the epoch genesis: %w", err) + } + return target{layout: layout, epochGenesis: epochGenesis, chainID: genDoc.ChainID, mode: cfg.Mode}, nil +} + +// withSignals runs f with a context that SIGINT and SIGTERM cancel, so f can clean up. +func withSignals(parent context.Context, f func(ctx context.Context) error) error { + if parent == nil { + parent = context.Background() + } + ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM) + defer stop() + return f(ctx) +} + +// runParentPreRun runs the persistent pre-run hook of the nearest parent of cmd that has one. +func runParentPreRun(cmd *cobra.Command, args []string) error { + for p := cmd.Parent(); p != nil; p = p.Parent() { + switch { + case p.PersistentPreRunE != nil: + return p.PersistentPreRunE(cmd, args) + case p.PersistentPreRun != nil: + p.PersistentPreRun(cmd, args) + return nil + } + } + return nil +} + +// takeStdout returns a private handle on standard output, and points file descriptor 1 at +// standard error. +func takeStdout() (*os.File, error) { + fd, err := unix.Dup(int(os.Stdout.Fd())) //nolint:gosec // a file descriptor fits in an int. + if err != nil { + return nil, fmt.Errorf("dup stdout: %w", err) + } + // Log lines that any logger writes to stdout then cannot enter the archive. + if err := unix.Dup2(int(os.Stderr.Fd()), int(os.Stdout.Fd())); err != nil { //nolint:gosec // a file descriptor fits in an int. + _ = unix.Close(fd) + return nil, fmt.Errorf("point stdout at stderr: %w", err) + } + return os.NewFile(uintptr(fd), "archive"), nil //nolint:gosec // fd is a valid descriptor from Dup. +} + +// printHeight writes the one machine-readable line {"height":H}. +func printHeight(w io.Writer, height int64) error { + line, err := json.Marshal(struct { + Height int64 `json:"height"` + }{height}) + if err != nil { + return err + } + _, err = fmt.Fprintf(w, "%s\n", line) + return err +} + +// ctxReader fails reads once ctx is done. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + +// stepHook is called at named steps of export and import. A non-nil error stops the command there. +type stepHook func(step string) error + +func noStop(string) error { return nil } diff --git a/cmd/seid/cmd/statesync/export.go b/cmd/seid/cmd/statesync/export.go new file mode 100644 index 0000000000..3c61aae004 --- /dev/null +++ b/cmd/seid/cmd/statesync/export.go @@ -0,0 +1,321 @@ +package statesync + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + + "golang.org/x/sys/unix" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/statesyncproof" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/node" +) + +const ( + exportStagingDirName = ".state-sync-export" + linkAttempts = 3 +) + +var ( + // errExportRunning is returned when another export holds a dir that this export needs. + errExportRunning = errors.New("another statesync export is running") + // errNoProofFile is returned when the node has not written the state sync proof file of a + // checkpoint that export would pack. + errNoProofFile = errors.New("the node has written no state sync proof file (is [giga.state_sync] enabled on it?)") +) + +// Steps of export at which a stepHook is called. +const ( + exportStepStaged = "export: staging ready" + exportStepLinked = "export: checkpoint linked" + exportStepManifest = "export: manifest built" + exportStepWritten = "export: archive written" +) + +// exportOptions are the flags of export. +type exportOptions struct { + out string + height int64 + stagingParent string +} + +// runExport packs a checkpoint of t into an archive at opts.out, or into stdout when opts.out is "-". +func runExport(ctx context.Context, t target, opts exportOptions, stdout, stderr io.Writer) error { + return exportWith(ctx, t, opts, stdout, stderr, noStop) +} + +// exportWith is runExport with a hook that can stop it at each step. +func exportWith(ctx context.Context, t target, opts exportOptions, stdout, stderr io.Writer, stop stepHook) error { + parent := opts.stagingParent + if parent == "" { + parent = t.layout.PersistentStateDir + } + unlock, err := lockExportDirs(parent, opts.out) + if err != nil { + return err + } + defer unlock() + + stagingRoot := filepath.Join(parent, exportStagingDirName) + if err := removeExportLeftovers(stagingRoot, opts.out); err != nil { + return err + } + defer func() { _ = os.RemoveAll(stagingRoot) }() + stage := filepath.Join(stagingRoot, "export") + if err := os.MkdirAll(stage, 0o750); err != nil { + return fmt.Errorf("create staging dir: %w", err) + } + if err := stop(exportStepStaged); err != nil { + return err + } + + staged, err := stageCheckpoint(t.layout, opts.height, stage) + if err != nil { + return err + } + if err := flatkv.VerifySnapshotDir(staged.checkpoint, staged.version); err != nil { + return fmt.Errorf("verify the linked checkpoint: %w", err) + } + if err := stop(exportStepLinked); err != nil { + return err + } + + m, err := buildManifest(t.chainID, staged.version, staged.stats) + if err != nil { + return err + } + if err := stop(exportStepManifest); err != nil { + return err + } + if opts.out == "-" { + if err := printHeight(stderr, staged.version); err != nil { + return err + } + if err := writeArchive(ctx, stdout, m, staged.proof, staged.checkpoint); err != nil { + return err + } + return stop(exportStepWritten) + } + if err := writeArchiveFile(ctx, opts.out, m, staged.proof, staged.checkpoint, stop); err != nil { + return err + } + return printHeight(stderr, staged.version) +} + +// lockExportDirs takes the export lock on the staging parent and, unless out is "-", on the dir of +// out, until the returned function is called. +func lockExportDirs(stagingParent, out string) (func(), error) { + unlockStaging, err := lockExport(stagingParent) + if err != nil { + return nil, err + } + if out == "-" { + return unlockStaging, nil + } + outDir := filepath.Dir(out) + same, err := sameDir(stagingParent, outDir) + if err != nil { + unlockStaging() + return nil, err + } + if same { + return unlockStaging, nil + } + unlockOut, err := lockExport(outDir) + if err != nil { + unlockStaging() + return nil, err + } + return func() { unlockOut(); unlockStaging() }, nil +} + +// lockExport takes an exclusive lock on dir until the returned function is called. It fails at once +// with errExportRunning when another export holds the lock. +func lockExport(dir string) (func(), error) { + f, err := os.Open(dir) //nolint:gosec // dir is the operator's staging parent or output dir. + if err != nil { + return nil, fmt.Errorf("open %s: %w", dir, err) + } + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { //nolint:gosec // a file descriptor fits in an int. + _ = f.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, fmt.Errorf("%w on %s", errExportRunning, dir) + } + return nil, fmt.Errorf("lock %s: %w", dir, err) + } + return func() { _ = f.Close() }, nil +} + +// sameDir reports whether a and b are the same dir. +func sameDir(a, b string) (bool, error) { + ai, err := os.Stat(a) + if err != nil { + return false, err + } + bi, err := os.Stat(b) + if err != nil { + return false, err + } + return os.SameFile(ai, bi), nil +} + +// removeExportLeftovers removes the staging root and the partial archive that an export which stopped +// before its cleanup left. +func removeExportLeftovers(stagingRoot, out string) error { + // The staging hard links keep pruned checkpoint files on disk until they are removed. + if err := os.RemoveAll(stagingRoot); err != nil { + return fmt.Errorf("remove the staging files of an earlier export: %w", err) + } + if out == "-" { + return nil + } + if err := os.Remove(out + ".partial"); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove the partial archive of an earlier export: %w", err) + } + return nil +} + +type stagedCheckpoint struct { + version int64 + checkpoint string + proof []byte + stats flatkv.SnapshotStats +} + +// stageCheckpoint stages into stage the checkpoint at height, or the newest checkpoint +// that has a state sync proof file when height is 0. +func stageCheckpoint(layout node.AutobahnLayout, height int64, stage string) (stagedCheckpoint, error) { + var lastErr error + for range linkAttempts { + version, err := chooseCheckpoint(layout, height) + if err != nil { + return stagedCheckpoint{}, err + } + staged, err := stageVersion(layout, version, stage) + if err == nil { + return staged, nil + } + lastErr = err + // The node prunes a checkpoint and its proof file while it runs, so a pruned one is retried. + if height != 0 || !errors.Is(err, os.ErrNotExist) { + break + } + if err := os.RemoveAll(filepath.Join(stage, flatkvPrefix, flatkv.SnapshotDirName(version))); err != nil { + return stagedCheckpoint{}, fmt.Errorf("remove the staged checkpoint: %w", err) + } + } + return stagedCheckpoint{}, lastErr +} + +// chooseCheckpoint returns height when the node wrote a state sync proof file for it, or the +// newest checkpoint with a state sync proof file when height is 0. +func chooseCheckpoint(layout node.AutobahnLayout, height int64) (int64, error) { + if height != 0 { + ok, err := hasProofFile(layout, height) + if err != nil { + return 0, err + } + if !ok { + return 0, fmt.Errorf("%w for height %d in %s", errNoProofFile, height, layout.StateSyncProofDir) + } + return height, nil + } + versions, err := flatkv.SnapshotVersions(layout.FlatKVDir) + if err != nil { + return 0, fmt.Errorf("list the checkpoints: %w", err) + } + for _, v := range slices.Backward(versions) { + ok, err := hasProofFile(layout, v) + if err != nil { + return 0, err + } + if ok { + return v, nil + } + } + return 0, fmt.Errorf("%w for any of the %d checkpoints in %s", errNoProofFile, len(versions), layout.FlatKVDir) +} + +// hasProofFile reports whether the node wrote the state sync proof file for height. +func hasProofFile(layout node.AutobahnLayout, height int64) (bool, error) { + _, err := os.Stat(proofFile(layout, height)) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("stat the state sync proof file: %w", err) + } + return true, nil +} + +// proofFile returns the path of the state sync proof file that the node writes for height. +func proofFile(layout node.AutobahnLayout, height int64) string { + return filepath.Join(layout.StateSyncProofDir, statesyncproof.FileName(atypes.RoadIndex(height))) //nolint:gosec // heights are positive. +} + +// stageVersion hard-links the checkpoint at version into stage and reads its state sync proof file. +func stageVersion(layout node.AutobahnLayout, version int64, stage string) (stagedCheckpoint, error) { + if err := os.MkdirAll(filepath.Join(stage, flatkvPrefix), 0o750); err != nil { + return stagedCheckpoint{}, fmt.Errorf("create the staged flatkv dir: %w", err) + } + checkpoint := filepath.Join(stage, flatkvPrefix, flatkv.SnapshotDirName(version)) + if err := flatkv.HardLinkSnapshot(layout.FlatKVDir, version, checkpoint); err != nil { + return stagedCheckpoint{}, fmt.Errorf("link the checkpoint: %w", err) + } + data, err := os.ReadFile(proofFile(layout, version)) + if err != nil { + return stagedCheckpoint{}, fmt.Errorf("read the state sync proof file: %w", err) + } + if _, err := statesyncproof.Decode(data, atypes.RoadIndex(version)); err != nil { //nolint:gosec // heights are positive. + return stagedCheckpoint{}, fmt.Errorf("decode the state sync proof file of height %d: %w", version, err) + } + stats, err := flatkv.SnapshotDirStats(checkpoint, version) + if err != nil { + return stagedCheckpoint{}, fmt.Errorf("read the checkpoint metadata: %w", err) + } + return stagedCheckpoint{version: version, checkpoint: checkpoint, proof: data, stats: stats}, nil +} + +// writeArchiveFile writes the archive to out.partial, fsyncs it, and renames it over out. +func writeArchiveFile( + ctx context.Context, + out string, + m *manifest, + proof []byte, + checkpoint string, + stop stepHook, +) (err error) { + partial := out + ".partial" + f, err := os.OpenFile(partial, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) //nolint:gosec // out is the operator's path. + if err != nil { + return fmt.Errorf("create %s: %w", partial, err) + } + defer func() { + if err != nil { + _ = f.Close() + _ = os.Remove(partial) + } + }() + if err := writeArchive(ctx, f, m, proof, checkpoint); err != nil { + return err + } + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + if err := stop(exportStepWritten); err != nil { + return err + } + if err := os.Rename(partial, out); err != nil { + return err + } + return syncPath(filepath.Dir(out)) +} diff --git a/cmd/seid/cmd/statesync/export_test.go b/cmd/seid/cmd/statesync/export_test.go new file mode 100644 index 0000000000..44029f5a64 --- /dev/null +++ b/cmd/seid/cmd/statesync/export_test.go @@ -0,0 +1,203 @@ +package statesync + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + gigaconfig "github.com/sei-protocol/sei-chain/giga/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" +) + +func readManifest(t *testing.T, archive string) *manifest { + t.Helper() + f, err := os.Open(archive) //nolint:gosec // test path. + require.NoError(t, err) + defer func() { _ = f.Close() }() + a, m, err := openArchive(t.Context(), f) + require.NoError(t, err) + a.Close() + return m +} + +func TestExportPacksTheCurrentCheckpointAndLeavesNoStagingFiles(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + h := p.commit(t, 2) + p.closeStore(t) + + stagingRoot := filepath.Join(p.layout.PersistentStateDir, exportStagingDirName) + require.NoError(t, os.MkdirAll(filepath.Join(stagingRoot, "export-old"), 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(stagingRoot, "export-old", "x"), []byte("x"), 0o600)) + out := filepath.Join(t.TempDir(), "archive.tar.zst") + require.NoError(t, os.WriteFile(out+".partial", []byte("stale"), 0o600)) + + var stderr bytes.Buffer + require.NoError(t, runExport(t.Context(), p.target, exportOptions{out: out}, nil, &stderr)) + require.Equal(t, h, heightLine(t, stderr.String())) + require.NoDirExists(t, stagingRoot) + require.NoFileExists(t, out+".partial") + + m := readManifest(t, out) + require.Equal(t, h, m.CheckpointHeight) + require.Equal(t, testChainID, m.ChainID) + require.Positive(t, m.RowCount) + require.Positive(t, m.StateBytes) + + f, err := os.Open(out) //nolint:gosec // test path. + require.NoError(t, err) + defer func() { _ = f.Close() }() + a, m, err := openArchive(t.Context(), f) + require.NoError(t, err) + defer a.Close() + proof, err := a.readProof(m.CheckpointHeight) + require.NoError(t, err) + require.Equal(t, p.proofs[h], proof) +} + +func TestExportPacksTheNewestCheckpointWithAStateSyncProofFile(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + h := p.commit(t, 2) + require.NoError(t, os.Remove(proofFile(p.layout, p.commit(t, 3)))) + + archive, got := p.exportArchive(t) + require.Equal(t, h, got) + dst := freshHome(t) + require.NoError(t, runImport(t.Context(), dst, archive, nil, &bytes.Buffer{})) + requireRestored(t, dst, p, h) +} + +func TestExportWithoutAStateSyncProofFileFails(t *testing.T) { + p := newProducer(t, 2) + h := p.commit(t, 1) + require.NoError(t, os.Remove(proofFile(p.layout, h))) + + for _, height := range []int64{0, h} { + var stdout bytes.Buffer + err := runExport(t.Context(), p.target, exportOptions{out: "-", height: height}, &stdout, &bytes.Buffer{}) + require.ErrorIs(t, err, errNoProofFile) + require.Zero(t, stdout.Len()) + } +} + +func TestExportRefusesAStateSyncProofFileOfAnotherHeight(t *testing.T) { + p := newProducer(t, 2) + h := p.commit(t, 1) + require.NoError(t, os.WriteFile(proofFile(p.layout, h), stateSyncProofAt(t, h+1), 0o600)) + + var stdout bytes.Buffer + require.Error(t, runExport(t.Context(), p.target, exportOptions{out: "-"}, &stdout, &bytes.Buffer{})) + require.Zero(t, stdout.Len()) +} + +func TestExportToStdoutPrintsOneHeightLine(t *testing.T) { + p := newProducer(t, 2) + h := p.commit(t, 1) + + var archive, stderr bytes.Buffer + require.NoError(t, runExport(t.Context(), p.target, exportOptions{out: "-"}, &archive, &stderr)) + require.Equal(t, h, heightLine(t, stderr.String())) + a, m, err := openArchive(t.Context(), &archive) + require.NoError(t, err) + a.Close() + require.Equal(t, h, m.CheckpointHeight) +} + +func TestExportFailsWhileAnotherExportRuns(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + + unlock, err := lockExport(p.layout.PersistentStateDir) + require.NoError(t, err) + err = runExport(t.Context(), p.target, exportOptions{out: "-"}, &bytes.Buffer{}, &bytes.Buffer{}) + require.ErrorIs(t, err, errExportRunning) + unlock() + require.NoError(t, runExport(t.Context(), p.target, exportOptions{out: "-"}, &bytes.Buffer{}, &bytes.Buffer{})) +} + +func TestExportFailsWhileAnotherExportWritesToTheSameDir(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + outDir := t.TempDir() + out := filepath.Join(outDir, "archive.tar.zst") + partial := out + ".partial" + require.NoError(t, os.WriteFile(partial, []byte("another export"), 0o600)) + + unlock, err := lockExport(outDir) + require.NoError(t, err) + opts := exportOptions{out: out, stagingParent: t.TempDir()} + err = runExport(t.Context(), p.target, opts, nil, &bytes.Buffer{}) + require.ErrorIs(t, err, errExportRunning) + require.FileExists(t, partial) + unlock() + require.NoError(t, runExport(t.Context(), p.target, opts, nil, &bytes.Buffer{})) + require.NoFileExists(t, partial) +} + +func TestExportWritesIntoItsStagingParent(t *testing.T) { + p := newProducer(t, 2) + h := p.commit(t, 1) + out := filepath.Join(p.layout.PersistentStateDir, "archive.tar.zst") + + require.NoError(t, runExport(t.Context(), p.target, exportOptions{out: out}, nil, &bytes.Buffer{})) + require.Equal(t, h, readManifest(t, out).CheckpointHeight) +} + +func TestExportLinksSurvivePruningWhileItPacks(t *testing.T) { + p := newProducer(t, 0) + h := p.commit(t, 1) + out := filepath.Join(t.TempDir(), "archive.tar.zst") + + pruneBeforeWrite := func(step string) error { + if step == exportStepManifest { + p.commit(t, 2) + p.commit(t, 3) + require.NoDirExists(t, filepath.Join(p.layout.FlatKVDir, flatkv.SnapshotDirName(h))) + } + return nil + } + require.NoError(t, exportWith(t.Context(), p.target, exportOptions{out: out}, nil, &bytes.Buffer{}, pruneBeforeWrite)) + + dst := freshHome(t) + require.NoError(t, runImport(t.Context(), dst, out, nil, &bytes.Buffer{})) + s := openStore(t, dst.layout.FlatKVDir, 2) + defer func() { require.NoError(t, s.Close()) }() + require.Equal(t, h, s.Version()) +} + +func TestExportWithAPrunedHeightFails(t *testing.T) { + p := newProducer(t, 0) + h := p.commit(t, 1) + p.commit(t, 2) + + err := runExport(t.Context(), p.target, exportOptions{out: "-", height: h}, &bytes.Buffer{}, &bytes.Buffer{}) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestResolveTargetRefusesBeforeItWritesAnyFile(t *testing.T) { + home := t.TempDir() + cfg := tmcfg.TestConfig().SetRoot(home) + + cfg.AutobahnConfigFile = "" + _, err := resolveTargetFromConfig(cfg, appOptions{gigaconfig.FlagStateSyncEnabled: true}) + require.ErrorContains(t, err, "autobahn-config-file is not set") + + cfg.AutobahnConfigFile = "config/autobahn.json" + _, err = resolveTargetFromConfig(cfg, appOptions{}) + require.ErrorIs(t, err, errDisabled) + _, err = resolveTargetFromConfig(cfg, appOptions{gigaconfig.FlagStateSyncEnabled: false}) + require.ErrorIs(t, err, errDisabled) + + entries, err := os.ReadDir(home) + require.NoError(t, err) + require.Empty(t, entries) +} + +type appOptions map[string]any + +func (o appOptions) Get(key string) any { return o[key] } diff --git a/cmd/seid/cmd/statesync/extract.go b/cmd/seid/cmd/statesync/extract.go new file mode 100644 index 0000000000..c3a5774129 --- /dev/null +++ b/cmd/seid/cmd/statesync/extract.go @@ -0,0 +1,221 @@ +package statesync + +import ( + "archive/tar" + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + + "github.com/klauspost/compress/zstd" + + sctypes "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types" +) + +// chunkReadBufferSize is the read buffer of a row chunk decoder. +const chunkReadBufferSize = 1 << 20 + +// archiveReader reads an archive stream once, in order. +type archiveReader struct { + zr *zstd.Decoder + tr *tar.Reader +} + +// openArchive reads the manifest, which must be the first entry of the archive in r. +func openArchive(ctx context.Context, r io.Reader) (*archiveReader, *manifest, error) { + zr, err := zstd.NewReader(ctxReader{ctx: ctx, r: r}, + zstd.WithDecoderConcurrency(1), + zstd.WithDecoderMaxWindow(archiveWindowSize), + zstd.WithDecoderMaxMemory(maxRowChunkSize+archiveWindowSize), + ) + if err != nil { + return nil, nil, fmt.Errorf("open zstd: %w", err) + } + a := &archiveReader{zr: zr, tr: tar.NewReader(zr)} + m, err := a.readManifest() + if err != nil { + a.Close() + return nil, nil, err + } + return a, m, nil +} + +// Close releases the decoder. +func (a *archiveReader) Close() { + a.zr.Close() +} + +func (a *archiveReader) readManifest() (*manifest, error) { + hdr, err := a.tr.Next() + if err != nil { + return nil, fmt.Errorf("read the first archive entry: %w", err) + } + if hdr.Name != manifestName || hdr.Typeflag != tar.TypeReg { + return nil, fmt.Errorf("the first archive entry is %q, want %s", hdr.Name, manifestName) + } + if hdr.Size > maxManifestSize { + return nil, fmt.Errorf("the manifest is %d bytes, more than %d", hdr.Size, maxManifestSize) + } + data, err := io.ReadAll(a.tr) + if err != nil { + return nil, fmt.Errorf("read the manifest: %w", err) + } + return decodeManifest(data) +} + +// readProof reads the proof entry of height, which must follow the manifest. +func (a *archiveReader) readProof(height int64) ([]byte, error) { + hdr, err := a.tr.Next() + if err != nil { + return nil, fmt.Errorf("read the proof archive entry: %w", err) + } + want := proofArchivePath(height) + if err := validateArchiveEntry(hdr, want, maxProofSize); err != nil { + return nil, err + } + data := make([]byte, hdr.Size) + if _, err := io.ReadFull(a.tr, data); err != nil { + return nil, fmt.Errorf("read %s: %w", want, err) + } + return data, nil +} + +// importRows decodes the row chunks that follow the proof into importer. +func (a *archiveReader) importRows(m *manifest, importer sctypes.Importer) error { + var rows, bytesSeen int64 + for chunkIndex := 0; ; chunkIndex++ { + hdr, err := a.tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("read row chunk %d: %w", chunkIndex, err) + } + want := rowChunkPath(chunkIndex) + if err := validateArchiveEntry(hdr, want, maxRowChunkSize); err != nil { + return err + } + if err := a.importRowChunk(hdr, m, importer, &rows, &bytesSeen); err != nil { + return err + } + } + if rows != m.RowCount || bytesSeen != m.StateBytes { + return fmt.Errorf("imported %d rows and %d bytes, manifest records %d rows and %d bytes", + rows, bytesSeen, m.RowCount, m.StateBytes) + } + return nil +} + +func (a *archiveReader) importRowChunk( + hdr *tar.Header, + m *manifest, + importer sctypes.Importer, + rows *int64, + bytesSeen *int64, +) error { + r := &chunkReader{r: bufio.NewReaderSize(io.LimitReader(a.tr, hdr.Size), chunkReadBufferSize), remaining: hdr.Size} + for r.remaining > 0 { + key, value, err := r.readRecord() + if err != nil { + return fmt.Errorf("%s: %w", hdr.Name, err) + } + (*rows)++ + *bytesSeen += int64(len(key)) + int64(len(value)) + if *rows > m.RowCount || *bytesSeen > m.StateBytes { + return fmt.Errorf("archive row stream exceeds manifest: now at %d rows and %d bytes, manifest records %d rows and %d bytes", + *rows, *bytesSeen, m.RowCount, m.StateBytes) + } + if err := importer.AddNode(&sctypes.SnapshotNode{ + Key: key, + Value: value, + Version: m.CheckpointHeight, + Height: 0, + }); err != nil { + return fmt.Errorf("import FlatKV row %d: %w", *rows, err) + } + } + return nil +} + +// chunkReader reads the row records of one row chunk from the archive stream. +type chunkReader struct { + r *bufio.Reader + // remaining is the number of chunk bytes not yet read. + remaining int64 +} + +func (c *chunkReader) ReadByte() (byte, error) { + b, err := c.r.ReadByte() + if err != nil { + return 0, err + } + c.remaining-- + return b, nil +} + +func (c *chunkReader) readRecord() ([]byte, []byte, error) { + key, err := c.readField("key", maxStateKeySize) + if err != nil { + return nil, nil, err + } + value, err := c.readField("value", maxStateValueSize) + if err != nil { + return nil, nil, err + } + return key, value, nil +} + +// readField reads one length-prefixed field, checking its length before it allocates the field. +func (c *chunkReader) readField(name string, max int) ([]byte, error) { + n, err := binary.ReadUvarint(c) + if err != nil { + return nil, fmt.Errorf("read %s length: %w", name, err) + } + if n == 0 { + return nil, fmt.Errorf("%s length is 0", name) + } + if n > uint64(max) { //nolint:gosec // max is a positive archive limit constant. + return nil, fmt.Errorf("%s length %d is more than %d", name, n, max) + } + if n > uint64(c.remaining) { //nolint:gosec // remaining is not negative. + return nil, fmt.Errorf("%s length %d exceeds the remaining chunk size %d", name, n, c.remaining) + } + field := make([]byte, n) + if _, err := io.ReadFull(c.r, field); err != nil { + return nil, fmt.Errorf("read %s: %w", name, err) + } + c.remaining -= int64(n) //nolint:gosec // n is bounded by maxStateKeySize or maxStateValueSize. + return field, nil +} + +func validateArchiveEntry(hdr *tar.Header, want string, maxSize int64) error { + if err := validateArchivePath(hdr.Name); err != nil { + return err + } + if hdr.Name != want { + return fmt.Errorf("archive entry %q, want %q", hdr.Name, want) + } + if hdr.Typeflag != tar.TypeReg { + return fmt.Errorf("archive entry %q has type %q, want a regular file", hdr.Name, hdr.Typeflag) + } + if hdr.Size < 0 || hdr.Size > maxSize { + return fmt.Errorf("archive entry %q has size %d, want 0..%d", hdr.Name, hdr.Size, maxSize) + } + return nil +} + +// syncPath fsyncs the file or directory at p. +func syncPath(p string) error { + f, err := os.Open(p) //nolint:gosec // p is a path under the node home. + if err != nil { + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return fmt.Errorf("fsync %s: %w", p, err) + } + return f.Close() +} diff --git a/cmd/seid/cmd/statesync/failure_test.go b/cmd/seid/cmd/statesync/failure_test.go new file mode 100644 index 0000000000..4170f977fc --- /dev/null +++ b/cmd/seid/cmd/statesync/failure_test.go @@ -0,0 +1,323 @@ +package statesync + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" +) + +const ( + childEnv = "STATESYNC_TEST_CHILD" + crashExitCode = 3 + maxRerunsToEnd = 3 +) + +// childSpec tells TestChildProcess what to run. +type childSpec struct { + Op string `json:"op"` + PSD string `json:"psd"` + From string `json:"from"` + Out string `json:"out"` + Step string `json:"step"` +} + +// TestChildProcess runs a command in a child process for the parent tests. At spec.Step it exits at +// once, without running any deferred cleanup, like a crash. +func TestChildProcess(t *testing.T) { + raw := os.Getenv(childEnv) + if raw == "" { + t.Skip("run by the parent tests") + } + var spec childSpec + require.NoError(t, json.Unmarshal([]byte(raw), &spec)) + tg := targetAt(spec.PSD) + crash := func(step string) error { + if step == spec.Step { + os.Exit(crashExitCode) + } + return nil + } + ctx := context.Background() + switch spec.Op { + case "export": + require.NoError(t, exportWith(ctx, tg, exportOptions{out: spec.Out}, os.Stdout, os.Stderr, crash)) + case "import": + require.NoError(t, importWith(ctx, tg, spec.From, nil, os.Stderr, crash)) + case "export-stdout-with-logs": + archive, err := takeStdout() + require.NoError(t, err) + _, _ = fmt.Fprintln(os.Stdout, "INFO a log line that a logger writes to stdout") + require.NoError(t, runExport(ctx, tg, exportOptions{out: "-"}, archive, os.Stderr)) + require.NoError(t, archive.Close()) + case "export-command-with-pre-run-log": + parent := &cobra.Command{Use: "seid", SilenceUsage: true, PersistentPreRunE: func(*cobra.Command, []string) error { + _, _ = fmt.Fprintln(os.Stdout, "INFO a log line that a pre-run hook writes to stdout") + return errStopped + }} + parent.AddCommand(Cmd()) + parent.SetArgs([]string{"statesync", "export", "--out", "-"}) + require.ErrorIs(t, parent.Execute(), errStopped) + default: + t.Fatalf("unknown op %q", spec.Op) + } +} + +// runChild runs spec in a child process and returns its exit code, stdout and stderr. +func runChild(t *testing.T, spec childSpec) (int, []byte, []byte) { + t.Helper() + raw, err := json.Marshal(spec) + require.NoError(t, err) + cmd := exec.Command(os.Args[0], "-test.run=^TestChildProcess$", "-test.count=1") //nolint:gosec // the test binary itself. + cmd.Env = append(os.Environ(), childEnv+"="+string(raw)) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode(), stdout.Bytes(), stderr.Bytes() + } + require.NoError(t, err, "stderr: %s", stderr.String()) + return 0, stdout.Bytes(), stderr.Bytes() +} + +// stopKind is how a test stops a command at a step. +type stopKind string + +const ( + stopWithError stopKind = "error" + stopWithCrash stopKind = "crash" +) + +// stopOnce runs a command that stops at step, and reports whether it reached step. +func stopOnce(t *testing.T, kind stopKind, step string, inProcess func(stepHook) error, spec childSpec) bool { + t.Helper() + if kind == stopWithError { + err := inProcess(stopAt(step)) + if err == nil { + return false + } + require.ErrorIs(t, err, errStopped) + return true + } + spec.Step = step + code, _, stderr := runChild(t, spec) + if code == 0 { + return false + } + require.Equal(t, crashExitCode, code, "child stderr: %s", stderr) + return true +} + +// rerunUntilSuccess runs f until it returns nil. +func rerunUntilSuccess(t *testing.T, f func() error) { + t.Helper() + var err error + for range maxRerunsToEnd { + if err = f(); err == nil { + return + } + } + require.NoError(t, err, "the same command did not complete after %d reruns", maxRerunsToEnd) +} + +func TestImportFailureGoals(t *testing.T) { + p := newProducer(t, 2) + earlierHeight := p.commit(t, 1) + earlier, _ := p.exportArchive(t) + h := p.commit(t, 2) + archive, _ := p.exportArchive(t) + p.closeStore(t) + + type home struct { + newHome func(t *testing.T) target + marker utils.Option[int64] + } + homes := map[string]home{ + "a fresh home": {newHome: freshHome, marker: utils.None[int64]()}, + "a home with an earlier import": { + newHome: func(t *testing.T) target { + tg := freshHome(t) + require.NoError(t, runImport(t.Context(), tg, earlier, nil, &bytes.Buffer{})) + return tg + }, + marker: utils.Some(earlierHeight), + }, + } + steps := []string{ + importStepChecked, importStepVaultCleared, importStepFlatKVCleared, importStepImported, + importStepMarked, importStepProofInstalled, importStepEpochSeeded, importStepAnchorStored, + importStepInstalled, + } + for homeName, hm := range homes { + for _, step := range steps { + for _, kind := range []stopKind{stopWithError, stopWithCrash} { + t.Run(fmt.Sprintf("%s/%s/%s", homeName, step, kind), func(t *testing.T) { + tg := hm.newHome(t) + before := homeTree(t, tg) + run := func(stop stepHook) error { + return importWith(t.Context(), tg, archive, nil, io.Discard, stop) + } + spec := childSpec{Op: "import", PSD: tg.layout.PersistentStateDir, From: archive} + if !stopOnce(t, kind, step, run, spec) { + t.Skipf("import does not reach %q on this home", step) + } + requireSafeStop(t, tg, step, before, hm.marker, p.proofs[h], h) + rerunUntilSuccess(t, func() error { return run(noStop) }) + requireRestored(t, tg, p, h) + }) + } + } + } +} + +// requireSafeStop asserts what an import of the checkpoint at h, whose state sync proof file is +// proof, left on the home of tg when it stopped at step. The home held before and the restore +// marker for marker when the import started. Before import writes its marker, the home keeps its +// marker, and keeps its files until import clears an earlier import. Once import writes its +// marker, FlatKV reaches h only at the last step, so a node never finds FlatKV at the marker height +// without the state sync proof file, the epoch registry and the restore anchor of that height. +func requireSafeStop( + t *testing.T, + tg target, + step string, + before map[string]string, + marker utils.Option[int64], + proof []byte, + h int64, +) { + t.Helper() + switch step { + case importStepChecked: + require.Equal(t, before, homeTree(t, tg)) + case importStepVaultCleared: + requireMarker(t, tg, marker) + require.NoDirExists(t, tg.layout.AppHashVaultDir) + require.Equal(t, marker.OrPanic("only an earlier import has a vault to clear"), flatKVHeight(t, tg)) + case importStepFlatKVCleared, importStepImported: + requireMarker(t, tg, marker) + if marker.IsPresent() { + require.NoDirExists(t, tg.layout.AppHashVaultDir) + require.Zero(t, flatKVHeight(t, tg)) + } else { + require.Equal(t, before, homeTree(t, tg)) + } + case importStepMarked, importStepEpochSeeded: + requireMarker(t, tg, utils.Some(h)) + require.NoDirExists(t, tg.layout.AppHashVaultDir) + require.Zero(t, flatKVHeight(t, tg)) + case importStepProofInstalled: + requireMarker(t, tg, utils.Some(h)) + got, err := os.ReadFile(tg.layout.RestoreProofFile) + require.NoError(t, err) + require.Equal(t, proof, got) + require.NoDirExists(t, tg.layout.AppHashVaultDir) + require.Zero(t, flatKVHeight(t, tg)) + case importStepAnchorStored: + requireMarker(t, tg, utils.Some(h)) + requireAnchor(t, tg, h) + require.Zero(t, flatKVHeight(t, tg)) + case importStepInstalled: + requireMarker(t, tg, utils.Some(h)) + require.Equal(t, h, flatKVHeight(t, tg)) + default: + t.Fatalf("no stop state for step %q", step) + } +} + +// homeTree is treeOf the persistent state dir of tg without the staging dir. +func homeTree(t *testing.T, tg target) map[string]string { + t.Helper() + tree := treeOf(t, tg.layout.PersistentStateDir) + for p := range tree { + if p == importStagingDirName || strings.HasPrefix(p, importStagingDirName+string(filepath.Separator)) { + delete(tree, p) + } + } + return tree +} + +func TestExportFailureGoals(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + h := p.commit(t, 2) + p.closeStore(t) + want := treeOf(t, p.layout.PersistentStateDir) + + steps := []string{exportStepStaged, exportStepLinked, exportStepManifest, exportStepWritten} + for _, toStdout := range []bool{false, true} { + for _, step := range steps { + for _, kind := range []stopKind{stopWithError, stopWithCrash} { + t.Run(fmt.Sprintf("stdout=%t/%s/%s", toStdout, step, kind), func(t *testing.T) { + outDir := t.TempDir() + out := filepath.Join(outDir, "archive.tar.zst") + if toStdout { + out = "-" + } + run := func(stop stepHook) error { + return exportWith(t.Context(), p.target, exportOptions{out: out}, io.Discard, io.Discard, stop) + } + spec := childSpec{Op: "export", PSD: p.layout.PersistentStateDir, Out: out} + require.True(t, stopOnce(t, kind, step, run, spec)) + rerunUntilSuccess(t, func() error { return run(noStop) }) + + require.Equal(t, want, treeOf(t, p.layout.PersistentStateDir)) + entries, err := os.ReadDir(outDir) + require.NoError(t, err) + if toStdout { + require.Empty(t, entries) + return + } + require.Len(t, entries, 1) + require.Equal(t, h, readManifest(t, out).CheckpointHeight) + }) + } + } + } +} + +func TestExportKeepsLogLinesOutOfTheArchive(t *testing.T) { + p := newProducer(t, 2) + h := p.commit(t, 1) + p.closeStore(t) + + code, stdout, stderr := runChild(t, childSpec{Op: "export-stdout-with-logs", PSD: p.layout.PersistentStateDir}) + require.Equal(t, 0, code, "child stderr: %s", stderr) + require.Contains(t, string(stderr), "a log line that a logger writes to stdout") + require.Equal(t, h, heightLine(t, filterHeightLines(stderr))) + + dst := freshHome(t) + require.NoError(t, runImport(t.Context(), dst, "-", bytes.NewReader(stdout), &bytes.Buffer{})) + requireRestored(t, dst, p, h) +} + +func TestExportCommandTakesStdoutBeforeTheParentPreRun(t *testing.T) { + code, stdout, stderr := runChild(t, childSpec{Op: "export-command-with-pre-run-log"}) + require.Equal(t, 0, code, "child stderr: %s", stderr) + require.Empty(t, stdout) + require.Contains(t, string(stderr), "a log line that a pre-run hook writes to stdout") +} + +// filterHeightLines keeps only the lines of out that start like a height line. +func filterHeightLines(out []byte) string { + var kept []byte + for _, line := range bytes.Split(out, []byte("\n")) { + if bytes.HasPrefix(line, []byte(`{"height":`)) { + kept = append(append(kept, line...), '\n') + } + } + return string(kept) +} diff --git a/cmd/seid/cmd/statesync/helpers_test.go b/cmd/seid/cmd/statesync/helpers_test.go new file mode 100644 index 0000000000..b20f72e268 --- /dev/null +++ b/cmd/seid/cmd/statesync/helpers_test.go @@ -0,0 +1,404 @@ +package statesync + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "io/fs" + "maps" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/giga/apphash/builder" + "github.com/sei-protocol/sei-chain/giga/apphash/vault" + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" + "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/node" + tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types" +) + +const testChainID = "chain-a" + +var errStopped = errors.New("stopped by the test") + +func testLayout(psd string) node.AutobahnLayout { + return node.AutobahnLayout{ + PersistentStateDir: psd, + FlatKVDir: filepath.Join(psd, "data", "flatkv"), + BlockDBDirs: []string{filepath.Join(psd, "blockdb", "0")}, + ReceiptStoreDir: filepath.Join(psd, "data", "receipt"), + StateStoreDir: filepath.Join(psd, "data", "state_store"), + AppHashVaultDir: filepath.Join(psd, "data", "ledger", "app_hash"), + RestoreMarkerFile: filepath.Join(psd, "state-sync-restored"), + RestoreProofFile: filepath.Join(psd, "state-sync-proof"), + StateSyncProofDir: filepath.Join(psd, "data", "state_commit", "state_sync_proofs"), + EpochRegistryDir: filepath.Join(psd, "epochs"), + } +} + +// validatorKey returns the key of the one genesis validator of the test chain. +func validatorKey() atypes.SecretKey { + return atypes.SecretKeyFromED25519(ed25519.TestSecretKey([]byte("validator"))) +} + +func testEpochGenesis() node.EpochGenesis { + return node.EpochGenesis{ + GenDoc: &tmtypes.GenesisDoc{ChainID: testChainID, InitialHeight: 1}, + Validators: []atypes.PublicKey{validatorKey().Public()}, + } +} + +// stateSyncProofAt returns a state sync proof file for road height, signed by the genesis validator, +// whose AppQC signs appHashAt(height). +func stateSyncProofAt(t *testing.T, height int64) []byte { + t.Helper() + return stateSyncProofSignedBy(t, validatorKey(), height) +} + +// stateSyncProofSignedBy returns a state sync proof file for road height, signed by key as the one +// validator of a chain with the genesis of the test chain otherwise. +func stateSyncProofSignedBy(t *testing.T, key atypes.SecretKey, height int64) []byte { + t.Helper() + genesis := testEpochGenesis() + genesis.Validators = []atypes.PublicKey{key.Public()} + appHash := appHashAt(height) + data, err := node.StateSyncProofForTesting(genesis, []atypes.SecretKey{key}, height, appHash[:], t.TempDir()) + require.NoError(t, err) + return data +} + +// appHashAt returns the app hash that the state sync proof file of height signs. +func appHashAt(height int64) [32]byte { + var appHash [32]byte + binary.BigEndian.PutUint64(appHash[:], uint64(height)) //nolint:gosec // test heights are positive. + return appHash +} + +func targetAt(psd string) target { + return target{layout: testLayout(psd), epochGenesis: testEpochGenesis(), chainID: testChainID, mode: tmcfg.ModeFull} +} + +// freshHome returns a fullnode home that holds only an empty FlatKV root with its lock file. +func freshHome(t *testing.T) target { + t.Helper() + tg := targetAt(t.TempDir()) + lock, err := flatkv.LockRoot(tg.layout.FlatKVDir) + require.NoError(t, err) + require.NoError(t, lock.Unlock()) + return tg +} + +// producer is a home with a FlatKV store that writes a checkpoint every block. +type producer struct { + target + store *flatkv.CommitStore + epochRow []byte + // proofs holds the state sync proof file of each committed height. + proofs map[int64][]byte + rows map[string][]byte + // rowsAt holds a copy of rows at each committed height. + rowsAt map[int64]map[string][]byte +} + +// newProducer opens a store that keeps keepRecent older checkpoints. +func newProducer(t *testing.T, keepRecent uint32) *producer { + t.Helper() + p := &producer{ + target: targetAt(t.TempDir()), + proofs: map[int64][]byte{}, + rows: map[string][]byte{}, + rowsAt: map[int64]map[string][]byte{}, + } + row, err := node.GenesisEpochRegistryRowForTesting(p.epochGenesis, t.TempDir()) + require.NoError(t, err) + p.epochRow = row + p.store = openStore(t, p.layout.FlatKVDir, keepRecent) + t.Cleanup(func() { + if p.store != nil { + _ = p.store.Close() + } + }) + return p +} + +// closeStore closes the producer's store, so the home no longer changes. +func (p *producer) closeStore(t *testing.T) { + t.Helper() + require.NoError(t, p.store.Close()) + p.store = nil +} + +// commit commits one block that writes to all four data databases, to the bank module and the +// epoch_registry row, waits for its checkpoint to become current, and writes the state sync proof +// file of that checkpoint as a running node does. +func (p *producer) commit(t *testing.T, n byte) int64 { + t.Helper() + var addr ktype.Address + addr[19] = n + var slot ktype.Slot + slot[31] = n + nonce := make([]byte, vtype.NonceLen) + binary.BigEndian.PutUint64(nonce, uint64(n)) + var codeHash vtype.CodeHash + codeHash[0] = n + value := make([]byte, 32) + value[31] = n + evmPairs := []*proto.KVPair{ + {Key: keys.BuildEVMKey(keys.EVMKeyNonce, addr[:]), Value: nonce}, + {Key: keys.BuildEVMKey(keys.EVMKeyCodeHash, addr[:]), Value: codeHash[:]}, + {Key: keys.BuildEVMKey(keys.EVMKeyCode, addr[:]), Value: []byte{0x60, n}}, + {Key: keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addr, slot)), Value: value}, + } + bankPairs := []*proto.KVPair{{Key: []byte{'b', n}, Value: []byte{n}}} + epochPairs := []*proto.KVPair{{Key: []byte("epoch_registry"), Value: p.epochRow}} + require.NoError(t, p.store.ApplyChangeSets(p.store.Version()+1, []*proto.NamedChangeSet{ + {Name: "evm", Changeset: proto.ChangeSet{Pairs: evmPairs}}, + {Name: "bank", Changeset: proto.ChangeSet{Pairs: bankPairs}}, + {Name: "autobahn", Changeset: proto.ChangeSet{Pairs: epochPairs}}, + })) + v, err := p.store.Commit(p.store.Version() + 1) + require.NoError(t, err) + require.NoError(t, p.store.FlushSnapshots()) + cur, err := flatkv.ActiveSnapshotVersion(p.layout.FlatKVDir) + require.NoError(t, err) + require.Equal(t, v, cur) + for _, kv := range evmPairs { + got, ok := p.store.Get("evm", kv.Key) + require.True(t, ok) + p.rows["evm/"+string(kv.Key)] = got + } + p.rows["bank/"+string(bankPairs[0].Key)] = bankPairs[0].Value + p.rows["autobahn/epoch_registry"] = p.epochRow + p.rowsAt[v] = maps.Clone(p.rows) + p.proofs[v] = stateSyncProofAt(t, v) + require.NoError(t, os.MkdirAll(p.layout.StateSyncProofDir, 0o750)) + require.NoError(t, os.WriteFile(proofFile(p.layout, v), p.proofs[v], 0o600)) + return v +} + +// archiveWithChange packs the producer's checkpoint at height into a file after change edits the +// logical archive entries, and returns its path. +func (p *producer) archiveWithChange(t *testing.T, height int64, change func(m *manifest, entries *[]tarEntry)) string { + t.Helper() + stage := t.TempDir() + staged, err := stageVersion(p.layout, height, stage) + require.NoError(t, err) + m, err := buildManifest(p.chainID, height, staged.stats) + require.NoError(t, err) + var chunk []byte + _, err = flatkv.ExportSnapshotDir(t.Context(), staged.checkpoint, height, func(key, value []byte) error { + chunk = appendRecord(chunk, key, value) + return nil + }) + require.NoError(t, err) + entries := []tarEntry{manifestEntry(t, m), proofEntry(height, staged.proof)} + if len(chunk) > 0 { + entries = append(entries, tarEntry{name: rowChunkPath(0), content: chunk}) + } + if change != nil { + change(m, &entries) + for i := range entries { + if entries[i].name == manifestName { + entries[i] = manifestEntry(t, m) + break + } + } + } + out := filepath.Join(t.TempDir(), "archive.tar.zst") + require.NoError(t, os.WriteFile(out, buildRawArchive(t, entries), 0o600)) + return out +} + +func replaceEntry(entries []tarEntry, name string, content []byte) bool { + for i := range entries { + if entries[i].name == name { + entries[i].content = content + return true + } + } + return false +} + +func appendLogicalRow(entries []tarEntry, key, value []byte) []tarEntry { + if len(entries) == 0 || entries[len(entries)-1].name != rowChunkPath(0) { + return append(entries, tarEntry{name: rowChunkPath(0), content: appendRecord(nil, key, value)}) + } + entries[len(entries)-1].content = appendRecord(entries[len(entries)-1].content, key, value) + return entries +} + +// exportArchive packs the producer's current checkpoint into a file and returns its path and height. +func (p *producer) exportArchive(t *testing.T) (string, int64) { + t.Helper() + out := filepath.Join(t.TempDir(), "archive.tar.zst") + var stderr bytes.Buffer + require.NoError(t, runExport(t.Context(), p.target, exportOptions{out: out}, nil, &stderr)) + return out, heightLine(t, stderr.String()) +} + +func openStore(t *testing.T, dir string, keepRecent uint32) *flatkv.CommitStore { + t.Helper() + cfg := config.DefaultTestConfig(t) + cfg.DataDir = dir + cfg.SnapshotInterval = 1 + cfg.SnapshotKeepRecent = keepRecent + wal, err := flatkv.OpenStateWAL(cfg) + require.NoError(t, err) + s, err := flatkv.NewCommitStore(context.Background(), cfg, wal) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + return s +} + +// requireRestored asserts that the home of tg loads at height with the producer's rows, holds the +// epoch registry, the state sync proof file and the restore anchor of the producer's checkpoint at +// height, and holds the marker for height. +func requireRestored(t *testing.T, tg target, p *producer, height int64) { + t.Helper() + requireMarker(t, tg, utils.Some(height)) + require.NoDirExists(t, filepath.Join(tg.layout.PersistentStateDir, importStagingDirName)) + proof, err := os.ReadFile(tg.layout.RestoreProofFile) + require.NoError(t, err) + require.Equal(t, p.proofs[height], proof) + requireAnchor(t, tg, height) + + checkpoint := filepath.Join(p.layout.FlatKVDir, flatkv.SnapshotDirName(height)) + epochs, err := node.OpenCheckpointEpochs(p.epochGenesis, checkpoint, height) + require.NoError(t, err) + psd := t.TempDir() + want := node.AutobahnLayout{PersistentStateDir: psd, EpochRegistryDir: filepath.Join(psd, "epochs")} + require.NoError(t, node.InstallEpochRegistry(want, epochs)) + require.Equal(t, treeOf(t, want.EpochRegistryDir), treeOf(t, tg.layout.EpochRegistryDir)) + + s := openStore(t, tg.layout.FlatKVDir, 2) + defer func() { require.NoError(t, s.Close()) }() + require.Equal(t, height, s.Version()) + for k, want := range p.rowsAt[height] { + module, key, _ := bytes.Cut([]byte(k), []byte("/")) + got, ok := s.Get(string(module), key) + require.True(t, ok, "row %q", k) + require.Equal(t, want, got, "row %q", k) + } +} + +// requireAnchor asserts that the app-hash vault of tg ends at the restore anchor of height, with the +// app hash that the state sync proof file of height signs. +func requireAnchor(t *testing.T, tg target, height int64) { + t.Helper() + cfg := builder.DefaultAppHashBuilderConfig() + cfg.HashVaultConfig.Path = tg.layout.AppHashVaultDir + hashVault, err := vault.NewStandardHashVault(cfg.HashVaultConfig, cfg.ChainID, cfg.GigaActivationHeight) + require.NoError(t, err) + defer func() { require.NoError(t, hashVault.Close()) }() + _, highest, err := hashVault.Bounds() + require.NoError(t, err) + require.Equal(t, uint64(height), highest) //nolint:gosec // test heights are positive. + records, err := hashVault.Iterator(highest, highest) + require.NoError(t, err) + defer func() { require.NoError(t, records.Close()) }() + ok, err := records.Next() + require.NoError(t, err) + require.True(t, ok) + require.True(t, records.Entry().IsRestoreAnchor()) + require.Equal(t, appHashAt(height), records.Entry().AppHash()) +} + +// requireMarker asserts that the home of tg holds the restore marker for height, or no marker when +// height is None. +func requireMarker(t *testing.T, tg target, height utils.Option[int64]) { + t.Helper() + h, ok := height.Get() + if !ok { + require.NoFileExists(t, tg.layout.RestoreMarkerFile) + return + } + marker, err := os.ReadFile(tg.layout.RestoreMarkerFile) + require.NoError(t, err) + require.Equal(t, strconv.FormatInt(h, 10)+"\n", string(marker)) +} + +// flatKVHeight returns the height of the current FlatKV checkpoint of tg, or 0 when FlatKV has none. +func flatKVHeight(t *testing.T, tg target) int64 { + t.Helper() + v, err := flatkv.ActiveSnapshotVersion(tg.layout.FlatKVDir) + if errors.Is(err, os.ErrNotExist) { + return 0 + } + require.NoError(t, err) + return v +} + +// heightLine returns H from the single line {"height":H} that out must consist of. +func heightLine(t *testing.T, out string) int64 { + t.Helper() + lines := bytes.Split(bytes.TrimSuffix([]byte(out), []byte("\n")), []byte("\n")) + var heights []int64 + for _, l := range lines { + if h, ok := bytes.CutPrefix(l, []byte(`{"height":`)); ok { + v, err := strconv.ParseInt(string(bytes.TrimSuffix(h, []byte("}"))), 10, 64) + require.NoError(t, err) + heights = append(heights, v) + } + } + require.Len(t, heights, 1, "output %q", out) + return heights[0] +} + +// treeOf lists every path under root with its type, and the size and SHA-256 of each regular file. +func treeOf(t *testing.T, root string) map[string]string { + t.Helper() + tree := map[string]string{} + require.NoError(t, filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + switch { + case d.IsDir(): + tree[rel] = "dir" + case d.Type()&fs.ModeSymlink != 0: + target, err := os.Readlink(p) + if err != nil { + return err + } + tree[rel] = "symlink " + target + default: + data, err := os.ReadFile(p) //nolint:gosec // test path. + if err != nil { + return err + } + sum := sha256.Sum256(data) + tree[rel] = "file " + strconv.Itoa(len(data)) + " " + hex.EncodeToString(sum[:]) + } + return nil + })) + return tree +} + +// stopAt returns a hook that stops the command at step. +func stopAt(step string) stepHook { + return func(s string) error { + if s == step { + return errStopped + } + return nil + } +} diff --git a/cmd/seid/cmd/statesync/import.go b/cmd/seid/cmd/statesync/import.go new file mode 100644 index 0000000000..ba5f5969da --- /dev/null +++ b/cmd/seid/cmd/statesync/import.go @@ -0,0 +1,422 @@ +package statesync + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "golang.org/x/sys/unix" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/statesyncproof" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" + "github.com/sei-protocol/sei-chain/sei-tendermint/node" +) + +const importStagingDirName = ".state-sync-import" + +// Steps of import at which a stepHook is called, in the order import reaches them. +const ( + importStepChecked = "import: home checked" + importStepVaultCleared = "import: app-hash vault cleared" + importStepFlatKVCleared = "import: FlatKV cleared" + importStepImported = "import: FlatKV imported" + importStepMarked = "import: marker written" + importStepProofInstalled = "import: state sync proof installed" + importStepEpochSeeded = "import: epoch registry seeded" + importStepAnchorStored = "import: restore anchor stored" + importStepInstalled = "import: checkpoint installed" +) + +// runImport imports the archive at from, or on stdin when from is "-", into the home of t. +func runImport(ctx context.Context, t target, from string, stdin io.Reader, stderr io.Writer) error { + return importWith(ctx, t, from, stdin, stderr, noStop) +} + +// importWith is runImport with a hook that can stop it at each step. +func importWith(ctx context.Context, t target, from string, stdin io.Reader, stderr io.Writer, stop stepHook) error { + if t.mode == tmcfg.ModeValidator { + return errors.New("import is for fullnodes; this home is configured as a validator") + } + lock, err := flatkv.LockRoot(t.layout.FlatKVDir) + if err != nil { + return fmt.Errorf("lock the FlatKV root (is a node running on this home?): %w", err) + } + defer func() { _ = lock.Unlock() }() + + staging := filepath.Join(t.layout.PersistentStateDir, importStagingDirName) + if err := removeImportLeftovers(staging, t.layout); err != nil { + return err + } + resume, err := checkHome(t.layout, lock) + if err != nil { + return err + } + if err := stop(importStepChecked); err != nil { + return err + } + + in, closeIn, err := openSource(from, stdin) + if err != nil { + return err + } + defer closeIn() + archive, m, err := openArchive(ctx, in) + if err != nil { + return err + } + defer archive.Close() + if err := checkManifest(m, t); err != nil { + return err + } + if resume { + // The vault accepts the restore anchor only as the first record of an empty vault, so the + // anchor of the earlier import goes before the new one is stored. + if err := clearEarlierImport(t.layout, lock, stop); err != nil { + return err + } + } + if err := requireFreeSpace(t.layout.PersistentStateDir, m.StateBytes); err != nil { + return err + } + if err := os.Mkdir(staging, 0o750); err != nil { + return fmt.Errorf("create staging dir: %w", err) + } + defer func() { _ = os.RemoveAll(staging) }() + staged, err := stageArchive(ctx, archive, m, t.epochGenesis, staging, stderr) + if err != nil { + return err + } + if err := stop(importStepImported); err != nil { + return err + } + + if err := writeFileAtomic(t.layout.RestoreMarkerFile, []byte(strconv.FormatInt(m.CheckpointHeight, 10)+"\n")); err != nil { + return fmt.Errorf("write the restore marker: %w", err) + } + if err := stop(importStepMarked); err != nil { + return err + } + if err := writeFileAtomic(t.layout.RestoreProofFile, staged.proof); err != nil { + return fmt.Errorf("install the state sync proof file: %w", err) + } + if err := stop(importStepProofInstalled); err != nil { + return err + } + if err := node.InstallEpochRegistry(t.layout, staged.epochs); err != nil { + return err + } + if err := stop(importStepEpochSeeded); err != nil { + return err + } + if err := node.StoreRestoreAnchor(t.layout, t.epochGenesis, m.CheckpointHeight, staged.appHash); err != nil { + return err + } + if err := stop(importStepAnchorStored); err != nil { + return err + } + if err := installArchive(lock, staging, m); err != nil { + return err + } + if err := stop(importStepInstalled); err != nil { + return err + } + if err := os.RemoveAll(staging); err != nil { + return fmt.Errorf("remove staging dir: %w", err) + } + + if err := printHeight(stderr, m.CheckpointHeight); err != nil { + return err + } + _, _ = fmt.Fprintf(stderr, "Imported height %d. Start the node; on start it seeds height %d from %s and syncs from there.\n", + m.CheckpointHeight, m.CheckpointHeight, t.layout.RestoreProofFile) + return nil +} + +// removeImportLeftovers removes the staging dir and the temporary files that an import which +// stopped before its cleanup left. +func removeImportLeftovers(staging string, layout node.AutobahnLayout) error { + if err := os.RemoveAll(staging); err != nil { + return fmt.Errorf("remove the staging dir of an earlier import: %w", err) + } + for _, path := range []string{layout.RestoreMarkerFile, layout.RestoreProofFile} { + if err := os.Remove(path + ".tmp"); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove a temporary file of an earlier import: %w", err) + } + } + return nil +} + +// checkHome returns an error if the home has block, receipt or state data, or has FlatKV data, an +// epoch registry or app hashes without the restore marker. It reports whether the marker is +// present, in which case that data is from an earlier import and import replaces it. +func checkHome(layout node.AutobahnLayout, lock *flatkv.RootLock) (resume bool, err error) { + stores := append(slices.Clone(layout.BlockDBDirs), layout.ReceiptStoreDir, layout.StateStoreDir) + var nonEmpty []string + for _, dir := range stores { + empty, err := dirEmptyOrMissing(dir) + if err != nil { + return false, err + } + if !empty { + nonEmpty = append(nonEmpty, dir) + } + } + if len(nonEmpty) > 0 { + return false, fmt.Errorf("import needs a home without block, receipt or state data; remove %s", strings.Join(nonEmpty, ", ")) + } + + // Only import writes the marker, before it replaces the epoch registry, the vault and FlatKV, + // and the seed deletes it only after it writes block data. With an empty block store the marker + // means all three are import's own, or import stopped before it installed them and may retry. + if _, err := os.Stat(layout.RestoreMarkerFile); err == nil { + return true, nil + } else if !errors.Is(err, os.ErrNotExist) { + return false, fmt.Errorf("stat the restore marker: %w", err) + } + flatKVEmpty, err := lock.Empty() + if err != nil { + return false, err + } + if !flatKVEmpty { + nonEmpty = append(nonEmpty, layout.FlatKVDir) + } + for _, dir := range []string{layout.EpochRegistryDir, layout.AppHashVaultDir} { + empty, err := dirEmptyOrMissing(dir) + if err != nil { + return false, err + } + if !empty { + nonEmpty = append(nonEmpty, dir) + } + } + if len(nonEmpty) > 0 { + return false, fmt.Errorf("this home holds Autobahn state that import did not write; remove %s and run import again", + strings.Join(nonEmpty, ", ")) + } + return false, nil +} + +// removeDir removes dir and everything under it, and syncs its parent. +func removeDir(dir string) error { + if _, err := os.Lstat(dir); errors.Is(err, os.ErrNotExist) { + return nil + } + if err := os.RemoveAll(dir); err != nil { + return err + } + return syncPath(filepath.Dir(dir)) +} + +func dirEmptyOrMissing(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + if err != nil { + return false, err + } + return len(entries) == 0, nil +} + +func openSource(from string, stdin io.Reader) (io.Reader, func(), error) { + if from == "-" { + return stdin, func() {}, nil + } + f, err := os.Open(from) //nolint:gosec // from is the operator's path. + if err != nil { + return nil, nil, err + } + return f, func() { _ = f.Close() }, nil +} + +// checkManifest refuses an archive of another chain. +func checkManifest(m *manifest, t target) error { + if m.ChainID != t.chainID { + return fmt.Errorf("archive chain id %q does not match genesis chain id %q", m.ChainID, t.chainID) + } + return nil +} + +// requireFreeSpace refuses when the filesystem of dir has less than size bytes free. It is a lower +// limit: the import also writes a WAL and compacts. +func requireFreeSpace(dir string, size int64) error { + var st unix.Statfs_t + if err := unix.Statfs(dir, &st); err != nil { + return fmt.Errorf("statfs %s: %w", dir, err) + } + free := st.Bavail * uint64(st.Bsize) //nolint:gosec // block size is positive. + if uint64(size) > free { //nolint:gosec // size is a sum of validated non-negative sizes. + return fmt.Errorf("the archive holds %d bytes, and %s has %d bytes free", size, dir, free) + } + return nil +} + +// writeFileAtomic writes data to path through a fsynced temporary file. +func writeFileAtomic(path string, data []byte) error { + tmp := path + ".tmp" + f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) //nolint:gosec // path is from the node layout. + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + return syncPath(filepath.Dir(path)) +} + +// stagedArchive is what import installs from an archive that stageArchive checked. +type stagedArchive struct { + // proof is the state sync proof file of the checkpoint. + proof []byte + // epochs is the epoch registry of the checkpoint. + epochs node.CheckpointEpochs + // appHash is the app hash of the checkpoint that the AppQC in proof signs. + appHash [32]byte +} + +// stageArchive imports the archive into staging, opens the epoch registry of that checkpoint, and +// verifies the state sync proof file of the checkpoint against that registry. +func stageArchive( + ctx context.Context, + archive *archiveReader, + m *manifest, + genesis node.EpochGenesis, + staging string, + stderr io.Writer, +) (stagedArchive, error) { + proof, err := readArchivedProof(archive, m.CheckpointHeight) + if err != nil { + return stagedArchive{}, err + } + imported, err := importArchivedRows(ctx, archive, m, filepath.Join(staging, flatkvPrefix)) + if err != nil { + return stagedArchive{}, err + } + _, _ = fmt.Fprintf(stderr, "FlatKV import complete: %d keys, %d bytes in %s\n", + m.RowCount, m.StateBytes, imported.Round(time.Millisecond)) + checkpoint := filepath.Join(staging, flatkvPrefix, flatkv.SnapshotDirName(m.CheckpointHeight)) + if err := flatkv.VerifySnapshotDir(checkpoint, m.CheckpointHeight); err != nil { + return stagedArchive{}, fmt.Errorf("verify the imported checkpoint: %w", err) + } + if err := requireImportedStats(checkpoint, m); err != nil { + return stagedArchive{}, err + } + epochs, err := node.OpenCheckpointEpochs(genesis, checkpoint, m.CheckpointHeight) + if err != nil { + return stagedArchive{}, fmt.Errorf("open the epoch registry: %w", err) + } + appHash, err := node.VerifyStateSyncProof(epochs, m.CheckpointHeight, proof) + if err != nil { + return stagedArchive{}, fmt.Errorf("verify the archived state sync proof file: %w", err) + } + return stagedArchive{proof: proof, epochs: epochs, appHash: appHash}, nil +} + +// requireImportedStats returns an error unless the metadata of the imported checkpoint records the +// row count, byte count and state hash of m. +func requireImportedStats(checkpoint string, m *manifest) error { + got, err := flatkv.SnapshotDirStats(checkpoint, m.CheckpointHeight) + if err != nil { + return fmt.Errorf("read the stats of the imported checkpoint: %w", err) + } + if want := m.stats(); got != want { + return fmt.Errorf("the imported checkpoint holds %s, the manifest records %s", formatStats(got), formatStats(want)) + } + return nil +} + +// readArchivedProof returns the archived state sync proof file of height, after it checks that the +// file decodes as the road block and AppQC of height. +func readArchivedProof(archive *archiveReader, height int64) ([]byte, error) { + data, err := archive.readProof(height) + if err != nil { + return nil, err + } + if _, err := statesyncproof.Decode(data, atypes.RoadIndex(height)); err != nil { //nolint:gosec // heights are positive. + return nil, fmt.Errorf("decode the archived state sync proof file: %w", err) + } + return data, nil +} + +func importArchivedRows(ctx context.Context, archive *archiveReader, m *manifest, root string) (time.Duration, error) { + start := time.Now() + cfg := flatkvconfig.DefaultConfig() + cfg.DataDir = root + store, err := flatkv.NewCommitStore(ctx, cfg, nil) + if err != nil { + return 0, fmt.Errorf("create FlatKV store for import: %w", err) + } + closed := false + defer func() { + if !closed { + _ = store.Close() + } + }() + if err := store.LoadLatest(); err != nil { + return 0, fmt.Errorf("open FlatKV store for import: %w", err) + } + importer, err := store.Importer(m.CheckpointHeight) + if err != nil { + return 0, fmt.Errorf("create FlatKV importer: %w", err) + } + if err := archive.importRows(m, importer); err != nil { + if kvi, ok := importer.(*flatkv.KVImporter); ok { + _ = kvi.Abort(err) + } + return 0, err + } + if err := importer.Close(); err != nil { + return 0, fmt.Errorf("finish FlatKV import: %w", err) + } + if err := store.Close(); err != nil { + return 0, fmt.Errorf("close FlatKV store after import: %w", err) + } + closed = true + return time.Since(start), nil +} + +// clearEarlierImport removes the app hashes and then the FlatKV data of an earlier import. +func clearEarlierImport(layout node.AutobahnLayout, lock *flatkv.RootLock, stop stepHook) error { + if err := removeDir(layout.AppHashVaultDir); err != nil { + return fmt.Errorf("clear the app-hash vault of an earlier import: %w", err) + } + if err := stop(importStepVaultCleared); err != nil { + return err + } + if err := lock.Clear(); err != nil { + return fmt.Errorf("clear the FlatKV data of an earlier import: %w", err) + } + if err := stop(importStepFlatKVCleared); err != nil { + return err + } + return nil +} + +// installArchive moves the staged checkpoint into place. +func installArchive(lock *flatkv.RootLock, staging string, m *manifest) error { + if err := lock.InstallSnapshot(filepath.Join(staging, flatkvPrefix, flatkv.SnapshotDirName(m.CheckpointHeight)), m.CheckpointHeight); err != nil { + return fmt.Errorf("install the checkpoint: %w", err) + } + return nil +} diff --git a/cmd/seid/cmd/statesync/import_test.go b/cmd/seid/cmd/statesync/import_test.go new file mode 100644 index 0000000000..859ec0a549 --- /dev/null +++ b/cmd/seid/cmd/statesync/import_test.go @@ -0,0 +1,242 @@ +package statesync + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config" + "github.com/sei-protocol/sei-chain/sei-tendermint/crypto/ed25519" + "github.com/sei-protocol/sei-chain/sei-tendermint/node" +) + +func TestExportImportRoundTrip(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + h := p.commit(t, 2) + + t.Run("through files", func(t *testing.T) { + archive, got := p.exportArchive(t) + require.Equal(t, h, got) + dst := freshHome(t) + var stderr bytes.Buffer + require.NoError(t, runImport(t.Context(), dst, archive, nil, &stderr)) + require.Equal(t, h, heightLine(t, stderr.String())) + requireRestored(t, dst, p, h) + }) + t.Run("through stdout and stdin", func(t *testing.T) { + var archive, exportErr bytes.Buffer + require.NoError(t, runExport(t.Context(), p.target, exportOptions{out: "-"}, &archive, &exportErr)) + require.Equal(t, h, heightLine(t, exportErr.String())) + dst := freshHome(t) + require.NoError(t, runImport(t.Context(), dst, "-", &archive, &bytes.Buffer{})) + requireRestored(t, dst, p, h) + }) +} + +func TestImportRefusals(t *testing.T) { + p := newProducer(t, 2) + p.commit(t, 1) + archive, _ := p.exportArchive(t) + data, err := os.ReadFile(archive) //nolint:gosec // test path. + require.NoError(t, err) + truncated := filepath.Join(t.TempDir(), "truncated.tar.zst") + require.NoError(t, os.WriteFile(truncated, data[:len(data)/2], 0o600)) + + writeFile := func(p string) { + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o750)) + require.NoError(t, os.WriteFile(p, []byte("x"), 0o600)) + } + cases := []struct { + name string + setup func(tg *target) + from string + wantErr func(l node.AutobahnLayout) string + }{ + { + name: "a validator home", + setup: func(tg *target) { tg.mode = tmcfg.ModeValidator }, + wantErr: func(node.AutobahnLayout) string { return "validator" }, + }, + { + name: "another chain", + setup: func(tg *target) { tg.chainID = "chain-b" }, + wantErr: func(node.AutobahnLayout) string { return "chain id" }, + }, + { + name: "block data", + setup: func(tg *target) { writeFile(filepath.Join(tg.layout.BlockDBDirs[0], "x")) }, + wantErr: func(l node.AutobahnLayout) string { return "remove " + l.BlockDBDirs[0] }, + }, + { + name: "block data with a marker", + setup: func(tg *target) { + writeFile(filepath.Join(tg.layout.BlockDBDirs[0], "x")) + writeFile(tg.layout.RestoreMarkerFile) + }, + wantErr: func(l node.AutobahnLayout) string { return "remove " + l.BlockDBDirs[0] }, + }, + { + name: "state store data", + setup: func(tg *target) { writeFile(filepath.Join(tg.layout.StateStoreDir, "x")) }, + wantErr: func(l node.AutobahnLayout) string { return l.StateStoreDir }, + }, + { + name: "FlatKV data without a marker", + setup: func(tg *target) { writeFile(filepath.Join(tg.layout.FlatKVDir, "snapshot-x", "y")) }, + wantErr: func(l node.AutobahnLayout) string { return "remove " + l.FlatKVDir }, + }, + { + name: "app hashes without a marker", + setup: func(tg *target) { writeFile(filepath.Join(tg.layout.AppHashVaultDir, "x")) }, + wantErr: func(l node.AutobahnLayout) string { return "remove " + l.AppHashVaultDir }, + }, + { + name: "a truncated archive", + from: truncated, + wantErr: func(node.AutobahnLayout) string { return "" }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tg := freshHome(t) + if tc.setup != nil { + tc.setup(&tg) + } + from := archive + if tc.from != "" { + from = tc.from + } + before := treeOf(t, tg.layout.PersistentStateDir) + err := runImport(t.Context(), tg, from, nil, &bytes.Buffer{}) + require.Error(t, err) + require.ErrorContains(t, err, tc.wantErr(tg.layout)) + require.Equal(t, before, treeOf(t, tg.layout.PersistentStateDir), "import changed the home") + }) + } + + t.Run("a running node", func(t *testing.T) { + tg := freshHome(t) + s := openStore(t, tg.layout.FlatKVDir, 2) + defer func() { require.NoError(t, s.Close()) }() + err := runImport(t.Context(), tg, archive, nil, &bytes.Buffer{}) + require.ErrorContains(t, err, "is a node running on this home") + require.NoFileExists(t, tg.layout.RestoreMarkerFile) + }) +} + +func TestImportReplacesAnEarlierImportThatHasNotSeeded(t *testing.T) { + p := newProducer(t, 2) + h1 := p.commit(t, 1) + first, _ := p.exportArchive(t) + p.commit(t, 2) + h2 := p.commit(t, 3) + second, _ := p.exportArchive(t) + + dst := freshHome(t) + require.NoError(t, runImport(t.Context(), dst, first, nil, &bytes.Buffer{})) + requireRestored(t, dst, p, h1) + require.NoError(t, runImport(t.Context(), dst, second, nil, &bytes.Buffer{})) + requireRestored(t, dst, p, h2) + require.NoError(t, runImport(t.Context(), dst, second, nil, &bytes.Buffer{}), "a rerun of a finished import") + requireRestored(t, dst, p, h2) +} + +func TestImportRefusesAnArchiveThatFailsItsChecks(t *testing.T) { + p := newProducer(t, 2) + h := p.commit(t, 1) + p.closeStore(t) + + cases := []struct { + name string + change func(t *testing.T, m *manifest, entries *[]tarEntry) + wantMsg string + }{ + { + name: "a state sync proof of another height", + change: func(t *testing.T, _ *manifest, entries *[]tarEntry) { + require.True(t, replaceEntry(*entries, proofArchivePath(h), stateSyncProofAt(t, h+1))) + }, + wantMsg: "decode the archived state sync proof file", + }, + { + name: "a state sync proof signed by another validator", + change: func(t *testing.T, _ *manifest, entries *[]tarEntry) { + other := atypes.SecretKeyFromED25519(ed25519.TestSecretKey([]byte("another validator"))) + require.True(t, replaceEntry(*entries, proofArchivePath(h), stateSyncProofSignedBy(t, other, h))) + }, + wantMsg: "verify the archived state sync proof file", + }, + { + name: "a state sync proof that does not decode", + change: func(t *testing.T, _ *manifest, entries *[]tarEntry) { + require.True(t, replaceEntry(*entries, proofArchivePath(h), []byte("not a proof"))) + }, + wantMsg: "decode the archived state sync proof file", + }, + { + name: "a row that is not in ascending order", + change: func(t *testing.T, m *manifest, entries *[]tarEntry) { + key := []byte("bank/\x00") + value := []byte("x") + *entries = appendLogicalRow(*entries, key, value) + m.RowCount++ + m.StateBytes += int64(len(key) + len(value)) + }, + wantMsg: "keys must be in ascending order", + }, + { + name: "a row count that does not match the manifest", + change: func(t *testing.T, m *manifest, entries *[]tarEntry) { + m.RowCount++ + }, + wantMsg: "manifest records", + }, + { + name: "a state hash that does not match the rows", + change: func(t *testing.T, m *manifest, _ *[]tarEntry) { + m.StateHash[0] ^= 1 + }, + wantMsg: "the imported checkpoint holds", + }, + { + name: "a row value changed at the same length", + change: func(t *testing.T, _ *manifest, entries *[]tarEntry) { + for i := range *entries { + if (*entries)[i].name == rowChunkPath(0) { + chunk := bytes.Clone((*entries)[i].content) + chunk[len(chunk)-1] ^= 1 + (*entries)[i].content = chunk + return + } + } + t.Fatal("the archive has no row chunk") + }, + wantMsg: "the imported checkpoint holds", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + archive := p.archiveWithChange(t, h, func(m *manifest, entries *[]tarEntry) { tc.change(t, m, entries) }) + tg := freshHome(t) + before := treeOf(t, tg.layout.PersistentStateDir) + err := runImport(t.Context(), tg, archive, nil, &bytes.Buffer{}) + require.Error(t, err) + require.ErrorContains(t, err, tc.wantMsg) + require.Equal(t, before, treeOf(t, tg.layout.PersistentStateDir), "import changed the home") + }) + } + + t.Run("an unchanged archive", func(t *testing.T) { + archive := p.archiveWithChange(t, h, nil) + tg := freshHome(t) + var stderr bytes.Buffer + require.NoError(t, runImport(t.Context(), tg, archive, nil, &stderr)) + require.Contains(t, stderr.String(), "FlatKV import complete") + requireRestored(t, tg, p, h) + }) +} diff --git a/cmd/seid/cmd/statesync/manifest.go b/cmd/seid/cmd/statesync/manifest.go new file mode 100644 index 0000000000..436c7b89bc --- /dev/null +++ b/cmd/seid/cmd/statesync/manifest.go @@ -0,0 +1,133 @@ +package statesync + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/statesyncproof" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" +) + +const ( + formatVersion = 2 + manifestName = "manifest.json" + maxManifestSize = 16 << 20 + maxProofSize = 16 << 20 + maxRowChunkSize = 128 << 20 + maxStateKeySize = 64 << 10 + maxStateValueSize = 64 << 20 + flatkvPrefix = "flatkv" + rowPrefix = "flatkv/rows" + proofPrefix = "proof" + rowChunkNameWidth = 8 +) + +// manifest is the first entry of an archive. It describes the logical row stream that follows it. +type manifest struct { + FormatVersion int `json:"format_version"` + ChainID string `json:"chain_id"` + CheckpointHeight int64 `json:"checkpoint_height"` + CreatedAt time.Time `json:"created_at"` + ProducerVersion string `json:"producer_version"` + RowCount int64 `json:"row_count"` + StateBytes int64 `json:"state_bytes"` + // StateHash is the checksum of the store-wide FlatKV LtHash of the checkpoint. + StateHash hexHash `json:"state_hash"` +} + +// hexHash is a 32-byte hash that encodes as hex in JSON. +type hexHash [32]byte + +func (h hexHash) MarshalText() ([]byte, error) { + return []byte(hex.EncodeToString(h[:])), nil +} + +func (h *hexHash) UnmarshalText(text []byte) error { + if len(text) != hex.EncodedLen(len(h)) { + return fmt.Errorf("hash is %d hex digits, want %d", len(text), hex.EncodedLen(len(h))) + } + _, err := hex.Decode(h[:], text) + return err +} + +// stats returns the FlatKV snapshot stats that m records. +func (m *manifest) stats() flatkv.SnapshotStats { + return flatkv.SnapshotStats{RowCount: m.RowCount, Bytes: m.StateBytes, StateHash: [32]byte(m.StateHash)} +} + +// formatStats describes FlatKV snapshot stats in an error message. +func formatStats(s flatkv.SnapshotStats) string { + return fmt.Sprintf("%d rows, %d bytes and state hash %x", s.RowCount, s.Bytes, s.StateHash) +} + +// validate checks the rules that both the writer and the reader rely on. +func (m *manifest) validate() error { + if m.FormatVersion != formatVersion { + return fmt.Errorf("format_version %d, want %d", m.FormatVersion, formatVersion) + } + if m.ChainID == "" { + return errors.New("chain_id is empty") + } + if m.CheckpointHeight < 1 { + return fmt.Errorf("checkpoint_height %d is below 1", m.CheckpointHeight) + } + if m.RowCount < 0 { + return fmt.Errorf("row_count %d is below 0", m.RowCount) + } + if m.StateBytes < 0 { + return fmt.Errorf("state_bytes %d is below 0", m.StateBytes) + } + if m.StateHash == (hexHash{}) { + return errors.New("state_hash is missing") + } + return nil +} + +// proofArchivePath returns the archive path of the state sync proof file for height. +func proofArchivePath(height int64) string { + return path.Join(proofPrefix, statesyncproof.FileName(atypes.RoadIndex(height))) //nolint:gosec // heights are positive. +} + +// rowChunkPath returns the archive path of the state row chunk at index. +func rowChunkPath(index int) string { + return path.Join(rowPrefix, fmt.Sprintf("%0*d", rowChunkNameWidth, index)) +} + +// validateArchivePath accepts only canonical relative paths. +func validateArchivePath(p string) error { + if p == "" || path.Clean(p) != p || path.IsAbs(p) || strings.Contains(p, "\\") { + return fmt.Errorf("path %q is not canonical", p) + } + parts := strings.Split(p, "/") + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return fmt.Errorf("path %q is not canonical", p) + } + } + return nil +} + +// decodeManifest decodes data as a manifest, refusing unknown fields and trailing data. +func decodeManifest(data []byte) (*manifest, error) { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + var m manifest + if err := dec.Decode(&m); err != nil { + return nil, fmt.Errorf("decode manifest: %w", err) + } + if err := dec.Decode(&json.RawMessage{}); !errors.Is(err, io.EOF) { + return nil, errors.New("decode manifest: trailing data") + } + if err := m.validate(); err != nil { + return nil, fmt.Errorf("manifest: %w", err) + } + return &m, nil +} diff --git a/cmd/seid/cmd/statesync/pack.go b/cmd/seid/cmd/statesync/pack.go new file mode 100644 index 0000000000..d50da52352 --- /dev/null +++ b/cmd/seid/cmd/statesync/pack.go @@ -0,0 +1,190 @@ +package statesync + +import ( + "archive/tar" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "time" + + "github.com/klauspost/compress/zstd" + + cosmosversion "github.com/sei-protocol/sei-chain/sei-cosmos/version" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" +) + +const ( + // encoderConcurrency is the number of zstd encoder goroutines. + encoderConcurrency = 2 + // archiveWindowSize is the zstd window of an archive, and the largest window import accepts. + archiveWindowSize = 8 << 20 +) + +// buildManifest returns the manifest that describes a logical FlatKV row stream. +func buildManifest(chainID string, height int64, stats flatkv.SnapshotStats) (*manifest, error) { + m := &manifest{ + FormatVersion: formatVersion, + ChainID: chainID, + CheckpointHeight: height, + CreatedAt: time.Now().UTC().Truncate(time.Second), + ProducerVersion: producerVersion(), + RowCount: stats.RowCount, + StateBytes: stats.Bytes, + StateHash: hexHash(stats.StateHash), + } + if err := m.validate(); err != nil { + return nil, fmt.Errorf("manifest: %w", err) + } + return m, nil +} + +func producerVersion() string { + if cosmosversion.Version == "" { + return "unknown" + } + return cosmosversion.Version +} + +// writeArchive writes the manifest, proof and FlatKV logical row chunks as tar in zstd. +func writeArchive(ctx context.Context, w io.Writer, m *manifest, proof []byte, checkpoint string) error { + // A small encoder concurrency keeps packing from taking every core of a running node. + zw, err := zstd.NewWriter( + w, + zstd.WithEncoderConcurrency(encoderConcurrency), + zstd.WithWindowSize(archiveWindowSize), + zstd.WithEncoderCRC(true), + ) + if err != nil { + return err + } + defer func() { _ = zw.Close() }() + tw := tar.NewWriter(zw) + data, err := json.Marshal(m) + if err != nil { + return err + } + if err := tw.WriteHeader(&tar.Header{Name: manifestName, Mode: 0o600, Size: int64(len(data)), Typeflag: tar.TypeReg, ModTime: m.CreatedAt}); err != nil { + return err + } + if _, err := tw.Write(data); err != nil { + return err + } + if err := writeProofEntry(tw, m, proof); err != nil { + return err + } + rowWriter := newRowChunkWriter(tw, m.CreatedAt) + stats, err := flatkv.ExportSnapshotDir(ctx, checkpoint, m.CheckpointHeight, rowWriter.add) + if err != nil { + return fmt.Errorf("export FlatKV rows: %w", err) + } + if err := rowWriter.close(); err != nil { + return err + } + if stats != m.stats() { + return fmt.Errorf("exported %s, manifest records %s", formatStats(stats), formatStats(m.stats())) + } + if err := tw.Close(); err != nil { + return fmt.Errorf("close tar: %w", err) + } + if err := zw.Close(); err != nil { + return fmt.Errorf("close zstd: %w", err) + } + return nil +} + +func writeProofEntry(tw *tar.Writer, m *manifest, proof []byte) error { + if len(proof) > maxProofSize { + return fmt.Errorf("state sync proof is %d bytes, more than %d", len(proof), maxProofSize) + } + if err := tw.WriteHeader(&tar.Header{ + Name: proofArchivePath(m.CheckpointHeight), + Mode: 0o600, + Size: int64(len(proof)), + Typeflag: tar.TypeReg, + ModTime: m.CreatedAt, + }); err != nil { + return err + } + _, err := tw.Write(proof) + return err +} + +type rowChunkWriter struct { + tw *tar.Writer + modTime time.Time + chunk []byte + index int +} + +func newRowChunkWriter(tw *tar.Writer, modTime time.Time) *rowChunkWriter { + return &rowChunkWriter{ + tw: tw, + modTime: modTime, + chunk: make([]byte, 0, maxRowChunkSize), + } +} + +func (w *rowChunkWriter) add(key, value []byte) error { + recordLen := recordEncodedLen(len(key), len(value)) + if recordLen > maxRowChunkSize { + return fmt.Errorf("FlatKV row is %d bytes, more than the %d-byte row chunk limit", recordLen, maxRowChunkSize) + } + if len(w.chunk) > 0 && len(w.chunk)+recordLen > maxRowChunkSize { + if err := w.flush(); err != nil { + return err + } + } + w.appendUvarint(uint64(len(key))) + w.chunk = append(w.chunk, key...) + w.appendUvarint(uint64(len(value))) + w.chunk = append(w.chunk, value...) + if len(w.chunk) == maxRowChunkSize { + return w.flush() + } + return nil +} + +func (w *rowChunkWriter) close() error { + if len(w.chunk) == 0 { + return nil + } + return w.flush() +} + +func (w *rowChunkWriter) flush() error { + if len(w.chunk) == 0 { + return nil + } + if err := w.tw.WriteHeader(&tar.Header{ + Name: rowChunkPath(w.index), + Mode: 0o600, + Size: int64(len(w.chunk)), + Typeflag: tar.TypeReg, + ModTime: w.modTime, + }); err != nil { + return err + } + if _, err := w.tw.Write(w.chunk); err != nil { + return err + } + w.index++ + w.chunk = w.chunk[:0] + return nil +} + +func recordEncodedLen(keyLen, valueLen int) int { + return uvarintLen(uint64(keyLen)) + keyLen + uvarintLen(uint64(valueLen)) + valueLen //nolint:gosec // lengths come from byte slices. +} + +func uvarintLen(v uint64) int { + var buf [10]byte + return binary.PutUvarint(buf[:], v) +} + +func (w *rowChunkWriter) appendUvarint(v uint64) { + var buf [10]byte + n := binary.PutUvarint(buf[:], v) + w.chunk = append(w.chunk, buf[:n]...) +} diff --git a/giga/config/config.go b/giga/config/config.go index ffb2389a43..925c405334 100644 --- a/giga/config/config.go +++ b/giga/config/config.go @@ -132,7 +132,7 @@ type RPCConfig struct { // StateSyncConfig is the [giga.state_sync] sub-table: whether this node uses giga state sync. type StateSyncConfig struct { - // Enabled allows the state-sync-snapshot create and restore commands and the seed of a restored + // Enabled allows the statesync export and import commands and the seed of a restored // home on start. It also runs a background task that writes a state sync proof file for each // retained FlatKV checkpoint; it does not create checkpoints or snapshots. Enabled bool `mapstructure:"enabled"` @@ -644,7 +644,7 @@ max_state_override_accounts = {{ .Giga.RPC.MaxStateOverrideAccounts }} max_state_override_slots = {{ .Giga.RPC.MaxStateOverrideSlots }} [giga.state_sync] -# enabled allows the state-sync-snapshot create and restore commands and the seed of a restored +# enabled allows the statesync export and import commands and the seed of a restored # home on start. It also runs a background task that writes a state sync proof file for each # retained FlatKV checkpoint. It does not create checkpoints or snapshots. enabled = {{ .Giga.StateSync.Enabled }} diff --git a/sei-tendermint/autobahn/statesyncproof/proof.go b/sei-tendermint/autobahn/statesyncproof/proof.go index b4f33d896e..36d860f711 100644 --- a/sei-tendermint/autobahn/statesyncproof/proof.go +++ b/sei-tendermint/autobahn/statesyncproof/proof.go @@ -40,6 +40,18 @@ func (p *Proof) AppQC() *types.AppQC { return p.appQC } +// Verify returns an error unless the road block verifies against ep and a quorum of the committee +// of ep signed the AppQC. +func (p *Proof) Verify(ep *types.Epoch) error { + if err := p.roadBlock.Verify(ep); err != nil { + return fmt.Errorf("road block: %w", err) + } + if err := p.appQC.Verify(ep.Committee()); err != nil { + return fmt.Errorf("AppQC: %w", err) + } + return nil +} + // FileName returns the canonical state sync proof file name for index. func FileName(index types.RoadIndex) string { return fmt.Sprintf("%0*d%s", heightDigits, index, fileSuffix) @@ -97,8 +109,7 @@ func frame(msg *pb.StateSyncProof) []byte { // Decode decodes a state sync proof file for road wantIndex. It checks the // file layout and that the AppQC is for the road and epoch of the CommitQC, but -// not signatures: the caller must verify the RoadBlock and the AppQC against the -// epoch before it trusts the proof. +// not signatures: the caller must call Verify before it trusts the proof. func Decode(data []byte, wantIndex types.RoadIndex) (*Proof, error) { if len(data) < headerLen { return nil, fmt.Errorf("truncated state sync proof file header") diff --git a/sei-tendermint/autobahn/statesyncproof/proof_test.go b/sei-tendermint/autobahn/statesyncproof/proof_test.go index 4eff5ff5ca..0081623dce 100644 --- a/sei-tendermint/autobahn/statesyncproof/proof_test.go +++ b/sei-tendermint/autobahn/statesyncproof/proof_test.go @@ -2,6 +2,7 @@ package statesyncproof import ( "path/filepath" + "slices" "testing" "google.golang.org/protobuf/encoding/protowire" @@ -154,14 +155,72 @@ func TestParseFileNameRejectsNonCanonicalNames(t *testing.T) { } } +func TestVerifyAcceptsTheEpochThatSignedIt(t *testing.T) { + epoch, roadBlock, appQC := testSignedProof(t, utils.TestRng()) + proof := decodeProof(t, roadBlock, appQC) + + require.NoError(t, proof.Verify(epoch)) +} + +func TestVerifyRejectsAnotherCommittee(t *testing.T) { + rng := utils.TestRng() + epoch, roadBlock, appQC := testSignedProof(t, rng) + proof := decodeProof(t, roadBlock, appQC) + other, _ := types.GenCommittee(rng, 4) + + require.Error(t, proof.Verify(types.NewEpoch(epoch.EpochIndex(), epoch.RoadRange(), other))) +} + +func TestVerifyRejectsAnAppQCOfAnotherCommittee(t *testing.T) { + rng := utils.TestRng() + epoch, roadBlock, appQC := testSignedProof(t, rng) + proof := decodeProof(t, roadBlock, data.TestAppQC(testKeys(t, rng), appQC.Proposal())) + + require.Error(t, proof.Verify(epoch)) +} + +func TestVerifyRejectsLaneBlocksThatDoNotMatchTheQC(t *testing.T) { + rng := utils.TestRng() + epoch, roadBlock, appQC := testSignedProof(t, rng) + laneBlocks := slices.Clone(roadBlock.LaneBlocks()) + laneBlocks[0] = types.GenBlock(rng) + encoded := frame(&pb.StateSyncProof{ + CommitQc: types.FullCommitQCConv.Encode(roadBlock.QC()), + LaneBlocks: types.BlockConv.EncodeSlice(laneBlocks), + AppQc: types.AppQCConv.Encode(appQC), + }) + proof, err := Decode(encoded, roadBlock.QC().Index()) + require.NoError(t, err) + + require.Error(t, proof.Verify(epoch)) +} + +// decodeProof encodes roadBlock and appQC as a state sync proof file and decodes it. +func decodeProof(t *testing.T, roadBlock *types.RoadBlock, appQC *types.AppQC) *Proof { + t.Helper() + encoded, err := Encode(roadBlock, appQC) + require.NoError(t, err) + proof, err := Decode(encoded, roadBlock.QC().Index()) + require.NoError(t, err) + return proof +} + func testProof(t *testing.T, rng utils.Rng) (*types.RoadBlock, *types.AppQC) { + t.Helper() + _, roadBlock, appQC := testSignedProof(t, rng) + return roadBlock, appQC +} + +// testSignedProof returns a road block and an AppQC of it, both signed by the committee of the +// returned epoch. +func testSignedProof(t *testing.T, rng utils.Rng) (*types.Epoch, *types.RoadBlock, *types.AppQC) { t.Helper() committee, keys := types.GenCommittee(rng, 4) epoch := types.GenEpochWithCommittee(rng, committee) qc, blocks := data.TestCommitQC(rng, epoch, keys, utils.None[*types.CommitQC]()) roadBlock := types.NewRoadBlock(qc, blocks) proposal := types.NewAppProposal(qc.QC().View(), []byte{1, 2, 3}) - return roadBlock, data.TestAppQC(keys, proposal) + return epoch, roadBlock, data.TestAppQC(keys, proposal) } func testKeys(t *testing.T, rng utils.Rng) []types.SecretKey { diff --git a/sei-tendermint/internal/p2p/state_sync_proof.go b/sei-tendermint/internal/p2p/state_sync_proof.go new file mode 100644 index 0000000000..b34ce1a17e --- /dev/null +++ b/sei-tendermint/internal/p2p/state_sync_proof.go @@ -0,0 +1,30 @@ +package p2p + +import ( + "fmt" + + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/statesyncproof" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/autobahn/epoch" +) + +// VerifyStateSyncProof decodes the state sync proof file data of road and verifies it against +// the epoch of road in registry. It returns the proof and the app hash that its AppQC signs. +func VerifyStateSyncProof(registry *epoch.Registry, data []byte, road atypes.RoadIndex) (*statesyncproof.Proof, [32]byte, error) { + proof, err := statesyncproof.Decode(data, road) + if err != nil { + return nil, [32]byte{}, fmt.Errorf("decode the state sync proof of road %d: %w", road, err) + } + ep, err := registry.EpochAt(road) + if err != nil { + return nil, [32]byte{}, fmt.Errorf("the epoch registry has no epoch for road %d: %w", road, err) + } + if err := proof.Verify(ep); err != nil { + return nil, [32]byte{}, fmt.Errorf("verify the state sync proof of road %d against epoch %d: %w", road, ep.EpochIndex(), err) + } + appHash := proof.AppQC().Proposal().AppHash() + if len(appHash) != 32 { + return nil, [32]byte{}, fmt.Errorf("the AppQC of road %d signs an app hash of %d bytes, want 32", road, len(appHash)) + } + return proof, [32]byte(appHash), nil +} diff --git a/sei-tendermint/node/restore_anchor.go b/sei-tendermint/node/restore_anchor.go new file mode 100644 index 0000000000..6047b61ec1 --- /dev/null +++ b/sei-tendermint/node/restore_anchor.go @@ -0,0 +1,50 @@ +package node + +import ( + "errors" + "fmt" + + "github.com/sei-protocol/sei-chain/giga/apphash" + "github.com/sei-protocol/sei-chain/giga/apphash/vault" + seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" +) + +// VerifyStateSyncProof returns the app hash of height that the state sync proof file data +// carries, after it verifies the road block and the AppQC of height against epochs. +func VerifyStateSyncProof(epochs CheckpointEpochs, height int64, data []byte) ([32]byte, error) { + if height < 1 { + return [32]byte{}, fmt.Errorf("checkpoint height %d is not positive", height) + } + _, appHash, err := p2p.VerifyStateSyncProof(epochs.registry, data, atypes.RoadIndex(height)) //nolint:gosec // height is positive. + return appHash, err +} + +// StoreRestoreAnchor stores appHash as the restore anchor of height, the first record of the +// empty app-hash vault of layout, on the chain that genesis starts. +func StoreRestoreAnchor(layout AutobahnLayout, genesis EpochGenesis, height int64, appHash [32]byte) (retErr error) { + activationHeight, err := gigaActivationHeight(genesis.GenDoc) + if err != nil { + return err + } + if height < 1 || uint64(height) < activationHeight { //nolint:gosec // height is positive. + return fmt.Errorf("height %d is below the giga activation height %d, so it has no app hash", height, activationHeight) + } + storageConfig, err := seidbconfig.AutobahnStorageConfig(layout.PersistentStateDir) + if err != nil { + return err + } + cfg := *storageConfig.AppHashBuilderConfig + cfg.HashVaultConfig.Path = layout.AppHashVaultDir + hashVault, err := vault.NewStandardHashVault(cfg.HashVaultConfig, cfg.ChainID, activationHeight) + if err != nil { + return fmt.Errorf("open the app-hash vault: %w", err) + } + defer func() { retErr = errors.Join(retErr, hashVault.Close()) }() + anchor := apphash.NewRestoreAnchorAppHashData(cfg.ChainID, uint64(height), appHash) //nolint:gosec // height is positive. + if err := hashVault.Append([]*apphash.AppHashData{anchor}); err != nil { + return fmt.Errorf("store the restore anchor of height %d: %w", height, err) + } + return nil +} diff --git a/sei-tendermint/node/restore_anchor_test.go b/sei-tendermint/node/restore_anchor_test.go new file mode 100644 index 0000000000..a0949a345a --- /dev/null +++ b/sei-tendermint/node/restore_anchor_test.go @@ -0,0 +1,126 @@ +package node + +import ( + "testing" + + "github.com/sei-protocol/sei-chain/giga/apphash/builder" + "github.com/sei-protocol/sei-chain/giga/apphash/vault" + gigaconfig "github.com/sei-protocol/sei-chain/giga/config" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require" + "github.com/sei-protocol/sei-chain/sei-tendermint/types" +) + +// TestStoreRestoreAnchorIsPublishedBySetupAtItsHeight opens the app hash builder with the +// storage config of node startup, and requires setup at the anchor's height to publish it. +func TestStoreRestoreAnchorIsPublishedBySetupAtItsHeight(t *testing.T) { + conf := layoutConfig(t, t.TempDir()) + storage := gigaconfig.DefaultConfig.Storage + layout, err := ResolveAutobahnLayout(conf, storage, genesisAt(3)) + require.NoError(t, err) + genesis, err := ResolveEpochGenesis(conf, genesisAt(3)) + require.NoError(t, err) + appHash := [32]byte{1, 2, 3} + require.NoError(t, StoreRestoreAnchor(layout, genesis, 10, appHash)) + + fc, _, err := loadAutobahnCommittee(conf.AutobahnConfigFile) + require.NoError(t, err) + _, storageConfig, err := autobahnStorageConfig(conf.RootDir, conf.Mode, fc, storage, genesisAt(3)) + require.NoError(t, err) + cfg := *storageConfig.AppHashBuilderConfig + hashVault, err := vault.NewStandardHashVault(cfg.HashVaultConfig, cfg.ChainID, cfg.GigaActivationHeight) + require.NoError(t, err) + b, err := builder.NewStandardAppHashBuilder(t.Context(), cfg, hashVault) + require.NoError(t, err) + require.NoError(t, b.SetupComplete(t.Context(), 10)) + got, err := b.GetAppHash(t.Context(), 10) + require.NoError(t, err) + require.True(t, got.IsRestoreAnchor()) + require.Equal(t, appHash, got.AppHash()) + require.NoError(t, b.Close()) + require.NoError(t, hashVault.Close()) +} + +func TestStoreRestoreAnchorRefusesAVaultThatHoldsARecord(t *testing.T) { + conf := layoutConfig(t, t.TempDir()) + layout, err := ResolveAutobahnLayout(conf, gigaconfig.DefaultConfig.Storage, genesisAt(1)) + require.NoError(t, err) + genesis, err := ResolveEpochGenesis(conf, genesisAt(1)) + require.NoError(t, err) + require.NoError(t, StoreRestoreAnchor(layout, genesis, 10, [32]byte{1})) + require.Error(t, StoreRestoreAnchor(layout, genesis, 10, [32]byte{1})) +} + +func TestStoreRestoreAnchorRefusesAHeightBelowTheGenesis(t *testing.T) { + conf := layoutConfig(t, t.TempDir()) + layout, err := ResolveAutobahnLayout(conf, gigaconfig.DefaultConfig.Storage, genesisAt(5)) + require.NoError(t, err) + genesis, err := ResolveEpochGenesis(conf, genesisAt(5)) + require.NoError(t, err) + require.Error(t, StoreRestoreAnchor(layout, genesis, 4, [32]byte{1})) +} + +// proofGenesis returns a chain of keys as its genesis validators. +func proofGenesis(keys []atypes.SecretKey) EpochGenesis { + validators := make([]atypes.PublicKey, 0, len(keys)) + for _, k := range keys { + validators = append(validators, k.Public()) + } + return EpochGenesis{GenDoc: &types.GenesisDoc{ChainID: "chain-a", InitialHeight: 1}, Validators: validators} +} + +// genesisCheckpointEpochs returns the epochs at height of a checkpoint with no epoch_registry row +// on the chain that genesis starts. +func genesisCheckpointEpochs(t *testing.T, genesis EpochGenesis, height int64) CheckpointEpochs { + t.Helper() + epochs, err := OpenCheckpointEpochs(genesis, committedCheckpoint(t, utils.None[[]byte]()), height) + require.NoError(t, err) + return epochs +} + +func TestVerifyStateSyncProofReturnsTheSignedAppHash(t *testing.T) { + rng := utils.TestRng() + keys := utils.GenSliceN(rng, 3, atypes.GenSecretKey) + genesis := proofGenesis(keys) + appHash := [32]byte{7, 8, 9} + proof, err := StateSyncProofForTesting(genesis, keys, 12, appHash[:], t.TempDir()) + require.NoError(t, err) + + got, err := VerifyStateSyncProof(genesisCheckpointEpochs(t, genesis, 12), 12, proof) + require.NoError(t, err) + require.Equal(t, appHash, got) +} + +func TestVerifyStateSyncProofRefusesAProofOfAnotherCommittee(t *testing.T) { + rng := utils.TestRng() + keys := utils.GenSliceN(rng, 3, atypes.GenSecretKey) + others := utils.GenSliceN(rng, 3, atypes.GenSecretKey) + proof, err := StateSyncProofForTesting(proofGenesis(others), others, 12, make(atypes.AppHash, 32), t.TempDir()) + require.NoError(t, err) + + _, err = VerifyStateSyncProof(genesisCheckpointEpochs(t, proofGenesis(keys), 12), 12, proof) + require.Error(t, err) +} + +func TestVerifyStateSyncProofRefusesAProofOfAnotherHeight(t *testing.T) { + rng := utils.TestRng() + keys := utils.GenSliceN(rng, 3, atypes.GenSecretKey) + genesis := proofGenesis(keys) + proof, err := StateSyncProofForTesting(genesis, keys, 12, make(atypes.AppHash, 32), t.TempDir()) + require.NoError(t, err) + + _, err = VerifyStateSyncProof(genesisCheckpointEpochs(t, genesis, 12), 13, proof) + require.Error(t, err) +} + +func TestVerifyStateSyncProofRefusesAnAppHashThatIsNot32Bytes(t *testing.T) { + rng := utils.TestRng() + keys := utils.GenSliceN(rng, 3, atypes.GenSecretKey) + genesis := proofGenesis(keys) + proof, err := StateSyncProofForTesting(genesis, keys, 12, atypes.AppHash{1, 2, 3}, t.TempDir()) + require.NoError(t, err) + + _, err = VerifyStateSyncProof(genesisCheckpointEpochs(t, genesis, 12), 12, proof) + require.Error(t, err) +} diff --git a/sei-tendermint/node/setup.go b/sei-tendermint/node/setup.go index eaafdff584..d4f883af5a 100644 --- a/sei-tendermint/node/setup.go +++ b/sei-tendermint/node/setup.go @@ -443,6 +443,20 @@ func autobahnStorageConfig( return directory, storageConfig, nil } +// gigaActivationHeight returns the first block with an app hash on the chain of genDoc, its +// initial height. +func gigaActivationHeight(genDoc *types.GenesisDoc) (uint64, error) { + initialHeight := int64(0) + if genDoc != nil { + initialHeight = genDoc.InitialHeight + } + if initialHeight < 1 { + return 0, fmt.Errorf("initial height = %d, want >= 1", initialHeight) + } + //nolint:gosec // G115: InitialHeight is at least 1. + return uint64(initialHeight), nil +} + // buildGigaStorageConfig returns the store configs for directory with the [giga.storage] // settings applied over the Autobahn defaults. The builder's first block is // genDoc.InitialHeight. @@ -456,15 +470,11 @@ func buildGigaStorageConfig( if err != nil { return nil, err } - initialHeight := int64(0) - if genDoc != nil { - initialHeight = genDoc.InitialHeight - } - if initialHeight < 1 { - return nil, fmt.Errorf("initial height = %d, want >= 1", initialHeight) + activationHeight, err := gigaActivationHeight(genDoc) + if err != nil { + return nil, err } - //nolint:gosec // G115: InitialHeight is at least 1. - storageConfig.AppHashBuilderConfig.GigaActivationHeight = uint64(initialHeight) + storageConfig.AppHashBuilderConfig.GigaActivationHeight = activationHeight storageConfig.PruningConfig.RollbackWindow = storage.RollbackWindow storageConfig.PruningConfig.LookbackWindow = storage.LookbackWindow storageConfig.PruningConfig.PruneInterval = storage.PruneInterval diff --git a/sei-tendermint/node/testonly.go b/sei-tendermint/node/testonly.go index 5d95f3487b..eeb5bd094a 100644 --- a/sei-tendermint/node/testonly.go +++ b/sei-tendermint/node/testonly.go @@ -1,6 +1,8 @@ package node import ( + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/statesyncproof" + atypes "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" "github.com/sei-protocol/sei-chain/sei-tendermint/internal/p2p" ) @@ -14,3 +16,39 @@ func GenesisEpochRegistryRowForTesting(genesis EpochGenesis, scratchDir string) } return registry.MarshalSnapshot(), nil } + +// StateSyncProofForTesting returns the state sync proof file of road height whose AppQC signs +// appHash, signed by keys, the keys of the genesis validators. It verifies against the epoch +// registry that holds only the genesis epochs of genesis, which it opens under scratchDir. +// Intended for use in tests only. +func StateSyncProofForTesting( + genesis EpochGenesis, + keys []atypes.SecretKey, + height int64, + appHash atypes.AppHash, + scratchDir string, +) ([]byte, error) { + registry, err := p2p.OpenEpochRegistry(genesis.GenDoc, genesis.Validators, scratchDir) + if err != nil { + return nil, err + } + road := atypes.RoadIndex(height) //nolint:gosec // test heights are positive. + ep, err := registry.EpochAt(road) + if err != nil { + return nil, err + } + proposal, blocks := atypes.ProposalAtBlocks(ep, atypes.View{Index: road}, 1) + commitVote := atypes.NewCommitVote(atypes.BuildPrepareQC(keys, proposal)) + commitVotes := make([]*atypes.Signed[*atypes.CommitVote], 0, len(keys)) + for _, k := range keys { + commitVotes = append(commitVotes, atypes.Sign(k, commitVote)) + } + commitQC := atypes.NewCommitQC(commitVotes) + appVote := atypes.NewAppVote(atypes.NewAppProposal(commitQC.View(), appHash)) + appVotes := make([]*atypes.Signed[*atypes.AppVote], 0, len(keys)) + for _, k := range keys { + appVotes = append(appVotes, atypes.Sign(k, appVote)) + } + roadBlock := atypes.NewRoadBlock(atypes.NewFullCommitQC(commitQC, []*atypes.BlockHeader{blocks[0].Header()}), blocks) + return statesyncproof.Encode(roadBlock, atypes.NewAppQC(appVotes)) +}