Skip to content

Commit 91f2e5b

Browse files
cody-littleyCody Littley
andauthored
FlatKV Raw Value Access (#4565)
Add methods for reading raw values from flatKV. --------- Co-authored-by: Cody Littley <cody.littley@seinetwork.io>
1 parent 0276c1f commit 91f2e5b

6 files changed

Lines changed: 309 additions & 0 deletions

File tree

‎giga/evmonly/giga_store_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@ func (s *memoryGigaSnapshot) Get(string, []byte) ([]byte, bool) {
131131
return nil, false
132132
}
133133

134+
func (s *memoryGigaSnapshot) GetRaw([]byte) ([]byte, bool) {
135+
panic("memory giga snapshot holds no physical rows")
136+
}
137+
138+
func (s *memoryGigaSnapshot) BatchGetRaw([][]byte) map[string][]byte {
139+
panic("memory giga snapshot holds no physical rows")
140+
}
141+
134142
func (s *memoryGigaSnapshot) Close() {
135143
s.closeCount++
136144
}

‎giga/evmonly/memory_store.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,16 @@ func (s *memoryStoreSnapshot) Get(module string, key []byte) ([]byte, bool) {
574574
}
575575
}
576576

577+
// GetRaw panics: a memory store snapshot holds no physical rows.
578+
func (s *memoryStoreSnapshot) GetRaw([]byte) ([]byte, bool) {
579+
panic("memory store snapshot holds no physical rows")
580+
}
581+
582+
// BatchGetRaw panics: a memory store snapshot holds no physical rows.
583+
func (s *memoryStoreSnapshot) BatchGetRaw([][]byte) map[string][]byte {
584+
panic("memory store snapshot holds no physical rows")
585+
}
586+
577587
func (s *memoryStoreSnapshot) Close() {
578588
s.closed.Store(true)
579589
}

‎sei-db/state_db/giga/historical_view.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,16 @@ func (v *historicalView) Get(module string, key []byte) ([]byte, bool) {
4444
return value, value != nil
4545
}
4646

47+
// GetRaw panics: a historical view holds no physical rows.
48+
func (v *historicalView) GetRaw([]byte) ([]byte, bool) {
49+
panic("giga: a historical view holds no physical rows")
50+
}
51+
52+
// BatchGetRaw panics: a historical view holds no physical rows.
53+
func (v *historicalView) BatchGetRaw([][]byte) map[string][]byte {
54+
panic("giga: a historical view holds no physical rows")
55+
}
56+
4757
func (v *historicalView) Close() {
4858
v.closeOnce.Do(v.release)
4959
}

‎sei-db/state_db/giga/types/state_view.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,24 @@ type StateView interface {
4242
// account that exists with no code is not found, where GetCodeHash reports EmptyCodeHash.
4343
Get(module string, key []byte) ([]byte, bool)
4444

45+
// GetRaw returns the bytes the store holds under a physical key, and whether the key was found. The
46+
// bytes are the stored row as is, metadata included, and are not decoded.
47+
//
48+
// Some view implementations do not support this method and may panic.
49+
GetRaw(
50+
// The physical key: the module name, "/", then the key (including any key type prefixes).
51+
key []byte,
52+
) ([]byte, bool)
53+
54+
// BatchGetRaw returns, for each found physical key, the bytes the store holds under it, keyed by
55+
// string(key). Keys that are not found are absent from the map. Values are as GetRaw() returns them.
56+
//
57+
// Some view implementations do not support this method and may panic.
58+
BatchGetRaw(
59+
// The physical keys to read, each in the form GetRaw() takes.
60+
keys [][]byte,
61+
) map[string][]byte
62+
4563
// Close releases the view's underlying ref counting.
4664
// Caller is required to Close the view after using it.
4765
// Not closing the view properly could lead to memory leak.

‎sei-db/state_db/sc/flatkv/state_view.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package flatkv
22

33
import (
44
"fmt"
5+
"maps"
56
"sync"
67

78
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
@@ -83,6 +84,44 @@ func (v *flatKVStateView) Get(module string, key []byte) ([]byte, bool) {
8384
}
8485
}
8586

87+
// GetRaw returns the bytes stored under a physical key in this block, and whether it was found. It
88+
// panics on a key under the store's metadata prefix.
89+
func (v *flatKVStateView) GetRaw(key []byte) ([]byte, bool) {
90+
dbView, err := v.storeViewFor(key)
91+
if err != nil {
92+
panic(err)
93+
}
94+
value, found, err := dbView.Get(key, true)
95+
if err != nil {
96+
panic(fmt.Sprintf("flatkv: %s read of key %x at height %d: %v",
97+
dbView.Name(), key, v.blockView.BlockHeight(), err))
98+
}
99+
return value, found
100+
}
101+
102+
// BatchGetRaw returns the bytes stored in this block under each found physical key, keyed by string(key).
103+
// It panics on a key under the store's metadata prefix.
104+
func (v *flatKVStateView) BatchGetRaw(keys [][]byte) map[string][]byte {
105+
keysByStore := make(map[view.View][][]byte)
106+
for _, key := range keys {
107+
dbView, err := v.storeViewFor(key)
108+
if err != nil {
109+
panic(err)
110+
}
111+
keysByStore[dbView] = append(keysByStore[dbView], key)
112+
}
113+
values := make(map[string][]byte, len(keys))
114+
for dbView, storeKeys := range keysByStore {
115+
found, err := dbView.BatchGet(storeKeys)
116+
if err != nil {
117+
panic(fmt.Sprintf("flatkv: %s batch read of %d keys at height %d: %v",
118+
dbView.Name(), len(storeKeys), v.blockView.BlockHeight(), err))
119+
}
120+
maps.Copy(values, found)
121+
}
122+
return values
123+
}
124+
86125
// AccountExists reports whether addr has an account in this block.
87126
func (v *flatKVStateView) AccountExists(addr gigatypes.Address) bool {
88127
_, ok := v.accountRow(addr)
@@ -270,6 +309,30 @@ func (v *flatKVStateView) readEVMRow(dbView view.View, kind keys.EVMKeyKind, key
270309
return value, found
271310
}
272311

312+
// storeViewFor returns the view of the store that holds physKey. It returns an error for a metadata
313+
// key, since the store's own metadata is not part of any block's state, and for a key no store holds.
314+
func (v *flatKVStateView) storeViewFor(physKey []byte) (view.View, error) {
315+
if ktype.IsMetaKey(physKey) {
316+
return nil, fmt.Errorf("flatkv: raw key %x is store metadata, not block state", physKey)
317+
}
318+
dbDir, err := routePhysicalKey(physKey)
319+
if err != nil {
320+
return nil, fmt.Errorf("flatkv: route raw key %x at height %d: %w", physKey, v.blockView.BlockHeight(), err)
321+
}
322+
switch dbDir {
323+
case accountDBDir:
324+
return v.blockView.AccountView(), nil
325+
case codeDBDir:
326+
return v.blockView.CodeView(), nil
327+
case storageDBDir:
328+
return v.blockView.StorageView(), nil
329+
case miscDBDir:
330+
return v.blockView.MiscView(), nil
331+
default:
332+
return nil, fmt.Errorf("flatkv: raw key %x routes to unknown store %q", physKey, dbDir)
333+
}
334+
}
335+
273336
// readRow returns the bytes stored under physKey, without deserializing them.
274337
func (v *flatKVStateView) readRow(dbView view.View, physKey []byte) ([]byte, bool) {
275338
value, found, err := dbView.Get(physKey, true)
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
package flatkv
2+
3+
import (
4+
"testing"
5+
6+
"github.com/stretchr/testify/require"
7+
8+
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
9+
"github.com/sei-protocol/sei-chain/sei-db/proto"
10+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
11+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
12+
)
13+
14+
// rawRow is a physical key and the bytes a block is expected to leave under it.
15+
type rawRow struct {
16+
key []byte
17+
value []byte
18+
}
19+
20+
// commitRowInEveryStore commits block 1 with one row in each of FlatKV's stores, and returns the
21+
// physical key and serialized bytes of each.
22+
func commitRowInEveryStore(t *testing.T, s *CommitStore) []rawRow {
23+
t.Helper()
24+
addr := addrN(1)
25+
slot := slotN(2)
26+
balance := balanceN(3)
27+
codeHash := codeHashN(4)
28+
bytecode := []byte{0x60, 0x00}
29+
evmMisc := evmMiscKey(0x05)
30+
bankKey := []byte{0x06}
31+
32+
require.NoError(t, s.CommitStateChanges(1, []*proto.NamedChangeSet{
33+
namedCS(
34+
noncePair(addr, 7),
35+
balancePair(addr, balance),
36+
codeHashPair(addr, codeHash),
37+
storagePair(addr, slot, []byte{0x08}),
38+
codePair(addr, bytecode),
39+
&proto.KVPair{Key: evmMisc, Value: []byte{0x09}},
40+
),
41+
{Name: "bank", Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{{Key: bankKey, Value: []byte{0x0a}}}}},
42+
}))
43+
44+
account := vtype.NewAccountData().SetBlockHeight(1).SetNonce(7).SetBalance(&balance).SetCodeHash(&codeHash)
45+
storage, err := vtype.SerializeStorage(1, padLeft32(0x08))
46+
require.NoError(t, err)
47+
return []rawRow{
48+
{key: accountPhysKey(addr), value: account.Serialize()},
49+
{key: storagePhysKey(addr, slot), value: storage},
50+
{key: ktype.EVMPhysicalKey(keys.EVMKeyCode, addr[:]), value: vtype.SerializeCode(1, bytecode)},
51+
{key: ktype.ModulePhysicalKey(keys.EVMStoreKey, evmMisc), value: vtype.SerializeMisc(1, []byte{0x09})},
52+
{key: ktype.ModulePhysicalKey("bank", bankKey), value: vtype.SerializeMisc(1, []byte{0x0a})},
53+
}
54+
}
55+
56+
// GetRaw answers with the row exactly as stored, metadata included, in every store a key can route to.
57+
func TestGetRawReturnsStoredRows(t *testing.T) {
58+
s := setupTestStore(t)
59+
defer func() { require.NoError(t, s.Close()) }()
60+
rows := commitRowInEveryStore(t, s)
61+
62+
stateView := s.OpenView()
63+
defer stateView.Close()
64+
65+
for _, row := range rows {
66+
value, found := stateView.GetRaw(row.key)
67+
require.True(t, found, "key %q", row.key)
68+
require.Equal(t, row.value, value, "key %q", row.key)
69+
}
70+
}
71+
72+
// A physical key that holds nothing is reported as not found, whichever store it routes to.
73+
func TestGetRawMissingKey(t *testing.T) {
74+
s := setupTestStore(t)
75+
defer func() { require.NoError(t, s.Close()) }()
76+
commitRowInEveryStore(t, s)
77+
78+
stateView := s.OpenView()
79+
defer stateView.Close()
80+
81+
missing := addrN(0x20)
82+
for _, key := range [][]byte{
83+
accountPhysKey(missing),
84+
storagePhysKey(missing, slotN(1)),
85+
ktype.EVMPhysicalKey(keys.EVMKeyCode, missing[:]),
86+
ktype.ModulePhysicalKey(keys.EVMStoreKey, evmMiscKey(0x20)),
87+
ktype.ModulePhysicalKey("bank", []byte{0x20}),
88+
} {
89+
value, found := stateView.GetRaw(key)
90+
require.False(t, found, "key %q", key)
91+
require.Nil(t, value, "key %q", key)
92+
}
93+
}
94+
95+
// One batch can span every store and include keys that hold nothing; only the found keys come back.
96+
func TestBatchGetRawMixesStoresAndMissingKeys(t *testing.T) {
97+
s := setupTestStore(t)
98+
defer func() { require.NoError(t, s.Close()) }()
99+
rows := commitRowInEveryStore(t, s)
100+
101+
stateView := s.OpenView()
102+
defer stateView.Close()
103+
104+
missing := addrN(0x20)
105+
requested := [][]byte{accountPhysKey(missing), storagePhysKey(missing, slotN(1))}
106+
expected := map[string][]byte{}
107+
for _, row := range rows {
108+
requested = append(requested, row.key)
109+
expected[string(row.key)] = row.value
110+
}
111+
112+
require.Equal(t, expected, stateView.BatchGetRaw(requested))
113+
}
114+
115+
// An empty batch reads nothing and returns an empty map.
116+
func TestBatchGetRawEmpty(t *testing.T) {
117+
s := setupTestStore(t)
118+
defer func() { require.NoError(t, s.Close()) }()
119+
commitRowInEveryStore(t, s)
120+
121+
stateView := s.OpenView()
122+
defer stateView.Close()
123+
124+
require.Empty(t, stateView.BatchGetRaw(nil))
125+
}
126+
127+
// Raw reads are pinned to the view's block like every other read: a later block's write is invisible
128+
// to a view opened before it, and visible to one opened after.
129+
func TestRawReadsArePinnedToTheViewBlock(t *testing.T) {
130+
s := setupTestStore(t)
131+
defer func() { require.NoError(t, s.Close()) }()
132+
rows := commitRowInEveryStore(t, s)
133+
accountRow := rows[0]
134+
135+
before := s.OpenView()
136+
defer before.Close()
137+
138+
addr := addrN(1)
139+
commitNonce(t, s, 2, addr, 8)
140+
141+
value, found := before.GetRaw(accountRow.key)
142+
require.True(t, found)
143+
require.Equal(t, accountRow.value, value)
144+
require.Equal(t, map[string][]byte{string(accountRow.key): accountRow.value},
145+
before.BatchGetRaw([][]byte{accountRow.key}))
146+
147+
after := s.OpenView()
148+
defer after.Close()
149+
150+
balance := balanceN(3)
151+
codeHash := codeHashN(4)
152+
updated := vtype.NewAccountData().SetBlockHeight(2).SetNonce(8).SetBalance(&balance).SetCodeHash(&codeHash)
153+
value, found = after.GetRaw(accountRow.key)
154+
require.True(t, found)
155+
require.Equal(t, updated.Serialize(), value)
156+
}
157+
158+
// A key with no module separator cannot be routed to any store.
159+
func TestGetRawRejectsUnroutableKey(t *testing.T) {
160+
s := setupTestStore(t)
161+
defer func() { require.NoError(t, s.Close()) }()
162+
commitRowInEveryStore(t, s)
163+
164+
stateView := s.OpenView()
165+
defer stateView.Close()
166+
167+
require.Panics(t, func() { stateView.GetRaw([]byte("no-separator")) })
168+
require.Panics(t, func() { stateView.BatchGetRaw([][]byte{[]byte("no-separator")}) })
169+
}
170+
171+
// A key under the metadata prefix names the store's own bookkeeping, not block state.
172+
func TestGetRawRejectsMetaKey(t *testing.T) {
173+
s := setupTestStore(t)
174+
defer func() { require.NoError(t, s.Close()) }()
175+
rows := commitRowInEveryStore(t, s)
176+
177+
stateView := s.OpenView()
178+
defer stateView.Close()
179+
180+
metaKey := []byte(ktype.MetaKeyPrefix + "version")
181+
require.Panics(t, func() { stateView.GetRaw(metaKey) })
182+
require.Panics(t, func() { stateView.BatchGetRaw([][]byte{rows[0].key, metaKey}) })
183+
}
184+
185+
// storeViewFor reports a key it cannot serve as an error rather than panicking.
186+
func TestStoreViewForRejectsMetaKey(t *testing.T) {
187+
s := setupTestStore(t)
188+
defer func() { require.NoError(t, s.Close()) }()
189+
commitRowInEveryStore(t, s)
190+
191+
stateView := s.OpenView()
192+
defer stateView.Close()
193+
flatView := stateView.(*flatKVStateView)
194+
195+
for _, key := range [][]byte{[]byte(ktype.MetaKeyPrefix + "version"), []byte("no-separator")} {
196+
var err error
197+
require.NotPanics(t, func() { _, err = flatView.storeViewFor(key) })
198+
require.Error(t, err, "key %q", key)
199+
}
200+
}

0 commit comments

Comments
 (0)