Skip to content

Commit 7de748d

Browse files
rjl493456442fjlholiman
authored
all: implement path-based state scheme (#25963)
* all: implement path-based state scheme * all: edits from review * core/rawdb, trie/triedb/pathdb: review changes * core, light, trie, eth, tests: reimplement pbss history * core, trie/triedb/pathdb: track block number in state history * trie/triedb/pathdb: add history documentation * core, trie/triedb/pathdb: address comments from Peter's review Important changes to list: - Cache trie nodes by path in clean cache - Remove root->id mappings when history is truncated * trie/triedb/pathdb: fallback to disk if unexpect node in clean cache * core/rawdb: fix tests * trie/triedb/pathdb: rename metrics, change clean cache key * trie/triedb: manage the clean cache inside of disk layer * trie/triedb/pathdb: move journal function * trie/triedb/path: fix tests * trie/triedb/pathdb: fix journal * trie/triedb/pathdb: fix history * trie/triedb/pathdb: try to fix tests on windows * core, trie: address comments * trie/triedb/pathdb: fix test issues --------- Co-authored-by: Felix Lange <[email protected]> Co-authored-by: Martin Holst Swende <[email protected]>
1 parent 9d744f0 commit 7de748d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+4422
-230
lines changed

core/blockchain.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -713,7 +713,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
713713
if num+1 <= frozen {
714714
// Truncate all relative data(header, total difficulty, body, receipt
715715
// and canonical hash) from ancient store.
716-
if err := bc.db.TruncateHead(num); err != nil {
716+
if _, err := bc.db.TruncateHead(num); err != nil {
717717
log.Crit("Failed to truncate ancient data", "number", num, "err", err)
718718
}
719719
// Remove the hash <-> number mapping from the active store.
@@ -1136,7 +1136,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
11361136
size += int64(batch.ValueSize())
11371137
if err = batch.Write(); err != nil {
11381138
snapBlock := bc.CurrentSnapBlock().Number.Uint64()
1139-
if err := bc.db.TruncateHead(snapBlock + 1); err != nil {
1139+
if _, err := bc.db.TruncateHead(snapBlock + 1); err != nil {
11401140
log.Error("Can't truncate ancient store after failed insert", "err", err)
11411141
}
11421142
return 0, err
@@ -1154,7 +1154,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
11541154
if !updateHead(blockChain[len(blockChain)-1]) {
11551155
// We end up here if the header chain has reorg'ed, and the blocks/receipts
11561156
// don't match the canonical chain.
1157-
if err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil {
1157+
if _, err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil {
11581158
log.Error("Can't truncate ancient store after failed insert", "err", err)
11591159
}
11601160
return 0, errSideChainReceipts

core/rawdb/accessors_chain_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func TestBodyStorage(t *testing.T) {
8585
WriteBody(db, hash, 0, body)
8686
if entry := ReadBody(db, hash, 0); entry == nil {
8787
t.Fatalf("Stored body not found")
88-
} else if types.DeriveSha(types.Transactions(entry.Transactions), newHasher()) != types.DeriveSha(types.Transactions(body.Transactions), newHasher()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) {
88+
} else if types.DeriveSha(types.Transactions(entry.Transactions), newTestHasher()) != types.DeriveSha(types.Transactions(body.Transactions), newTestHasher()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) {
8989
t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body)
9090
}
9191
if entry := ReadBodyRLP(db, hash, 0); entry == nil {
@@ -139,7 +139,7 @@ func TestBlockStorage(t *testing.T) {
139139
}
140140
if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry == nil {
141141
t.Fatalf("Stored body not found")
142-
} else if types.DeriveSha(types.Transactions(entry.Transactions), newHasher()) != types.DeriveSha(block.Transactions(), newHasher()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) {
142+
} else if types.DeriveSha(types.Transactions(entry.Transactions), newTestHasher()) != types.DeriveSha(block.Transactions(), newTestHasher()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) {
143143
t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body())
144144
}
145145
// Delete the block and verify the execution

core/rawdb/accessors_indexes_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import (
2929
"github.com/ethereum/go-ethereum/rlp"
3030
)
3131

32-
var newHasher = blocktest.NewHasher
32+
var newTestHasher = blocktest.NewHasher
3333

3434
// Tests that positional lookup metadata can be stored and retrieved.
3535
func TestLookupStorage(t *testing.T) {
@@ -76,7 +76,7 @@ func TestLookupStorage(t *testing.T) {
7676
tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
7777
txs := []*types.Transaction{tx1, tx2, tx3}
7878

79-
block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil, newHasher())
79+
block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil, newTestHasher())
8080

8181
// Check that no transactions entries are in a pristine database
8282
for i, tx := range txs {

core/rawdb/accessors_state.go

+172
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
package rawdb
1818

1919
import (
20+
"encoding/binary"
21+
2022
"github.com/ethereum/go-ethereum/common"
2123
"github.com/ethereum/go-ethereum/ethdb"
2224
"github.com/ethereum/go-ethereum/log"
@@ -92,3 +94,173 @@ func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
9294
log.Crit("Failed to delete contract code", "err", err)
9395
}
9496
}
97+
98+
// ReadStateID retrieves the state id with the provided state root.
99+
func ReadStateID(db ethdb.KeyValueReader, root common.Hash) *uint64 {
100+
data, err := db.Get(stateIDKey(root))
101+
if err != nil || len(data) == 0 {
102+
return nil
103+
}
104+
number := binary.BigEndian.Uint64(data)
105+
return &number
106+
}
107+
108+
// WriteStateID writes the provided state lookup to database.
109+
func WriteStateID(db ethdb.KeyValueWriter, root common.Hash, id uint64) {
110+
var buff [8]byte
111+
binary.BigEndian.PutUint64(buff[:], id)
112+
if err := db.Put(stateIDKey(root), buff[:]); err != nil {
113+
log.Crit("Failed to store state ID", "err", err)
114+
}
115+
}
116+
117+
// DeleteStateID deletes the specified state lookup from the database.
118+
func DeleteStateID(db ethdb.KeyValueWriter, root common.Hash) {
119+
if err := db.Delete(stateIDKey(root)); err != nil {
120+
log.Crit("Failed to delete state ID", "err", err)
121+
}
122+
}
123+
124+
// ReadPersistentStateID retrieves the id of the persistent state from the database.
125+
func ReadPersistentStateID(db ethdb.KeyValueReader) uint64 {
126+
data, _ := db.Get(persistentStateIDKey)
127+
if len(data) != 8 {
128+
return 0
129+
}
130+
return binary.BigEndian.Uint64(data)
131+
}
132+
133+
// WritePersistentStateID stores the id of the persistent state into database.
134+
func WritePersistentStateID(db ethdb.KeyValueWriter, number uint64) {
135+
if err := db.Put(persistentStateIDKey, encodeBlockNumber(number)); err != nil {
136+
log.Crit("Failed to store the persistent state ID", "err", err)
137+
}
138+
}
139+
140+
// ReadTrieJournal retrieves the serialized in-memory trie nodes of layers saved at
141+
// the last shutdown.
142+
func ReadTrieJournal(db ethdb.KeyValueReader) []byte {
143+
data, _ := db.Get(trieJournalKey)
144+
return data
145+
}
146+
147+
// WriteTrieJournal stores the serialized in-memory trie nodes of layers to save at
148+
// shutdown.
149+
func WriteTrieJournal(db ethdb.KeyValueWriter, journal []byte) {
150+
if err := db.Put(trieJournalKey, journal); err != nil {
151+
log.Crit("Failed to store tries journal", "err", err)
152+
}
153+
}
154+
155+
// DeleteTrieJournal deletes the serialized in-memory trie nodes of layers saved at
156+
// the last shutdown.
157+
func DeleteTrieJournal(db ethdb.KeyValueWriter) {
158+
if err := db.Delete(trieJournalKey); err != nil {
159+
log.Crit("Failed to remove tries journal", "err", err)
160+
}
161+
}
162+
163+
// ReadStateHistoryMeta retrieves the metadata corresponding to the specified
164+
// state history. Compute the position of state history in freezer by minus
165+
// one since the id of first state history starts from one(zero for initial
166+
// state).
167+
func ReadStateHistoryMeta(db ethdb.AncientReaderOp, id uint64) []byte {
168+
blob, err := db.Ancient(stateHistoryMeta, id-1)
169+
if err != nil {
170+
return nil
171+
}
172+
return blob
173+
}
174+
175+
// ReadStateHistoryMetaList retrieves a batch of meta objects with the specified
176+
// start position and count. Compute the position of state history in freezer by
177+
// minus one since the id of first state history starts from one(zero for initial
178+
// state).
179+
func ReadStateHistoryMetaList(db ethdb.AncientReaderOp, start uint64, count uint64) ([][]byte, error) {
180+
return db.AncientRange(stateHistoryMeta, start-1, count, 0)
181+
}
182+
183+
// ReadStateAccountIndex retrieves the state root corresponding to the specified
184+
// state history. Compute the position of state history in freezer by minus one
185+
// since the id of first state history starts from one(zero for initial state).
186+
func ReadStateAccountIndex(db ethdb.AncientReaderOp, id uint64) []byte {
187+
blob, err := db.Ancient(stateHistoryAccountIndex, id-1)
188+
if err != nil {
189+
return nil
190+
}
191+
return blob
192+
}
193+
194+
// ReadStateStorageIndex retrieves the state root corresponding to the specified
195+
// state history. Compute the position of state history in freezer by minus one
196+
// since the id of first state history starts from one(zero for initial state).
197+
func ReadStateStorageIndex(db ethdb.AncientReaderOp, id uint64) []byte {
198+
blob, err := db.Ancient(stateHistoryStorageIndex, id-1)
199+
if err != nil {
200+
return nil
201+
}
202+
return blob
203+
}
204+
205+
// ReadStateAccountHistory retrieves the state root corresponding to the specified
206+
// state history. Compute the position of state history in freezer by minus one
207+
// since the id of first state history starts from one(zero for initial state).
208+
func ReadStateAccountHistory(db ethdb.AncientReaderOp, id uint64) []byte {
209+
blob, err := db.Ancient(stateHistoryAccountData, id-1)
210+
if err != nil {
211+
return nil
212+
}
213+
return blob
214+
}
215+
216+
// ReadStateStorageHistory retrieves the state root corresponding to the specified
217+
// state history. Compute the position of state history in freezer by minus one
218+
// since the id of first state history starts from one(zero for initial state).
219+
func ReadStateStorageHistory(db ethdb.AncientReaderOp, id uint64) []byte {
220+
blob, err := db.Ancient(stateHistoryStorageData, id-1)
221+
if err != nil {
222+
return nil
223+
}
224+
return blob
225+
}
226+
227+
// ReadStateHistory retrieves the state history from database with provided id.
228+
// Compute the position of state history in freezer by minus one since the id
229+
// of first state history starts from one(zero for initial state).
230+
func ReadStateHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []byte, []byte, []byte, error) {
231+
meta, err := db.Ancient(stateHistoryMeta, id-1)
232+
if err != nil {
233+
return nil, nil, nil, nil, nil, err
234+
}
235+
accountIndex, err := db.Ancient(stateHistoryAccountIndex, id-1)
236+
if err != nil {
237+
return nil, nil, nil, nil, nil, err
238+
}
239+
storageIndex, err := db.Ancient(stateHistoryStorageIndex, id-1)
240+
if err != nil {
241+
return nil, nil, nil, nil, nil, err
242+
}
243+
accountData, err := db.Ancient(stateHistoryAccountData, id-1)
244+
if err != nil {
245+
return nil, nil, nil, nil, nil, err
246+
}
247+
storageData, err := db.Ancient(stateHistoryStorageData, id-1)
248+
if err != nil {
249+
return nil, nil, nil, nil, nil, err
250+
}
251+
return meta, accountIndex, storageIndex, accountData, storageData, nil
252+
}
253+
254+
// WriteStateHistory writes the provided state history to database. Compute the
255+
// position of state history in freezer by minus one since the id of first state
256+
// history starts from one(zero for initial state).
257+
func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIndex []byte, storageIndex []byte, accounts []byte, storages []byte) {
258+
db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
259+
op.AppendRaw(stateHistoryMeta, id-1, meta)
260+
op.AppendRaw(stateHistoryAccountIndex, id-1, accountIndex)
261+
op.AppendRaw(stateHistoryStorageIndex, id-1, storageIndex)
262+
op.AppendRaw(stateHistoryAccountData, id-1, accounts)
263+
op.AppendRaw(stateHistoryStorageData, id-1, storages)
264+
return nil
265+
})
266+
}

core/rawdb/accessors_trie.go

+24-22
Original file line numberDiff line numberDiff line change
@@ -46,21 +46,23 @@ const HashScheme = "hashScheme"
4646
// on extra state diffs to survive deep reorg.
4747
const PathScheme = "pathScheme"
4848

49-
// nodeHasher used to derive the hash of trie node.
50-
type nodeHasher struct{ sha crypto.KeccakState }
49+
// hasher is used to compute the sha256 hash of the provided data.
50+
type hasher struct{ sha crypto.KeccakState }
5151

5252
var hasherPool = sync.Pool{
53-
New: func() interface{} { return &nodeHasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} },
53+
New: func() interface{} { return &hasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} },
5454
}
5555

56-
func newNodeHasher() *nodeHasher { return hasherPool.Get().(*nodeHasher) }
57-
func returnHasherToPool(h *nodeHasher) { hasherPool.Put(h) }
56+
func newHasher() *hasher {
57+
return hasherPool.Get().(*hasher)
58+
}
59+
60+
func (h *hasher) hash(data []byte) common.Hash {
61+
return crypto.HashData(h.sha, data)
62+
}
5863

59-
func (h *nodeHasher) hashData(data []byte) (n common.Hash) {
60-
h.sha.Reset()
61-
h.sha.Write(data)
62-
h.sha.Read(n[:])
63-
return n
64+
func (h *hasher) release() {
65+
hasherPool.Put(h)
6466
}
6567

6668
// ReadAccountTrieNode retrieves the account trie node and the associated node
@@ -70,9 +72,9 @@ func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.H
7072
if err != nil {
7173
return nil, common.Hash{}
7274
}
73-
hasher := newNodeHasher()
74-
defer returnHasherToPool(hasher)
75-
return data, hasher.hashData(data)
75+
h := newHasher()
76+
defer h.release()
77+
return data, h.hash(data)
7678
}
7779

7880
// HasAccountTrieNode checks the account trie node presence with the specified
@@ -82,9 +84,9 @@ func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash)
8284
if err != nil {
8385
return false
8486
}
85-
hasher := newNodeHasher()
86-
defer returnHasherToPool(hasher)
87-
return hasher.hashData(data) == hash
87+
h := newHasher()
88+
defer h.release()
89+
return h.hash(data) == hash
8890
}
8991

9092
// WriteAccountTrieNode writes the provided account trie node into database.
@@ -108,9 +110,9 @@ func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path
108110
if err != nil {
109111
return nil, common.Hash{}
110112
}
111-
hasher := newNodeHasher()
112-
defer returnHasherToPool(hasher)
113-
return data, hasher.hashData(data)
113+
h := newHasher()
114+
defer h.release()
115+
return data, h.hash(data)
114116
}
115117

116118
// HasStorageTrieNode checks the storage trie node presence with the provided
@@ -120,9 +122,9 @@ func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path [
120122
if err != nil {
121123
return false
122124
}
123-
hasher := newNodeHasher()
124-
defer returnHasherToPool(hasher)
125-
return hasher.hashData(data) == hash
125+
h := newHasher()
126+
defer h.release()
127+
return h.hash(data) == hash
126128
}
127129

128130
// WriteStorageTrieNode writes the provided storage trie node into database.

core/rawdb/ancient_scheme.go

+29-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package rawdb
1818

19+
import "path/filepath"
20+
1921
// The list of table names of chain freezer.
2022
const (
2123
// ChainFreezerHeaderTable indicates the name of the freezer header table.
@@ -44,10 +46,36 @@ var chainFreezerNoSnappy = map[string]bool{
4446
ChainFreezerDifficultyTable: true,
4547
}
4648

49+
const (
50+
// stateHistoryTableSize defines the maximum size of freezer data files.
51+
stateHistoryTableSize = 2 * 1000 * 1000 * 1000
52+
53+
// stateHistoryAccountIndex indicates the name of the freezer state history table.
54+
stateHistoryMeta = "history.meta"
55+
stateHistoryAccountIndex = "account.index"
56+
stateHistoryStorageIndex = "storage.index"
57+
stateHistoryAccountData = "account.data"
58+
stateHistoryStorageData = "storage.data"
59+
)
60+
61+
var stateHistoryFreezerNoSnappy = map[string]bool{
62+
stateHistoryMeta: true,
63+
stateHistoryAccountIndex: false,
64+
stateHistoryStorageIndex: false,
65+
stateHistoryAccountData: false,
66+
stateHistoryStorageData: false,
67+
}
68+
4769
// The list of identifiers of ancient stores.
4870
var (
4971
chainFreezerName = "chain" // the folder name of chain segment ancient store.
72+
stateFreezerName = "state" // the folder name of reverse diff ancient store.
5073
)
5174

5275
// freezers the collections of all builtin freezers.
53-
var freezers = []string{chainFreezerName}
76+
var freezers = []string{chainFreezerName, stateFreezerName}
77+
78+
// NewStateHistoryFreezer initializes the freezer for state history.
79+
func NewStateHistoryFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
80+
return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateHistoryFreezerNoSnappy)
81+
}

0 commit comments

Comments
 (0)