Skip to content

Commit 4535230

Browse files
holimanrjl493456442karalabe
authored
cmd, core, eth: background transaction indexing (#20302)
* cmd, core, eth: init tx lookup in background * core/rawdb: tiny log fixes to make it clearer what's happening * core, eth: fix rebase errors * core/rawdb: make reindexing less generic, but more optimal * rlp: implement rlp list iterator * core/rawdb: new implementation of tx indexing/unindex using generic tx iterator and hashing rlp-data * core/rawdb, cmd/utils: fix review concerns * cmd/utils: fix merge issue * core/rawdb: add some log formatting polishes Co-authored-by: rjl493456442 <[email protected]> Co-authored-by: Péter Szilágyi <[email protected]>
1 parent 6f54ae2 commit 4535230

37 files changed

+1268
-200
lines changed

‎accounts/abi/bind/backends/simulated.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ type SimulatedBackend struct {
7676
func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
7777
genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
7878
genesis.MustCommit(database)
79-
blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
79+
blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
8080

8181
backend := &SimulatedBackend{
8282
database: database,

‎cmd/geth/chaincmd.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
9191
utils.MetricsInfluxDBUsernameFlag,
9292
utils.MetricsInfluxDBPasswordFlag,
9393
utils.MetricsInfluxDBTagsFlag,
94+
utils.TxLookupLimitFlag,
9495
},
9596
Category: "BLOCKCHAIN COMMANDS",
9697
Description: `
@@ -158,6 +159,7 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
158159
utils.FakePoWFlag,
159160
utils.RopstenFlag,
160161
utils.RinkebyFlag,
162+
utils.TxLookupLimitFlag,
161163
utils.GoerliFlag,
162164
utils.LegacyTestnetFlag,
163165
},
@@ -274,7 +276,7 @@ func importChain(ctx *cli.Context) error {
274276
stack := makeFullNode(ctx)
275277
defer stack.Close()
276278

277-
chain, db := utils.MakeChain(ctx, stack)
279+
chain, db := utils.MakeChain(ctx, stack, false)
278280
defer db.Close()
279281

280282
// Start periodically gathering memory profiles
@@ -364,7 +366,7 @@ func exportChain(ctx *cli.Context) error {
364366
stack := makeFullNode(ctx)
365367
defer stack.Close()
366368

367-
chain, _ := utils.MakeChain(ctx, stack)
369+
chain, _ := utils.MakeChain(ctx, stack, true)
368370
start := time.Now()
369371

370372
var err error
@@ -439,7 +441,7 @@ func copyDb(ctx *cli.Context) error {
439441
stack := makeFullNode(ctx)
440442
defer stack.Close()
441443

442-
chain, chainDb := utils.MakeChain(ctx, stack)
444+
chain, chainDb := utils.MakeChain(ctx, stack, false)
443445
syncMode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
444446

445447
var syncBloom *trie.SyncBloom
@@ -547,7 +549,7 @@ func dump(ctx *cli.Context) error {
547549
stack := makeFullNode(ctx)
548550
defer stack.Close()
549551

550-
chain, chainDb := utils.MakeChain(ctx, stack)
552+
chain, chainDb := utils.MakeChain(ctx, stack, true)
551553
defer chainDb.Close()
552554
for _, arg := range ctx.Args() {
553555
var block *types.Block
@@ -586,7 +588,7 @@ func inspect(ctx *cli.Context) error {
586588
node, _ := makeConfigNode(ctx)
587589
defer node.Close()
588590

589-
_, chainDb := utils.MakeChain(ctx, node)
591+
_, chainDb := utils.MakeChain(ctx, node, true)
590592
defer chainDb.Close()
591593

592594
return rawdb.InspectDatabase(chainDb)

‎cmd/geth/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ var (
9292
utils.ExitWhenSyncedFlag,
9393
utils.GCModeFlag,
9494
utils.SnapshotFlag,
95+
utils.TxLookupLimitFlag,
9596
utils.LightServeFlag,
9697
utils.LegacyLightServFlag,
9798
utils.LightIngressFlag,

‎cmd/geth/retesteth.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa
400400
}
401401
engine := &NoRewardEngine{inner: inner, rewardsOn: chainParams.SealEngine != "NoReward"}
402402

403-
blockchain, err := core.NewBlockChain(ethDb, nil, chainConfig, engine, vm.Config{}, nil)
403+
blockchain, err := core.NewBlockChain(ethDb, nil, chainConfig, engine, vm.Config{}, nil, nil)
404404
if err != nil {
405405
return false, err
406406
}

‎cmd/geth/usage.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ var AppHelpFlagGroups = []flagGroup{
7878
utils.SyncModeFlag,
7979
utils.ExitWhenSyncedFlag,
8080
utils.GCModeFlag,
81+
utils.TxLookupLimitFlag,
8182
utils.EthStatsURLFlag,
8283
utils.IdentityFlag,
8384
utils.LightKDFFlag,

‎cmd/utils/flags.go‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,11 @@ var (
229229
Name: "snapshot",
230230
Usage: `Enables snapshot-database mode -- experimental work in progress feature`,
231231
}
232+
TxLookupLimitFlag = cli.Int64Flag{
233+
Name: "txlookuplimit",
234+
Usage: "Number of recent blocks to maintain transactions index by-hash for (default = index all blocks)",
235+
Value: 0,
236+
}
232237
LightKDFFlag = cli.BoolFlag{
233238
Name: "lightkdf",
234239
Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@@ -1469,7 +1474,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
14691474
CheckExclusive(ctx, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag)
14701475
CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light")
14711476
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
1472-
1477+
CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag)
1478+
// todo(rjl493456442) make it available for les server
1479+
// Ancient tx indices pruning is not available for les server now
1480+
// since light client relies on the server for transaction status query.
1481+
CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, TxLookupLimitFlag)
14731482
var ks *keystore.KeyStore
14741483
if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
14751484
ks = keystores[0].(*keystore.KeyStore)
@@ -1505,6 +1514,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
15051514
if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) {
15061515
cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name)
15071516
}
1517+
if ctx.GlobalIsSet(TxLookupLimitFlag.Name) {
1518+
cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name)
1519+
}
15081520
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
15091521
cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
15101522
}
@@ -1746,7 +1758,7 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
17461758
}
17471759

17481760
// MakeChain creates a chain manager from set command line flags.
1749-
func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) {
1761+
func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.BlockChain, chainDb ethdb.Database) {
17501762
var err error
17511763
chainDb = MakeChainDatabase(ctx, stack)
17521764
config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
@@ -1792,7 +1804,12 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
17921804
cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
17931805
}
17941806
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
1795-
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)
1807+
var limit *uint64
1808+
if ctx.GlobalIsSet(TxLookupLimitFlag.Name) && !readOnly {
1809+
l := ctx.GlobalUint64(TxLookupLimitFlag.Name)
1810+
limit = &l
1811+
}
1812+
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, limit)
17961813
if err != nil {
17971814
Fatalf("Can't create BlockChain: %v", err)
17981815
}

‎consensus/clique/clique_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func TestReimportMirroredState(t *testing.T) {
5454
genesis := genspec.MustCommit(db)
5555

5656
// Generate a batch of blocks, each properly signed
57-
chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil)
57+
chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil)
5858
defer chain.Stop()
5959

6060
blocks, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, 3, func(i int, block *core.BlockGen) {
@@ -88,7 +88,7 @@ func TestReimportMirroredState(t *testing.T) {
8888
db = rawdb.NewMemoryDatabase()
8989
genspec.MustCommit(db)
9090

91-
chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil)
91+
chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil)
9292
defer chain.Stop()
9393

9494
if _, err := chain.InsertChain(blocks[:2]); err != nil {
@@ -101,7 +101,7 @@ func TestReimportMirroredState(t *testing.T) {
101101
// Simulate a crash by creating a new chain on top of the database, without
102102
// flushing the dirty states out. Insert the last block, trigerring a sidechain
103103
// reimport.
104-
chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil)
104+
chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil)
105105
defer chain.Stop()
106106

107107
if _, err := chain.InsertChain(blocks[2:]); err != nil {

‎consensus/clique/snapshot_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ func TestClique(t *testing.T) {
448448
batches[len(batches)-1] = append(batches[len(batches)-1], block)
449449
}
450450
// Pass all the headers through clique and ensure tallying succeeds
451-
chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil)
451+
chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil)
452452
if err != nil {
453453
t.Errorf("test %d: failed to create test chain: %v", i, err)
454454
continue

‎core/bench_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
175175

176176
// Time the insertion of the new chain.
177177
// State and blocks are stored in the same DB.
178-
chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
178+
chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil)
179179
defer chainman.Stop()
180180
b.ReportAllocs()
181181
b.ResetTimer()
@@ -287,7 +287,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
287287
if err != nil {
288288
b.Fatalf("error opening database at %v: %v", dir, err)
289289
}
290-
chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
290+
chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
291291
if err != nil {
292292
b.Fatalf("error creating chain: %v", err)
293293
}

‎core/block_validator_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func TestHeaderVerification(t *testing.T) {
4242
headers[i] = block.Header()
4343
}
4444
// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
45-
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
45+
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
4646
defer chain.Stop()
4747

4848
for i := 0; i < len(blocks); i++ {
@@ -106,11 +106,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) {
106106
var results <-chan error
107107

108108
if valid {
109-
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
109+
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
110110
_, results = chain.engine.VerifyHeaders(chain, headers, seals)
111111
chain.Stop()
112112
} else {
113-
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil)
113+
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil)
114114
_, results = chain.engine.VerifyHeaders(chain, headers, seals)
115115
chain.Stop()
116116
}
@@ -173,7 +173,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) {
173173
defer runtime.GOMAXPROCS(old)
174174

175175
// Start the verifications and immediately abort
176-
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil)
176+
chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil)
177177
defer chain.Stop()
178178

179179
abort, results := chain.engine.VerifyHeaders(chain, headers, seals)

0 commit comments

Comments
 (0)