chore(core/rawdb): InspectDatabase uses libevm (8) (#791)

- Part of the core/rawdb migration to use libevm
- Using libevm `InspectDatabase`
- Define options passed to libevm `InspectDatabase`

Author: qdm12

core/rawdb/database.go

@@ -31,15 +31,14 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/ava-labs/libevm/common"
+	ethrawdb "github.com/ava-labs/libevm/core/rawdb"
 	"github.com/ava-labs/libevm/ethdb"
 	"github.com/ava-labs/libevm/ethdb/leveldb"
 	"github.com/ava-labs/libevm/ethdb/memorydb"
 	"github.com/ava-labs/libevm/ethdb/pebble"
 	"github.com/ava-labs/libevm/log"
-	"github.com/olekukonko/tablewriter"
 )
 
 // nofreezedb is a database wrapper that disables freezer data retrievals.
@@ -242,192 +241,65 @@ func Open(o OpenOptions) (ethdb.Database, error) {
 	return kvdb, nil
 }
 
-type counter uint64
-
-func (c counter) String() string {
-	return fmt.Sprintf("%d", c)
-}
-
-func (c counter) Percentage(current uint64) string {
-	return fmt.Sprintf("%d", current*100/uint64(c))
-}
-
-// stat stores sizes and count for a parameter
-type stat struct {
-	size  common.StorageSize
-	count counter
-}
-
-// Add size to the stat and increase the counter by 1
-func (s *stat) Add(size common.StorageSize) {
-	s.size += size
-	s.count++
-}
-
-func (s *stat) Size() string {
-	return s.size.String()
-}
-
-func (s *stat) Count() string {
-	return s.count.String()
-}
-
 // InspectDatabase traverses the entire database and checks the size
 // of all different categories of data.
 func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
-	it := db.NewIterator(keyPrefix, keyStart)
-	defer it.Release()
-
 	var (
-		count  int64
-		start  = time.Now()
-		logged = time.Now()
-
-		// Key-value store statistics
-		headers         stat
-		bodies          stat
-		receipts        stat
-		numHashPairings stat
-		hashNumPairings stat
-		legacyTries     stat
-		stateLookups    stat
-		accountTries    stat
-		storageTries    stat
-		codes           stat
-		txLookups       stat
-		accountSnaps    stat
-		storageSnaps    stat
-		preimages       stat
-		bloomBits       stat
-		cliqueSnaps     stat
-
-		// State sync statistics
-		codeToFetch   stat
-		syncProgress  stat
-		syncSegments  stat
-		syncPerformed stat
-
-		// Les statistic
-		chtTrieNodes   stat
-		bloomTrieNodes stat
-
-		// Meta- and unaccounted data
-		metadata    stat
-		unaccounted stat
-
-		// Totals
-		total common.StorageSize
+		codeToFetch   ethrawdb.DatabaseStat
+		syncPerformed ethrawdb.DatabaseStat
+		syncProgress  ethrawdb.DatabaseStat
+		syncSegments  ethrawdb.DatabaseStat
 	)
-	// Inspect key-value database first.
-	for it.Next() {
-		var (
-			key  = it.Key()
-			size = common.StorageSize(len(key) + len(it.Value()))
-		)
-		total += size
-		switch {
-		case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
-			headers.Add(size)
-		case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
-			bodies.Add(size)
-		case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
-			receipts.Add(size)
-		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
-			numHashPairings.Add(size)
-		case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
-			hashNumPairings.Add(size)
-		case IsLegacyTrieNode(key, it.Value()):
-			legacyTries.Add(size)
-		case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
-			stateLookups.Add(size)
-		case IsAccountTrieNode(key):
-			accountTries.Add(size)
-		case IsStorageTrieNode(key):
-			storageTries.Add(size)
-		case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
-			codes.Add(size)
-		case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
-			txLookups.Add(size)
-		case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
-			accountSnaps.Add(size)
-		case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
-			storageSnaps.Add(size)
-		case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength):
-			preimages.Add(size)
-		case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength):
-			metadata.Add(size)
-		case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
-			bloomBits.Add(size)
-		case bytes.HasPrefix(key, BloomBitsIndexPrefix):
-			bloomBits.Add(size)
-		case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength:
-			syncProgress.Add(size)
-		case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength:
-			syncSegments.Add(size)
-		case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength:
-			codeToFetch.Add(size)
-		case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength:
-			syncPerformed.Add(size)
-		default:
-			var accounted bool
-			for _, meta := range [][]byte{
-				databaseVersionKey, headHeaderKey, headBlockKey,
-				snapshotRootKey, snapshotBlockHashKey, snapshotGeneratorKey,
-				uncleanShutdownKey, syncRootKey, txIndexTailKey,
-				persistentStateIDKey, trieJournalKey,
-			} {
-				if bytes.Equal(key, meta) {
-					metadata.Add(size)
-					accounted = true
-					break
-				}
+
+	options := []ethrawdb.InspectDatabaseOption{
+		ethrawdb.WithDatabaseMetadataKeys(func(key []byte) bool {
+			return bytes.Equal(key, snapshotBlockHashKey) ||
+				bytes.Equal(key, syncRootKey)
+		}),
+		ethrawdb.WithDatabaseStatRecorder(func(key []byte, size common.StorageSize) bool {
+			switch {
+			case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength:
+				syncSegments.Add(size)
+				return true
+			case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength:
+				syncProgress.Add(size)
+				return true
+			case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength:
+				codeToFetch.Add(size)
+				return true
+			case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength:
+				syncPerformed.Add(size)
+				return true
+			default:
+				return false
 			}
-			if !accounted {
-				unaccounted.Add(size)
+		}),
+		ethrawdb.WithDatabaseStatsTransformer(func(rows [][]string) [][]string {
+			newRows := make([][]string, 0, len(rows))
+			for _, row := range rows {
+				database := row[0]
+				category := row[1]
+				switch {
+				case database == "Key-Value store" && category == "Difficulties",
+					database == "Key-Value store" && category == "Beacon sync headers",
+					database == "Ancient store (Chain)":
+					// Discard rows specific to libevm (geth) but irrelevant to coreth.
+					continue
+				}
+				newRows = append(newRows, row)
 			}
-		}
-		count++
-		if count%1000 == 0 && time.Since(logged) > 8*time.Second {
-			log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
-			logged = time.Now()
-		}
-	}
-	// Display the database statistic.
-	stats := [][]string{
-		{"Key-Value store", "Headers", headers.Size(), headers.Count()},
-		{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
-		{"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()},
-		{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()},
-		{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
-		{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
-		{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
-		{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
-		{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
-		{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
-		{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
-		{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
-		{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
-		{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
-		{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},
-		{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
-		{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
-		{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
-		{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
-		{"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()},
-		{"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()},
-		{"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()},
-		{"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()},
-	}
-	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Database", "Category", "Size", "Items"})
-	table.SetFooter([]string{"", "Total", total.String(), " "})
-	table.AppendBulk(stats)
-	table.Render()
-
-	if unaccounted.size > 0 {
-		log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
+
+			return append(
+				newRows,
+				[]string{"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()},
+				[]string{"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()},
+				[]string{"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()},
+				[]string{"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()},
+			)
+		}),
 	}
-	return nil
+
+	return ethrawdb.InspectDatabase(db, keyPrefix, keyStart, options...)
 }
 
 // ClearPrefix removes all keys in db that begin with prefix and match an

go.mod

@@ -19,7 +19,6 @@ require (
 	github.com/holiman/uint256 v1.2.4
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-isatty v0.0.17
-	github.com/olekukonko/tablewriter v0.0.5
 	github.com/prometheus/client_golang v1.16.0
 	github.com/prometheus/client_model v0.3.0
 	github.com/spf13/cast v1.5.0
@@ -91,6 +90,7 @@ require (
 	github.com/mitchellh/pointerstructure v1.2.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect