From a3991058beda3b62a437c9a557a4ac0fdfc34403 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 21 Dec 2023 20:30:04 +0800 Subject: [PATCH 1/6] core, cmd, trie: fix the condition of pathdb initialization --- cmd/geth/dbcmd.go | 86 +++++++++++++++++----------------- core/rawdb/ancient_scheme.go | 8 ++-- core/rawdb/ancient_utils.go | 12 ++--- core/rawdb/database.go | 2 +- trie/triedb/pathdb/database.go | 36 ++++++++++---- 5 files changed, 83 insertions(+), 61 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index c60147b86217..a2aa37d28a83 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -198,60 +198,62 @@ WARNING: This is a low-level operation which may cause database corruption!`, func removeDB(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) - // Remove the full node state database - path := stack.ResolvePath("chaindata") - if common.FileExist(path) { - confirmAndRemoveDB(path, "full node state database") - } else { - log.Info("Full node state database missing", "path", path) - } - // Remove the full node ancient database - path = config.Eth.DatabaseFreezer + // Resolve folder paths. + var ( + rootDir = stack.ResolvePath("chaindata") + ancientDir = config.Eth.DatabaseFreezer + ) switch { - case path == "": - path = filepath.Join(stack.ResolvePath("chaindata"), "ancient") - case !filepath.IsAbs(path): - path = config.Node.ResolvePath(path) - } - if common.FileExist(path) { - confirmAndRemoveDB(path, "full node ancient database") - } else { - log.Info("Full node ancient database missing", "path", path) - } - // Remove the light node database - path = stack.ResolvePath("lightchaindata") - if common.FileExist(path) { - confirmAndRemoveDB(path, "light node database") - } else { - log.Info("Light node database missing", "path", path) - } + case ancientDir == "": + ancientDir = filepath.Join(stack.ResolvePath("chaindata"), "ancient") + case !filepath.IsAbs(ancientDir): + ancientDir = config.Node.ResolvePath(ancientDir) + } + // Delete state data + statePaths := []string{rootDir, filepath.Join(ancientDir, rawdb.StateFreezerName)} + confirmAndRemoveDB(statePaths, "full node state database") + + // Delete ancient chain + chainPaths := []string{filepath.Join(ancientDir, rawdb.ChainFreezerName)} + confirmAndRemoveDB(chainPaths, "full node ancient chain") return nil } +// removeFolder deletes all the files inside the folder unexcept the subfolders. +func removeFolder(dir string) { + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // If we're at the top level folder, recurse into + if path == dir { + return nil + } + // Delete all the files, but not subfolders + if !info.IsDir() { + os.Remove(path) + return nil + } + return filepath.SkipDir + }) +} + // confirmAndRemoveDB prompts the user for a last confirmation and removes the -// folder if accepted. -func confirmAndRemoveDB(database string, kind string) { - confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database)) +// list of folders if accepted. +func confirmAndRemoveDB(paths []string, kind string) { + confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, paths)) switch { case err != nil: utils.Fatalf("%v", err) case !confirm: - log.Info("Database deletion skipped", "path", database) + log.Info("Database deletion skipped", "kind", kind, "paths", paths) default: start := time.Now() - filepath.Walk(database, func(path string, info os.FileInfo, err error) error { - // If we're at the top level folder, recurse into - if path == database { - return nil - } - // Delete all the files, but not subfolders - if !info.IsDir() { - os.Remove(path) - return nil + for _, path := range paths { + if common.FileExist(path) { + removeFolder(path) + } else { + log.Info("Folder is not existent", "path", path) } - return filepath.SkipDir - }) - log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start))) + } + log.Info("Database successfully deleted", "kind", kind, "paths", paths, "elapsed", common.PrettyDuration(time.Since(start))) } } diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index 6f409fff1d0a..e88867af0e64 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -68,14 +68,14 @@ var stateFreezerNoSnappy = map[string]bool{ // The list of identifiers of ancient stores. var ( - chainFreezerName = "chain" // the folder name of chain segment ancient store. - stateFreezerName = "state" // the folder name of reverse diff ancient store. + ChainFreezerName = "chain" // the folder name of chain segment ancient store. + StateFreezerName = "state" // the folder name of reverse diff ancient store. ) // freezers the collections of all builtin freezers. -var freezers = []string{chainFreezerName, stateFreezerName} +var freezers = []string{ChainFreezerName, StateFreezerName} // NewStateFreezer initializes the freezer for state history. func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) { - return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy) + return NewResettableFreezer(filepath.Join(ancientDir, StateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy) } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 1b93a9aa5a85..428cda544b03 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -81,14 +81,14 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { var infos []freezerInfo for _, freezer := range freezers { switch freezer { - case chainFreezerName: - info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db) + case ChainFreezerName: + info, err := inspect(ChainFreezerName, chainFreezerNoSnappy, db) if err != nil { return nil, err } infos = append(infos, info) - case stateFreezerName: + case StateFreezerName: if ReadStateScheme(db) != PathScheme { continue } @@ -102,7 +102,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { } defer f.Close() - info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f) + info, err := inspect(StateFreezerName, stateFreezerNoSnappy, f) if err != nil { return nil, err } @@ -125,9 +125,9 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s tables map[string]bool ) switch freezerName { - case chainFreezerName: + case ChainFreezerName: path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy - case stateFreezerName: + case StateFreezerName: path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy default: return fmt.Errorf("unknown freezer, supported ones: %v", freezers) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 1d7b7d1ca89c..18b5bccb517c 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -178,7 +178,7 @@ func resolveChainFreezerDir(ancient string) string { // sub folder, if not then two possibilities: // - chain freezer is not initialized // - chain freezer exists in legacy location (root ancient folder) - freezer := path.Join(ancient, chainFreezerName) + freezer := path.Join(ancient, ChainFreezerName) if !common.FileExist(freezer) { if !common.FileExist(ancient) { // The entire ancient store is not initialized, still use the sub diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index dc64414e9b52..2bb52fe7f4e0 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -170,14 +170,31 @@ func New(diskdb ethdb.Database, config *Config) *Database { } db.freezer = freezer - // Truncate the extra state histories above in freezer in case - // it's not aligned with the disk layer. - pruned, err := truncateFromHead(db.diskdb, freezer, db.tree.bottom().stateID()) - if err != nil { - log.Crit("Failed to truncate extra state histories", "err", err) - } - if pruned != 0 { - log.Warn("Truncated extra state histories", "number", pruned) + diskLayerID := db.tree.bottom().stateID() + if diskLayerID == 0 { + // Reset the entire state histories in case the trie database is + // not initialized yet, as these state histories are not expected. + frozen, err := db.freezer.Ancients() + if err != nil { + log.Crit("Failed to retrieve head of state history", "err", err) + } + if frozen != 0 { + err := db.freezer.Reset() + if err != nil { + log.Crit("Failed to reset state histories", "err", err) + } + log.Info("Truncated the unexpected state history") + } + } else { + // Truncate the extra state histories above in freezer in case + // it's not aligned with the disk layer. + pruned, err := truncateFromHead(db.diskdb, freezer, diskLayerID) + if err != nil { + log.Crit("Failed to truncate extra state histories", "err", err) + } + if pruned != 0 { + log.Warn("Truncated extra state histories", "number", pruned) + } } } // Disable database in case node is still in the initial state sync stage. @@ -431,6 +448,9 @@ func (db *Database) Initialized(genesisRoot common.Hash) bool { inited = true } }) + if !inited { + inited = rawdb.ReadSnapSyncStatusFlag(db.diskdb) != rawdb.StateSyncUnknown + } return inited } From 81bbc4f1c0eaec1fee15897c0df862ccba6a87f2 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 21 Dec 2023 21:46:13 +0800 Subject: [PATCH 2/6] cmd/geth: improve prompt --- cmd/geth/dbcmd.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index a2aa37d28a83..c057675bdd50 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -238,22 +238,30 @@ func removeFolder(dir string) { // confirmAndRemoveDB prompts the user for a last confirmation and removes the // list of folders if accepted. func confirmAndRemoveDB(paths []string, kind string) { - confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, paths)) + msg := fmt.Sprintf("Remove %s?\n", kind) + for _, path := range paths { + msg += fmt.Sprintf("\t- %s\n", path) + } + confirm, err := prompt.Stdin.PromptConfirm(msg) switch { case err != nil: utils.Fatalf("%v", err) case !confirm: log.Info("Database deletion skipped", "kind", kind, "paths", paths) default: - start := time.Now() + var ( + deleted []string + start = time.Now() + ) for _, path := range paths { if common.FileExist(path) { removeFolder(path) + deleted = append(deleted, path) } else { log.Info("Folder is not existent", "path", path) } } - log.Info("Database successfully deleted", "kind", kind, "paths", paths, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Database successfully deleted", "kind", kind, "paths", deleted, "elapsed", common.PrettyDuration(time.Since(start))) } } From f2ccd41c37640ce5122a8ff4f5770d276cc2b596 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 21 Dec 2023 21:56:36 +0800 Subject: [PATCH 3/6] cmd/geth: improve msg --- cmd/geth/dbcmd.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index c057675bdd50..502f6d2f1d76 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -211,11 +211,11 @@ func removeDB(ctx *cli.Context) error { } // Delete state data statePaths := []string{rootDir, filepath.Join(ancientDir, rawdb.StateFreezerName)} - confirmAndRemoveDB(statePaths, "full node state database") + confirmAndRemoveDB(statePaths, "state database") // Delete ancient chain chainPaths := []string{filepath.Join(ancientDir, rawdb.ChainFreezerName)} - confirmAndRemoveDB(chainPaths, "full node ancient chain") + confirmAndRemoveDB(chainPaths, "ancient chain") return nil } @@ -238,9 +238,9 @@ func removeFolder(dir string) { // confirmAndRemoveDB prompts the user for a last confirmation and removes the // list of folders if accepted. func confirmAndRemoveDB(paths []string, kind string) { - msg := fmt.Sprintf("Remove %s?\n", kind) + msg := fmt.Sprintf("Remove %s? ", kind) for _, path := range paths { - msg += fmt.Sprintf("\t- %s\n", path) + msg += fmt.Sprintf("(%s) ", path) } confirm, err := prompt.Stdin.PromptConfirm(msg) switch { From 98247fe4ae644de619b34e9fd7e8c1534193826a Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 21 Dec 2023 21:57:17 +0800 Subject: [PATCH 4/6] Update cmd/geth/dbcmd.go Co-authored-by: Martin HS --- cmd/geth/dbcmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 502f6d2f1d76..6f07c0381bb7 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -219,7 +219,7 @@ func removeDB(ctx *cli.Context) error { return nil } -// removeFolder deletes all the files inside the folder unexcept the subfolders. +// removeFolder deletes all files (not folders) inside the directory 'dir' (but not files in subfolders). func removeFolder(dir string) { filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { // If we're at the top level folder, recurse into From 4caaf52982be484be05b8a02bf9c9712dde4d603 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 21 Dec 2023 21:57:28 +0800 Subject: [PATCH 5/6] Update trie/triedb/pathdb/database.go Co-authored-by: Martin HS --- trie/triedb/pathdb/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 2bb52fe7f4e0..f2d6cea635a9 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -183,7 +183,7 @@ func New(diskdb ethdb.Database, config *Config) *Database { if err != nil { log.Crit("Failed to reset state histories", "err", err) } - log.Info("Truncated the unexpected state history") + log.Info("Truncated extraneous state history") } } else { // Truncate the extra state histories above in freezer in case From 15c35808aad9e9a5a3ad1eae46df3dcadfc1e1fc Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 21 Dec 2023 22:10:52 +0800 Subject: [PATCH 6/6] cmd: improve msg --- cmd/geth/dbcmd.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 6f07c0381bb7..1ae026fd29c0 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -211,7 +211,7 @@ func removeDB(ctx *cli.Context) error { } // Delete state data statePaths := []string{rootDir, filepath.Join(ancientDir, rawdb.StateFreezerName)} - confirmAndRemoveDB(statePaths, "state database") + confirmAndRemoveDB(statePaths, "state data") // Delete ancient chain chainPaths := []string{filepath.Join(ancientDir, rawdb.ChainFreezerName)} @@ -219,7 +219,8 @@ func removeDB(ctx *cli.Context) error { return nil } -// removeFolder deletes all files (not folders) inside the directory 'dir' (but not files in subfolders). +// removeFolder deletes all files (not folders) inside the directory 'dir' (but +// not files in subfolders). func removeFolder(dir string) { filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { // If we're at the top level folder, recurse into @@ -238,11 +239,13 @@ func removeFolder(dir string) { // confirmAndRemoveDB prompts the user for a last confirmation and removes the // list of folders if accepted. func confirmAndRemoveDB(paths []string, kind string) { - msg := fmt.Sprintf("Remove %s? ", kind) + msg := fmt.Sprintf("Location(s) of '%s': \n", kind) for _, path := range paths { - msg += fmt.Sprintf("(%s) ", path) + msg += fmt.Sprintf("\t- %s\n", path) } - confirm, err := prompt.Stdin.PromptConfirm(msg) + fmt.Println(msg) + + confirm, err := prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove '%s'?", kind)) switch { case err != nil: utils.Fatalf("%v", err)