diff --git a/CHANGELOG.md b/CHANGELOG.md index 26734202ca..2a0c994a7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## UNRELEASED +* [#1908](https://github.com/crypto-org-chain/cronos/pull/1908) Add db migration/patch CLI tool * [#1869](https://github.com/crypto-org-chain/cronos/pull/1869) Add missing tx context during vm initialisation * [#1872](https://github.com/crypto-org-chain/cronos/pull/1872) Support 4byteTracer for tracer * [#1875](https://github.com/crypto-org-chain/cronos/pull/1875) Support for preinstalls diff --git a/Makefile b/Makefile index b81cea4520..cbb123aa84 100644 --- a/Makefile +++ b/Makefile @@ -109,17 +109,17 @@ build: check-network print-ledger go.sum install: check-network print-ledger go.sum @go install -mod=readonly $(BUILD_FLAGS) ./cmd/cronosd -test: test-memiavl test-store +test: test-memiavl test-store test-versiondb @go test -tags=objstore -v -mod=readonly $(PACKAGES) -coverprofile=$(COVERAGE) -covermode=atomic test-memiavl: - @cd memiavl; go test -tags=objstore -v -mod=readonly ./... -coverprofile=$(COVERAGE) -covermode=atomic; + @cd memiavl && go test -tags=objstore -v -mod=readonly ./... -coverprofile=$(COVERAGE) -covermode=atomic; test-store: - @cd store; go test -tags=objstore -v -mod=readonly ./... -coverprofile=$(COVERAGE) -covermode=atomic; + @cd store && go test -tags=objstore -v -mod=readonly ./... -coverprofile=$(COVERAGE) -covermode=atomic; test-versiondb: - @cd versiondb; go test -tags=objstore,rocksdb -v -mod=readonly ./... -coverprofile=$(COVERAGE) -covermode=atomic; + @cd versiondb && go test -tags=objstore,rocksdb -v -mod=readonly ./... -coverprofile=$(COVERAGE) -covermode=atomic; .PHONY: clean build install test test-memiavl test-store test-versiondb diff --git a/cmd/cronosd/cmd/database.go b/cmd/cronosd/cmd/database.go new file mode 100644 index 0000000000..129a210208 --- /dev/null +++ b/cmd/cronosd/cmd/database.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +// DatabaseCmd returns the database command with subcommands +func DatabaseCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "database", + Short: "Database management commands", + Long: `Commands for managing Cronos databases. + +Available subcommands: + migrate - Migrate databases between different backend types + patch - Patch specific block heights into existing databases + +Use "cronosd database [command] --help" for more information about a command.`, + Aliases: []string{"db"}, + } + + // Add subcommands + cmd.AddCommand( + MigrateCmd(), + PatchCmd(), + ) + + return cmd +} diff --git a/cmd/cronosd/cmd/migrate_db.go b/cmd/cronosd/cmd/migrate_db.go new file mode 100644 index 0000000000..f4f393450f --- /dev/null +++ b/cmd/cronosd/cmd/migrate_db.go @@ -0,0 +1,355 @@ +package cmd + +import ( + "fmt" + "strings" + + dbm "github.com/cosmos/cosmos-db" + "github.com/crypto-org-chain/cronos/cmd/cronosd/dbmigrate" + "github.com/spf13/cobra" + + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" +) + +const ( + flagSourceBackend = "source-backend" + flagTargetBackend = "target-backend" + flagTargetHome = "target-home" + flagBatchSize = "batch-size" + flagVerify = "verify" + flagDBType = "db-type" + flagDatabases = "databases" +) + +type DbType string + +// Database type constants +const ( + App DbType = "app" + CometBFT DbType = "cometbft" + All DbType = "all" +) + +type BackendType string + +// Backend type constants +const ( + GoLevelDB BackendType = "goleveldb" + LevelDB BackendType = "leveldb" // Alias for goleveldb + RocksDB BackendType = "rocksdb" +) + +type DatabaseName string + +// Database name constants +const ( + Application DatabaseName = "application" + Blockstore DatabaseName = "blockstore" + State DatabaseName = "state" + TxIndex DatabaseName = "tx_index" + Evidence DatabaseName = "evidence" +) + +// Valid database names +var validDatabaseNames = map[string]bool{ + string(Application): true, + string(Blockstore): true, + string(State): true, + string(TxIndex): true, + string(Evidence): true, +} + +// MigrateDBCmd returns the migrate command +func MigrateDBCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Migrate databases from one backend to another (e.g., leveldb to rocksdb)", + Long: `Migrate databases from one backend to another. + +This command migrates databases from a source backend to a target backend. +It can migrate the application database, CometBFT databases, or both. + +The migration process: +1. Opens the source database(s) in read-only mode +2. Creates new temporary target database(s) +3. Copies all key-value pairs in batches +4. Optionally verifies the migration +5. Creates the target database(s) in a temporary location + +Database types (--db-type): + - app: Application database only (application.db) + - cometbft: CometBFT databases only (blockstore.db, state.db, tx_index.db, evidence.db) + - all: Both application and CometBFT databases + +Specific databases (--databases): +You can also specify individual databases as a comma-separated list: + - application: Chain state + - blockstore: Block data + - state: Latest state + - tx_index: Transaction indexing + - evidence: Misbehavior evidence + +NOTE: This command performs FULL database migration (all keys). +For selective height-based patching, use 'database patch' or 'db patch' instead. + +IMPORTANT: +- Always backup your databases before migration +- The source databases are opened in read-only mode and are not modified +- The target databases are created with a .migrate-temp.db suffix (e.g., application.migrate-temp.db) +- After successful migration, you need to manually replace the original databases +- Stop your node before running this command + +Examples: + # Migrate application database only (using --db-type) + cronosd db migrate --source-backend goleveldb --target-backend rocksdb --db-type app --home ~/.cronos + + # Migrate CometBFT databases only (using --db-type) + cronosd db migrate --source-backend goleveldb --target-backend rocksdb --db-type cometbft --home ~/.cronos + + # Migrate all databases (using --db-type) + cronosd db migrate --source-backend goleveldb --target-backend rocksdb --db-type all --home ~/.cronos + + # Migrate specific databases (using --databases) + cronosd db migrate --source-backend goleveldb --target-backend rocksdb --databases blockstore,tx_index --home ~/.cronos + + # Migrate multiple specific databases + cronosd db migrate --source-backend goleveldb --target-backend rocksdb --databases application,blockstore,state --home ~/.cronos + + # Migrate with verification + cronosd db migrate --source-backend goleveldb --target-backend rocksdb --db-type all --verify --home ~/.cronos +`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := server.GetServerContextFromCmd(cmd) + logger := ctx.Logger + + homeDir := ctx.Viper.GetString(flags.FlagHome) + sourceBackend := ctx.Viper.GetString(flagSourceBackend) + targetBackend := ctx.Viper.GetString(flagTargetBackend) + targetHome := ctx.Viper.GetString(flagTargetHome) + batchSize := ctx.Viper.GetInt(flagBatchSize) + verify := ctx.Viper.GetBool(flagVerify) + dbType := ctx.Viper.GetString(flagDBType) + databases := ctx.Viper.GetString(flagDatabases) + + // Parse backend types + sourceBackendType, err := parseBackendType(BackendType(sourceBackend)) + if err != nil { + return fmt.Errorf("invalid source backend: %w", err) + } + + targetBackendType, err := parseBackendType(BackendType(targetBackend)) + if err != nil { + return fmt.Errorf("invalid target backend: %w", err) + } + + if sourceBackendType == targetBackendType { + return fmt.Errorf("source and target backends must be different") + } + + if targetHome == "" { + targetHome = homeDir + logger.Info("Target home not specified, using source home directory", "target_home", targetHome) + } + + // Determine which databases to migrate + var dbNames []DatabaseName + + // If --databases flag is provided, use it (takes precedence over --db-type) + if databases != "" { + var err error + dbNamesStr, err := parseDatabaseNames(databases) + if err != nil { + return err + } + // Convert []string to []DatabaseName + dbNames = make([]DatabaseName, len(dbNamesStr)) + for i, name := range dbNamesStr { + dbNames[i] = DatabaseName(name) + } + } else { + // Fall back to --db-type flag + var err error + dbNames, err = getDBNamesFromType(DbType(dbType)) + if err != nil { + return err + } + } + + // Convert dbNames to []string for logging + dbNamesStr := make([]string, len(dbNames)) + for i, name := range dbNames { + dbNamesStr[i] = string(name) + } + + logger.Info("Database migration configuration", + "source_home", homeDir, + "target_home", targetHome, + "source_backend", sourceBackend, + "target_backend", targetBackend, + "databases", dbNamesStr, + "batch_size", batchSize, + "verify", verify, + ) + + // Prepare RocksDB options if target is RocksDB + var rocksDBOpts interface{} + if targetBackendType == dbm.RocksDBBackend { + // Use the same RocksDB options as the application (implemented in build-tagged files) + rocksDBOpts = dbmigrate.PrepareRocksDBOptions() + } + + // Migrate each database + var totalStats dbmigrate.MigrationStats + for _, dbName := range dbNames { + dbNameStr := string(dbName) + logger.Info("Starting migration", "database", dbNameStr) + + opts := dbmigrate.MigrateOptions{ + SourceHome: homeDir, + TargetHome: targetHome, + SourceBackend: sourceBackendType, + TargetBackend: targetBackendType, + BatchSize: batchSize, + Logger: logger, + RocksDBOptions: rocksDBOpts, + Verify: verify, + DBName: dbNameStr, + } + + stats, err := dbmigrate.Migrate(opts) + if err != nil { + if stats != nil { + logger.Error("Migration failed", + "database", dbNameStr, + "error", err, + "processed_keys", stats.ProcessedKeys.Load(), + "total_keys", stats.TotalKeys.Load(), + "duration", stats.Duration(), + ) + } else { + logger.Error("Migration failed", + "database", dbNameStr, + "error", err, + ) + } + return fmt.Errorf("failed to migrate %s: %w", dbNameStr, err) + } + + logger.Info("Database migration completed", + "database", dbNameStr, + "total_keys", stats.TotalKeys.Load(), + "processed_keys", stats.ProcessedKeys.Load(), + "errors", stats.ErrorCount.Load(), + "duration", stats.Duration(), + ) + + // Accumulate stats + totalStats.TotalKeys.Add(stats.TotalKeys.Load()) + totalStats.ProcessedKeys.Add(stats.ProcessedKeys.Load()) + totalStats.ErrorCount.Add(stats.ErrorCount.Load()) + } + + logger.Info(strings.Repeat("=", 80)) + logger.Info("ALL MIGRATIONS COMPLETED SUCCESSFULLY") + logger.Info(strings.Repeat("=", 80)) + + if databases != "" { + logger.Info("Migration summary", + "databases", strings.Join(dbNamesStr, ", "), + "total_keys", totalStats.TotalKeys.Load(), + "processed_keys", totalStats.ProcessedKeys.Load(), + "errors", totalStats.ErrorCount.Load(), + ) + } else { + logger.Info("Migration summary", + "database_type", dbType, + "total_keys", totalStats.TotalKeys.Load(), + "processed_keys", totalStats.ProcessedKeys.Load(), + "errors", totalStats.ErrorCount.Load(), + ) + } + + logger.Info("IMPORTANT NEXT STEPS:") + logger.Info("1. Backup your original databases") + logger.Info("2. Verify the migration was successful") + logger.Info("3. Migrated databases are located at:") + for _, dbName := range dbNames { + logger.Info(" Migrated database location", "path", fmt.Sprintf("%s/data/%s.migrate-temp.db", targetHome, string(dbName))) + } + logger.Info("4. Replace the original databases with the migrated ones") + logger.Info("5. Update your config.toml to use the new backend type") + logger.Info(strings.Repeat("=", 80)) + + return nil + }, + } + + cmd.Flags().StringP(flagSourceBackend, "s", string(GoLevelDB), "Source database backend type (goleveldb, rocksdb)") + cmd.Flags().StringP(flagTargetBackend, "t", string(RocksDB), "Target database backend type (goleveldb, rocksdb)") + cmd.Flags().StringP(flagTargetHome, "o", "", "Target home directory (default: same as --home)") + cmd.Flags().IntP(flagBatchSize, "b", dbmigrate.DefaultBatchSize, "Number of key-value pairs to process in a batch") + cmd.Flags().BoolP(flagVerify, "v", true, "Verify migration by comparing source and target databases") + cmd.Flags().StringP(flagDBType, "y", string(App), "Database type to migrate: app (application.db only), cometbft (CometBFT databases only), all (both)") + cmd.Flags().StringP(flagDatabases, "d", "", "Comma-separated list of specific databases to migrate (e.g., 'blockstore,tx_index'). Valid names: application, blockstore, state, tx_index, evidence. If specified, this flag takes precedence over --db-type") + + return cmd +} + +// MigrateCmd returns the migrate subcommand (for database command group) +func MigrateCmd() *cobra.Command { + cmd := MigrateDBCmd() + cmd.Use = "migrate" + cmd.Deprecated = "" + return cmd +} + +// parseBackendType parses a backend type into dbm.BackendType +func parseBackendType(backend BackendType) (dbm.BackendType, error) { + switch backend { + case GoLevelDB, LevelDB: + return dbm.GoLevelDBBackend, nil + case RocksDB: + return dbm.RocksDBBackend, nil + default: + return "", fmt.Errorf("unsupported backend type: %s (supported: goleveldb, leveldb, rocksdb)", backend) + } +} + +// parseDatabaseNames parses a comma-separated list of database names and validates them +func parseDatabaseNames(databases string) ([]string, error) { + if databases == "" { + return nil, fmt.Errorf("no databases specified") + } + + dbList := strings.Split(databases, ",") + var dbNames []string + for _, dbName := range dbList { + dbName = strings.TrimSpace(dbName) + if dbName == "" { + continue + } + if !validDatabaseNames[dbName] { + return nil, fmt.Errorf("invalid database name: %s (valid names: application, blockstore, state, tx_index, evidence)", dbName) + } + dbNames = append(dbNames, dbName) + } + if len(dbNames) == 0 { + return nil, fmt.Errorf("no valid databases specified in --databases flag") + } + return dbNames, nil +} + +// getDBNamesFromType returns the list of database names for a given db-type +func getDBNamesFromType(dbType DbType) ([]DatabaseName, error) { + switch dbType { + case App: + return []DatabaseName{Application}, nil + case CometBFT: + return []DatabaseName{Blockstore, State, TxIndex, Evidence}, nil + case All: + return []DatabaseName{Application, Blockstore, State, TxIndex, Evidence}, nil + default: + return nil, fmt.Errorf("invalid db-type: %s (must be: app, cometbft, or all)", dbType) + } +} diff --git a/cmd/cronosd/cmd/migrate_db_test.go b/cmd/cronosd/cmd/migrate_db_test.go new file mode 100644 index 0000000000..49abf146ed --- /dev/null +++ b/cmd/cronosd/cmd/migrate_db_test.go @@ -0,0 +1,307 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestParseBackendType tests the backend type parsing function +func TestParseBackendType(t *testing.T) { + tests := []struct { + name string + input BackendType + expectError bool + }{ + { + name: "goleveldb", + input: GoLevelDB, + expectError: false, + }, + { + name: "leveldb alias", + input: LevelDB, + expectError: false, + }, + { + name: "rocksdb", + input: RocksDB, + expectError: false, + }, + { + name: "invalid backend", + input: "invaliddb", + expectError: true, + }, + { + name: "empty string", + input: "", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := parseBackendType(tt.input) + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.NotEmpty(t, result) + } + }) + } +} + +// TestValidDatabaseNames tests that all expected database names are valid +func TestValidDatabaseNames(t *testing.T) { + expectedDatabases := []string{ + "application", + "blockstore", + "state", + "tx_index", + "evidence", + } + + for _, dbName := range expectedDatabases { + t.Run(dbName, func(t *testing.T) { + require.True(t, validDatabaseNames[dbName], "database %s should be valid", dbName) + }) + } + + // Test invalid names + invalidNames := []string{ + "invalid", + "app", + "cometbft", + "", + "application.db", + "blockstore_db", + } + + for _, dbName := range invalidNames { + t.Run("invalid_"+dbName, func(t *testing.T) { + require.False(t, validDatabaseNames[dbName], "database %s should be invalid", dbName) + }) + } +} + +// TestDatabaseNameParsing tests parsing of comma-separated database names +func TestDatabaseNameParsing(t *testing.T) { + tests := []struct { + name string + input string + expectedDBs []string + expectError bool + errorSubstring string + }{ + { + name: "single database", + input: "application", + expectedDBs: []string{"application"}, + expectError: false, + }, + { + name: "two databases", + input: "blockstore,tx_index", + expectedDBs: []string{"blockstore", "tx_index"}, + expectError: false, + }, + { + name: "all databases", + input: "application,blockstore,state,tx_index,evidence", + expectedDBs: []string{"application", "blockstore", "state", "tx_index", "evidence"}, + expectError: false, + }, + { + name: "with spaces", + input: "blockstore, tx_index, state", + expectedDBs: []string{"blockstore", "tx_index", "state"}, + expectError: false, + }, + { + name: "with extra spaces", + input: " application , blockstore ", + expectedDBs: []string{"application", "blockstore"}, + expectError: false, + }, + { + name: "invalid database name", + input: "application,invalid_db,blockstore", + expectError: true, + errorSubstring: "invalid database name", + }, + { + name: "only invalid database", + input: "invalid_db", + expectError: true, + errorSubstring: "invalid database name", + }, + { + name: "empty after trimming", + input: "application,,blockstore", + expectedDBs: []string{"application", "blockstore"}, + expectError: false, + }, + { + name: "only empty strings", + input: ",,,", + expectError: true, + errorSubstring: "no valid databases specified", + }, + { + name: "empty string", + input: "", + expectError: true, + errorSubstring: "no databases specified", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbNames, err := parseDatabaseNames(tt.input) + + if tt.expectError { + require.Error(t, err) + if tt.errorSubstring != "" { + require.Contains(t, err.Error(), tt.errorSubstring) + } + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedDBs, dbNames) + } + }) + } +} + +// TestBackendTypeConstants tests the backend type constant values +func TestBackendTypeConstants(t *testing.T) { + require.Equal(t, BackendType("goleveldb"), GoLevelDB) + require.Equal(t, BackendType("leveldb"), LevelDB) + require.Equal(t, BackendType("rocksdb"), RocksDB) +} + +// TestDatabaseNameConstants tests the database name constant values +func TestDatabaseNameConstants(t *testing.T) { + require.Equal(t, DatabaseName("application"), Application) + require.Equal(t, DatabaseName("blockstore"), Blockstore) + require.Equal(t, DatabaseName("state"), State) + require.Equal(t, DatabaseName("tx_index"), TxIndex) + require.Equal(t, DatabaseName("evidence"), Evidence) +} + +// TestDBTypeConstants tests the db-type constant values +func TestDBTypeConstants(t *testing.T) { + require.Equal(t, DbType("app"), App) + require.Equal(t, DbType("cometbft"), CometBFT) + require.Equal(t, DbType("all"), All) +} + +// TestDBTypeMapping tests the mapping of db-type to database names +func TestDBTypeMapping(t *testing.T) { + tests := []struct { + name string + dbType DbType + expectedDBs []DatabaseName + expectError bool + errorSubstring string + }{ + { + name: "app type", + dbType: App, + expectedDBs: []DatabaseName{Application}, + expectError: false, + }, + { + name: "cometbft type", + dbType: CometBFT, + expectedDBs: []DatabaseName{Blockstore, State, TxIndex, Evidence}, + expectError: false, + }, + { + name: "all type", + dbType: All, + expectedDBs: []DatabaseName{Application, Blockstore, State, TxIndex, Evidence}, + expectError: false, + }, + { + name: "invalid type", + dbType: "invalid", + expectError: true, + errorSubstring: "invalid db-type", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dbNames, err := getDBNamesFromType(tt.dbType) + + if tt.expectError { + require.Error(t, err) + if tt.errorSubstring != "" { + require.Contains(t, err.Error(), tt.errorSubstring) + } + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedDBs, dbNames) + } + }) + } +} + +// TestDatabasesFlagPrecedence tests that --databases flag takes precedence over --db-type +func TestDatabasesFlagPrecedence(t *testing.T) { + tests := []struct { + name string + databasesFlag string + dbTypeFlag DbType + expectedDBs []string + useDatabases bool + }{ + { + name: "only db-type", + databasesFlag: "", + dbTypeFlag: App, + expectedDBs: []string{"application"}, + useDatabases: false, + }, + { + name: "only databases", + databasesFlag: "blockstore,tx_index", + dbTypeFlag: App, + expectedDBs: []string{"blockstore", "tx_index"}, + useDatabases: true, + }, + { + name: "both flags - databases takes precedence", + databasesFlag: "state", + dbTypeFlag: All, + expectedDBs: []string{"state"}, + useDatabases: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var dbNamesStr []string + var err error + + // Use the same logic as the command + if tt.databasesFlag != "" { + dbNamesStr, err = parseDatabaseNames(tt.databasesFlag) + require.NoError(t, err) + } else { + dbNames, err := getDBNamesFromType(tt.dbTypeFlag) + require.NoError(t, err) + // Convert []DatabaseName to []string + dbNamesStr = make([]string, len(dbNames)) + for i, name := range dbNames { + dbNamesStr[i] = string(name) + } + } + + require.Equal(t, tt.expectedDBs, dbNamesStr) + require.Equal(t, tt.useDatabases, tt.databasesFlag != "") + }) + } +} diff --git a/cmd/cronosd/cmd/patch_db.go b/cmd/cronosd/cmd/patch_db.go new file mode 100644 index 0000000000..33ef1c3d52 --- /dev/null +++ b/cmd/cronosd/cmd/patch_db.go @@ -0,0 +1,362 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + dbm "github.com/cosmos/cosmos-db" + "github.com/crypto-org-chain/cronos/cmd/cronosd/dbmigrate" + "github.com/spf13/cobra" + + "github.com/cosmos/cosmos-sdk/server" +) + +const ( + flagPatchSourceBackend = "source-backend" + flagPatchTargetBackend = "target-backend" + flagPatchSourceHome = "source-home" + flagPatchTargetPath = "target-path" + flagPatchDatabase = "database" + flagPatchHeight = "height" + flagPatchBatchSize = "batch-size" + flagPatchDryRun = "dry-run" +) + +// PatchDBCmd returns the patch command +func PatchDBCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "patch", + Short: "Patch specific block heights from source database into target database", + Long: `Patch specific block heights from a source database into an existing target database. + +This command is designed for: + - Adding missing blocks to an existing database + - Backfilling specific heights + - Patching gaps in block data + - Copying individual blocks between databases + +Unlike db migrate which creates a new database, db patch UPDATES an existing target database +by adding or overwriting keys for the specified heights. + +Supported databases: + - blockstore: Block data (H:, P:, C:, SC:, EC: prefixes for ABCI 2.0) + * Automatically patches BH: (block header by hash) keys by parsing block hashes from H: keys + - tx_index: Transaction indexing (tx.height/* namespace) + - Multiple: blockstore,tx_index (comma-separated for both) + +Features: + - Dry-run mode: Preview changes without modifying the database (--dry-run) + - Height filtering: Uses prefix-only iterators with Go-level numeric filtering + - Auto BH: patching: When patching blockstore H: keys, corresponding BH: keys are automatically patched + +Height specification (--height): + - Range: --height 10000-20000 (patch heights 10000 to 20000) + - Single: --height 123456 (patch only height 123456) + - Multiple: --height 123456,234567,999999 (patch specific heights) + +IMPORTANT: + - The target database MUST already exist + - Source database is opened in read-only mode + - Target database will be modified (keys added/updated) + - Always backup your target database before patching + - You MUST specify --target-path explicitly (required flag to prevent accidental modification) + - For blockstore, BH: (block header by hash) keys are automatically patched alongside H: keys + - Use --dry-run to preview changes without modifying the database + - Height filtering uses prefix-only iterators to handle string-encoded heights correctly + +Examples: + # Patch a single missing block + cronosd db patch \ + --database blockstore \ + --height 123456 \ + --source-home ~/.cronos-archive \ + --target-path ~/.cronos/data/blockstore.db \ + --source-backend rocksdb \ + --target-backend rocksdb + + # Patch a range of blocks + cronosd db patch \ + --database blockstore \ + --height 1000000-1001000 \ + --source-home ~/.cronos-backup \ + --target-path /mnt/data/cronos/blockstore.db \ + --source-backend goleveldb \ + --target-backend rocksdb + + # Patch multiple specific blocks + cronosd db patch \ + --database tx_index \ + --height 100000,200000,300000 \ + --source-home ~/.cronos-old \ + --target-path ~/.cronos/data/tx_index.db + + # Patch both blockstore and tx_index at once + cronosd db patch \ + --database blockstore,tx_index \ + --height 1000000-1001000 \ + --source-home ~/.cronos-backup \ + --target-path ~/.cronos/data \ + --source-backend goleveldb \ + --target-backend rocksdb + + # Patch from different backend + cronosd db patch \ + --database blockstore \ + --height 5000000-5001000 \ + --source-home /backup/cronos \ + --target-path /production/cronos/data/blockstore.db \ + --source-backend goleveldb \ + --target-backend rocksdb + + # Dry-run to preview changes (with short flags) + cronosd db patch \ + -d blockstore \ + -H 123456 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db \ + -n + + # Dry-run shows what would be patched including BH: keys + cronosd db patch \ + --database blockstore \ + --height 1000000 \ + --source-home ~/.cronos-backup \ + --target-path ~/.cronos/data/blockstore.db \ + --dry-run +`, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := server.GetServerContextFromCmd(cmd) + logger := ctx.Logger + + sourceBackend := ctx.Viper.GetString(flagPatchSourceBackend) + targetBackend := ctx.Viper.GetString(flagPatchTargetBackend) + sourceHome := ctx.Viper.GetString(flagPatchSourceHome) + targetPath := ctx.Viper.GetString(flagPatchTargetPath) + databases := ctx.Viper.GetString(flagPatchDatabase) + heightFlag := ctx.Viper.GetString(flagPatchHeight) + batchSize := ctx.Viper.GetInt(flagPatchBatchSize) + dryRun := ctx.Viper.GetBool(flagPatchDryRun) + + // Validate required flags + if sourceHome == "" { + return fmt.Errorf("--source-home is required") + } + if databases == "" { + return fmt.Errorf("--database is required (blockstore, tx_index, or both comma-separated)") + } + if heightFlag == "" { + return fmt.Errorf("--height is required (specify which heights to patch)") + } + if targetPath == "" { + return fmt.Errorf("--target-path is required: you must explicitly specify the target database path to prevent accidental modification of the source database") + } + + // Parse database names (comma-separated) + dbNames := strings.Split(databases, ",") + var validDBNames []string + for _, db := range dbNames { + db = strings.TrimSpace(db) + if db == "" { + continue + } + // Validate database + if db != "blockstore" && db != "tx_index" { + return fmt.Errorf("invalid database: %s (must be: blockstore or tx_index)", db) + } + validDBNames = append(validDBNames, db) + } + + if len(validDBNames) == 0 { + return fmt.Errorf("no valid databases specified") + } + + // Parse backend types + sourceBackendType, err := parseBackendType(BackendType(sourceBackend)) + if err != nil { + return fmt.Errorf("invalid source backend: %w", err) + } + + targetBackendType, err := parseBackendType(BackendType(targetBackend)) + if err != nil { + return fmt.Errorf("invalid target backend: %w", err) + } + + // Parse height specification + heightRange, err := dbmigrate.ParseHeightFlag(heightFlag) + if err != nil { + return fmt.Errorf("invalid height specification: %w", err) + } + + // Validate height range + if err := heightRange.Validate(); err != nil { + return fmt.Errorf("invalid height specification: %w", err) + } + + if heightRange.IsEmpty() { + return fmt.Errorf("height specification is required (cannot patch all heights)") + } + + logger.Info("Database patch configuration", + "databases", strings.Join(validDBNames, ", "), + "source_home", sourceHome, + "source_backend", sourceBackend, + "target_backend", targetBackend, + "height", heightRange.String(), + "batch_size", batchSize, + ) + + // Prepare RocksDB options if target is RocksDB + var rocksDBOpts interface{} + if targetBackendType == dbm.RocksDBBackend { + rocksDBOpts = dbmigrate.PrepareRocksDBOptions() + } + + // Track aggregate statistics + var totalKeysPatched int64 + var totalErrors int64 + var totalDuration time.Duration + + // Patch each database + for _, dbName := range validDBNames { + // Determine target path + var dbTargetPath string + // For single DB: targetPath is the full DB path (e.g., ~/.cronos/data/blockstore.db) + // For multiple DBs: targetPath is the data directory (e.g., ~/.cronos/data) + if len(validDBNames) == 1 { + dbTargetPath = targetPath + } else { + // For multiple databases, validate that targetPath is a directory, not a *.db file + cleanedTargetPath := filepath.Clean(targetPath) + if filepath.Ext(cleanedTargetPath) == dbmigrate.DbExtension { + return fmt.Errorf("when patching multiple databases, --target-path must be a data directory (e.g., ~/.cronos/data), not a *.db file path (got %q); remove the .db suffix", targetPath) + } + // Treat targetPath as data directory + dbTargetPath = filepath.Join(targetPath, dbName+dbmigrate.DbExtension) + } + + cleanTargetPath := filepath.Clean(dbTargetPath) + if filepath.Ext(cleanTargetPath) != dbmigrate.DbExtension { + return fmt.Errorf("--target-path must reference a *.db directory (got %q)", dbTargetPath) + } + + // Verify target database exists + if _, err := os.Stat(cleanTargetPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("target database does not exist: %s (the target database must already exist before patching; use the migrate command to create a new database)", cleanTargetPath) + } + return fmt.Errorf("failed to access target database: %w", err) + } + + logger.Info("Patching database", + "database", dbName, + "target_path", dbTargetPath, + ) + + // Perform the patch operation + opts := dbmigrate.PatchOptions{ + SourceHome: sourceHome, + TargetPath: dbTargetPath, + SourceBackend: sourceBackendType, + TargetBackend: targetBackendType, + BatchSize: batchSize, + Logger: logger, + RocksDBOptions: rocksDBOpts, + DBName: dbName, + HeightRange: heightRange, + ConflictStrategy: dbmigrate.ConflictAsk, // Ask user for each conflict + SkipConflictChecks: false, // Enable conflict checking + DryRun: dryRun, // Dry run mode + } + + stats, err := dbmigrate.PatchDatabase(opts) + if err != nil { + if stats != nil { + logger.Error("Patch failed", + "database", dbName, + "error", err, + "processed_keys", stats.ProcessedKeys.Load(), + "duration", stats.Duration(), + ) + } else { + logger.Error("Patch failed", + "database", dbName, + "error", err, + ) + } + return fmt.Errorf("failed to patch %s: %w", dbName, err) + } + + logger.Info("Database patch completed", + "database", dbName, + "total_keys", stats.TotalKeys.Load(), + "processed_keys", stats.ProcessedKeys.Load(), + "errors", stats.ErrorCount.Load(), + "duration", stats.Duration(), + ) + + // Accumulate statistics + totalKeysPatched += stats.ProcessedKeys.Load() + totalErrors += stats.ErrorCount.Load() + totalDuration += stats.Duration() + } + + // Print summary + logger.Info(strings.Repeat("=", 80)) + if dryRun { + logger.Info("DATABASE PATCH DRY RUN COMPLETED") + } else { + logger.Info("DATABASE PATCH COMPLETED SUCCESSFULLY") + } + logger.Info(strings.Repeat("=", 80)) + + logArgs := []interface{}{ + "databases", strings.Join(validDBNames, ", "), + "height", heightRange.String(), + "errors", totalErrors, + "duration", totalDuration.String(), + } + + if dryRun { + logArgs = append(logArgs, "mode", "DRY RUN (no changes made)") + logArgs = append(logArgs, "keys_found", totalKeysPatched) + logger.Info("Patch summary", logArgs...) + logger.Info("This was a dry run. No changes were made to the target database(s).") + } else { + logArgs = append(logArgs, "keys_patched", totalKeysPatched) + logger.Info("Patch summary", logArgs...) + logger.Info("The target database(s) have been updated with the specified heights.") + } + logger.Info(strings.Repeat("=", 80)) + + return nil + }, + } + + cmd.Flags().StringP(flagPatchSourceBackend, "s", string(GoLevelDB), "Source database backend type (goleveldb, rocksdb)") + cmd.Flags().StringP(flagPatchTargetBackend, "t", string(RocksDB), "Target database backend type (goleveldb, rocksdb)") + cmd.Flags().StringP(flagPatchSourceHome, "f", "", "Source home directory (required)") + cmd.Flags().StringP(flagPatchTargetPath, "p", "", "Target path: for single DB (e.g., ~/.cronos/data/blockstore.db), for multiple DBs (e.g., ~/.cronos/data) (required)") + cmd.Flags().StringP(flagPatchDatabase, "d", "", "Database(s) to patch: blockstore, tx_index, or both comma-separated (e.g., blockstore,tx_index) (required)") + cmd.Flags().StringP(flagPatchHeight, "H", "", "Height specification: range (10000-20000), single (123456), or multiple (123456,234567) (required)") + cmd.Flags().IntP(flagPatchBatchSize, "b", dbmigrate.DefaultBatchSize, "Number of key-value pairs to process in a batch") + cmd.Flags().BoolP(flagPatchDryRun, "n", false, "Dry run mode: simulate the operation without making any changes") + + // Mark required flags + _ = cmd.MarkFlagRequired(flagPatchSourceHome) + _ = cmd.MarkFlagRequired(flagPatchTargetPath) + _ = cmd.MarkFlagRequired(flagPatchDatabase) + _ = cmd.MarkFlagRequired(flagPatchHeight) + + return cmd +} + +// PatchCmd returns the patch subcommand (for database command group) +func PatchCmd() *cobra.Command { + cmd := PatchDBCmd() + cmd.Use = "patch" + cmd.Deprecated = "" + return cmd +} diff --git a/cmd/cronosd/cmd/patch_db_test.go b/cmd/cronosd/cmd/patch_db_test.go new file mode 100644 index 0000000000..1412586e8f --- /dev/null +++ b/cmd/cronosd/cmd/patch_db_test.go @@ -0,0 +1,157 @@ +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/crypto-org-chain/cronos/cmd/cronosd/dbmigrate" + "github.com/stretchr/testify/require" +) + +// TestTargetPathValidation tests that multi-database patching rejects *.db file paths +func TestTargetPathValidation(t *testing.T) { + tests := []struct { + name string + targetPath string + dbCount int + shouldError bool + errorString string + }{ + { + name: "single DB with .db extension - allowed", + targetPath: "/path/to/blockstore.db", + dbCount: 1, + shouldError: false, + }, + { + name: "single DB without .db extension - allowed (will be validated later)", + targetPath: "/path/to/blockstore", + dbCount: 1, + shouldError: false, + }, + { + name: "multiple DBs with data directory - allowed", + targetPath: "/path/to/data", + dbCount: 2, + shouldError: false, + }, + { + name: "multiple DBs with .db file path - rejected", + targetPath: "/path/to/blockstore.db", + dbCount: 2, + shouldError: true, + errorString: "must be a data directory", + }, + { + name: "multiple DBs with .db file path (trailing slash) - rejected", + targetPath: "/path/to/blockstore.db/", + dbCount: 2, + shouldError: true, + errorString: "must be a data directory", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Simulate the validation logic from patch_db.go + var dbNames []string + for i := 0; i < tt.dbCount; i++ { + dbNames = append(dbNames, "testdb") + } + + var err error + if len(dbNames) == 1 { + // Single DB: no validation in this branch + _ = tt.targetPath + } else { + // Multiple DBs: validate targetPath is not a *.db file + cleanedTargetPath := filepath.Clean(tt.targetPath) + if filepath.Ext(cleanedTargetPath) == dbmigrate.DbExtension { + err = &targetPathError{path: tt.targetPath} + } + } + + if tt.shouldError { + require.Error(t, err) + if tt.errorString != "" { + require.Contains(t, err.Error(), tt.errorString) + } + } else { + require.NoError(t, err) + } + }) + } +} + +// TestTargetPathExistence tests that patching fails if target database doesn't exist +func TestTargetPathExistence(t *testing.T) { + // Create a temporary directory for testing + tmpDir := t.TempDir() + + // Create an existing test database + existingDB := filepath.Join(tmpDir, "existing.db") + err := os.MkdirAll(existingDB, 0o755) + require.NoError(t, err) + + tests := []struct { + name string + targetPath string + shouldError bool + errorString string + }{ + { + name: "existing database - allowed", + targetPath: existingDB, + shouldError: false, + }, + { + name: "non-existing database - rejected", + targetPath: filepath.Join(tmpDir, "nonexistent.db"), + shouldError: true, + errorString: "target database does not exist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Simulate the existence check from patch_db.go + cleanTargetPath := filepath.Clean(tt.targetPath) + var err error + if _, statErr := os.Stat(cleanTargetPath); statErr != nil { + if os.IsNotExist(statErr) { + err = &targetExistenceError{path: cleanTargetPath} + } else { + err = statErr + } + } + + if tt.shouldError { + require.Error(t, err) + if tt.errorString != "" { + require.Contains(t, err.Error(), tt.errorString) + } + } else { + require.NoError(t, err) + } + }) + } +} + +// targetPathError is a helper type to simulate the error from patch_db.go +type targetPathError struct { + path string +} + +func (e *targetPathError) Error() string { + return "when patching multiple databases, --target-path must be a data directory (e.g., ~/.cronos/data), not a *.db file path (got \"" + e.path + "\"); remove the .db suffix" +} + +// targetExistenceError is a helper type to simulate the existence error from patch_db.go +type targetExistenceError struct { + path string +} + +func (e *targetExistenceError) Error() string { + return "target database does not exist: " + e.path + " (the target database must already exist before patching; use the migrate command to create a new database)" +} diff --git a/cmd/cronosd/cmd/root.go b/cmd/cronosd/cmd/root.go index 24f487a206..97c2180ed1 100644 --- a/cmd/cronosd/cmd/root.go +++ b/cmd/cronosd/cmd/root.go @@ -193,6 +193,11 @@ func initRootCmd( e2eecli.E2EECommand(), ) + databaseCmd := DatabaseCmd() + if databaseCmd != nil { + rootCmd.AddCommand(databaseCmd) + } + rootCmd, err := srvflags.AddGlobalFlags(rootCmd) if err != nil { panic(err) diff --git a/cmd/cronosd/dbmigrate/QUICKSTART.md b/cmd/cronosd/dbmigrate/QUICKSTART.md new file mode 100644 index 0000000000..5e32ac1bbd --- /dev/null +++ b/cmd/cronosd/dbmigrate/QUICKSTART.md @@ -0,0 +1,683 @@ +# Database Tools - Quick Start Guide + +This guide covers two commands under the `database` (or `db`) command group: +- **`database migrate`**: Full database migration between backends +- **`database patch`**: Patch specific block heights into existing databases + +> **Command Aliases**: You can use `cronosd database` or `cronosd db` interchangeably. + +--- + +## Part 1: database migrate (Full Migration) + +### Overview + +The `database migrate` command supports migrating: +- **Application database** (`application.db`) - Your chain state +- **CometBFT databases** (`blockstore.db`, `state.db`, `tx_index.db`, `evidence.db`) - Consensus data + +### Database Selection + +**Option 1: Use `--db-type` (or `-y`) flag** (migrate predefined groups): +- `app` (default): Application database only +- `cometbft`: CometBFT databases only +- `all`: Both application and CometBFT databases + +**Option 2: Use `--databases` (or `-d`) flag** (migrate specific databases): +- Comma-separated list of database names +- Valid names: `application`, `blockstore`, `state`, `tx_index`, `evidence` +- Example: `--databases blockstore,tx_index` or `-d blockstore,tx_index` +- Takes precedence over `--db-type` if both are specified + +## Prerequisites + +- Cronos node stopped +- Database backup created +- Sufficient disk space (at least 2x database size) +- For RocksDB: Build with `make build` or `go build -tags rocksdb` + +## Basic Migration Steps + +### 1. Stop Your Node + +```bash +# systemd +sudo systemctl stop cronosd + +# or manually +pkill cronosd +``` + +### 2. Backup Your Databases + +```bash +# Backup application database +BACKUP_NAME="application.db.backup-$(date +%Y%m%d-%H%M%S)" +cp -r ~/.cronos/data/application.db ~/.cronos/data/$BACKUP_NAME + +# If migrating CometBFT databases too +for db in blockstore state tx_index evidence; do + cp -r ~/.cronos/data/${db}.db ~/.cronos/data/${db}.db.backup-$(date +%Y%m%d-%H%M%S) +done + +# Verify backups +du -sh ~/.cronos/data/*.backup-* +``` + +### 3. Run Migration + +#### Application Database Only (Default) +```bash +cronosd database migrate \ + -s goleveldb \ + -t rocksdb \ + -y app \ + --home ~/.cronos +``` + +#### CometBFT Databases Only +```bash +cronosd database migrate \ + -s goleveldb \ + -t rocksdb \ + -y cometbft \ + --home ~/.cronos +``` + +#### All Databases (Recommended) +```bash +cronosd database migrate \ + -s goleveldb \ + -t rocksdb \ + -y all \ + --home ~/.cronos +``` + +#### RocksDB to LevelDB +```bash +cronosd database migrate \ + -s rocksdb \ + -t goleveldb \ + -y all \ + --home ~/.cronos +``` + +#### Specific Databases Only +```bash +# Migrate only blockstore and tx_index +cronosd database migrate \ + -s goleveldb \ + -t rocksdb \ + -d blockstore,tx_index \ + --home ~/.cronos + +# Migrate application and state databases +cronosd database migrate \ + -s goleveldb \ + -t rocksdb \ + -d application,state \ + --home ~/.cronos +``` + +### 4. Verify Migration Output + +#### Single Database Migration +Look for: +``` +================================================================================ +MIGRATION COMPLETED SUCCESSFULLY +================================================================================ +Total Keys: 1234567 +Processed Keys: 1234567 +Errors: 0 +Duration: 5m30s +``` + +#### Multiple Database Migration (db-type=all) +Look for: +``` +4:30PM INF Starting migration database=application +4:30PM INF Migration completed database=application processed_keys=21 total_keys=21 +4:30PM INF Starting migration database=blockstore +4:30PM INF Migration completed database=blockstore processed_keys=1523 total_keys=1523 +... + +================================================================================ +ALL MIGRATIONS COMPLETED SUCCESSFULLY +================================================================================ +Database Type: all +Total Keys: 3241 +Processed Keys: 3241 +Errors: 0 +``` + +### 5. Replace Original Databases + +#### Using the Swap Script (Recommended) + +The easiest way to replace databases is using the provided script: + +```bash +# Preview what will happen (dry run) +./cmd/cronosd/dbmigrate/swap-migrated-db.sh \ + --home ~/.cronos \ + --db-type all \ + --dry-run + +# Perform the actual swap +./cmd/cronosd/dbmigrate/swap-migrated-db.sh \ + --home ~/.cronos \ + --db-type all +``` + +The script will: +- ✅ Create timestamped backups (using fast `mv` operation) +- ✅ Replace originals with migrated databases +- ✅ Show summary with next steps +- ⚡ Faster than copying (no disk space duplication) + +**Script Options:** +```bash +--home DIR # Node home directory (default: ~/.cronos) +--db-type TYPE # Database type: app, cometbft, all (default: app) +--backup-suffix STR # Custom backup name (default: backup-YYYYMMDD-HHMMSS) +--dry-run # Preview without making changes +``` + +#### Manual Replacement (Alternative) + +##### Application Database Only +```bash +cd ~/.cronos/data + +# Keep old database as backup +mv application.db application.db.old + +# Use migrated database +mv application.db.migrate-temp application.db + +# Verify +ls -lh application.db +``` + +##### All Databases +```bash +cd ~/.cronos/data + +# Backup originals +mkdir -p backups +for db in application blockstore state tx_index evidence; do + if [ -d "${db}.db" ]; then + mv ${db}.db backups/${db}.db.old + fi +done + +# Replace with migrated databases +for db in application blockstore state tx_index evidence; do + if [ -d "${db}.db.migrate-temp" ]; then + mv ${db}.db.migrate-temp ${db}.db + fi +done + +# Verify +ls -lh *.db +``` + +### 6. Update Configuration + +#### Application Database +Edit `~/.cronos/config/app.toml`: + +```toml +# Change from: +app-db-backend = "goleveldb" + +# To: +app-db-backend = "rocksdb" +``` + +#### CometBFT Databases +Edit `~/.cronos/config/config.toml`: + +```toml +[consensus] +# Change from: +db_backend = "goleveldb" + +# To: +db_backend = "rocksdb" +``` + +### 7. Start Node + +```bash +# systemd +sudo systemctl start cronosd + +# or manually +cronosd start --home ~/.cronos +``` + +### 8. Verify Node Health + +```bash +# Check node is syncing +cronosd status + +# Check logs +tail -f ~/.cronos/logs/cronos.log + +# Or systemd logs +journalctl -u cronosd -f +``` + +## Quick Complete Workflow + +For the fastest migration experience: + +```bash +# 1. Stop node +systemctl stop cronosd + +# 2. Run migration +cronosd database migrate \ + -s goleveldb \ + -t rocksdb \ + -y all \ + --home ~/.cronos + +# 3. Swap databases (with automatic backup) +./cmd/cronosd/dbmigrate/swap-migrated-db.sh \ + --home ~/.cronos \ + --db-type all + +# 4. Update configs (edit app.toml and config.toml) + +# 5. Start node +systemctl start cronosd +``` + +## Common Options + +### Migrate Specific Database Type +```bash +# Application only +cronosd database migrate -y app ... + +# CometBFT only +cronosd database migrate -y cometbft ... + +# All databases +cronosd database migrate -y all ... +``` + +### Skip Verification (Faster) +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --verify=false \ + --home ~/.cronos +``` + +### Custom Batch Size +```bash +# Smaller batches for low memory +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --batch-size 1000 \ + --home ~/.cronos + +# Larger batches for high-end systems +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --batch-size 50000 \ + --home ~/.cronos +``` + +### Migrate to Different Location +```bash +# Useful for moving to faster disk +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --target-home /mnt/nvme/cronos \ + --home ~/.cronos +``` + +## Troubleshooting + +### Migration is Slow + +**Solution 1: Increase Batch Size** +```bash +cronosd database migrate --batch-size 50000 ... +``` + +**Solution 2: Disable Verification** +```bash +cronosd database migrate --verify=false ... +``` + +### Out of Disk Space + +**Check Space:** +```bash +df -h ~/.cronos/data +``` + +**Free Up Space:** +```bash +# Remove old snapshots +rm -rf ~/.cronos/data/snapshots/* + +# Remove old backups +rm -rf ~/.cronos/data/*.old +``` + +### Migration Failed + +**Check Logs:** +The migration tool outputs detailed progress. Look for: +- "Migration failed" error message +- Error counts > 0 +- Verification failures + +**Recovery:** +```bash +# Remove failed migration +rm -rf ~/.cronos/data/application.db.migrate-temp + +# Restore from backup if needed +cp -r ~/.cronos/data/application.db.backup-* ~/.cronos/data/application.db + +# Try again with different options +cronosd database migrate --batch-size 1000 --verify=false ... +``` + +### RocksDB Build Error + +**Error:** `fatal error: 'rocksdb/c.h' file not found` + +**Solution:** Build with RocksDB support: +```bash +# Install RocksDB dependencies (Ubuntu/Debian) +sudo apt-get install librocksdb-dev + +# Or build from project root +make build +# Or with explicit tags +go build -tags rocksdb -o ./cronosd ./cmd/cronosd +``` + +## Performance Tips + +### For Large Databases (> 100GB) + +1. **Use SSD/NVMe** if possible +2. **Increase batch size**: `--batch-size 50000` +3. **Skip verification initially**: `--verify=false` +4. **Run during low-traffic**: Minimize disk I/O competition +5. **Verify separately**: Check a few keys manually after migration + +### For Limited Memory Systems + +1. **Decrease batch size**: `--batch-size 1000` +2. **Close other applications**: Free up RAM +3. **Monitor memory**: `watch -n 1 free -h` + + +## Rollback + +If migration fails or node won't start: + +```bash +cd ~/.cronos/data + +# Remove new database +rm -rf application.db.migrate-temp application.db + +# Restore backup +cp -r application.db.backup-* application.db + +# Restore original app.toml settings +# Change app-db-backend back to original value + +# Start node +sudo systemctl start cronosd +``` + +## Estimated Migration Times + +### Single Database (Application) +Based on typical disk speeds: + +| Database Size | HDD (100MB/s) | SSD (500MB/s) | NVMe (3GB/s) | +|--------------|---------------|---------------|--------------| +| 10 GB | ~3 minutes | ~30 seconds | ~5 seconds | +| 50 GB | ~15 minutes | ~2.5 minutes | ~25 seconds | +| 100 GB | ~30 minutes | ~5 minutes | ~50 seconds | +| 500 GB | ~2.5 hours | ~25 minutes | ~4 minutes | + +*Note: Times include verification. Subtract approximately 50% time if verification is disabled with --verify=false.* + +### All Databases (app + cometbft) +Multiply by approximate factor based on your database sizes: +- **Application**: Usually largest (state data) +- **Blockstore**: Medium-large (block history) +- **State**: Small-medium (latest state) +- **TX Index**: Medium-large (transaction lookups) +- **Evidence**: Small (misbehavior evidence) + +**Example:** For a typical node with 100GB application.db and 50GB of CometBFT databases combined, expect ~40 minutes on SSD with verification. + + +## Part 2: database patch (Patch Specific Heights) + +### Overview + +The `database patch` command patches specific block heights from a source database into an **existing** target database. + +**Use cases**: +- Fix missing blocks +- Repair corrupted blocks +- Backfill specific heights +- Add blocks without full resync + +**Key differences from `database migrate`**: +- Target database MUST already exist +- Only patches specified heights (required) +- Only supports `blockstore` and `tx_index` +- Updates existing database (doesn't create new one) +- CometBFT uses **string-encoded heights** in keys (e.g., `C:38307809`) + +### Prerequisites + +- Both nodes stopped +- **Target database must exist** +- Backup of target database +- Source database with the blocks you need + +### Quick Start: Patch Missing Block + +#### 1. Stop Nodes + +```bash +# Stop both source and target nodes +sudo systemctl stop cronosd +``` + +#### 2. Backup Target Database + +```bash +# Always backup before patching! +BACKUP_NAME="blockstore.db.backup-$(date +%Y%m%d-%H%M%S)" +cp -r ~/.cronos/data/blockstore.db ~/.cronos/data/$BACKUP_NAME +``` + +#### 3. Patch the Block + +**Single block**: +```bash +cronosd database patch \ + -d blockstore \ + -H 123456 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db +``` + +**Range of blocks**: +```bash +cronosd database patch \ + -d blockstore \ + -H 1000000-1001000 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db +``` + +**Multiple specific blocks**: +```bash +cronosd database patch \ + -d blockstore \ + -H 100000,200000,300000 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db +``` + +**Both databases at once** (recommended): +```bash +cronosd database patch \ + -d blockstore,tx_index \ + -H 1000000-1001000 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data +``` + +**With debug logging** (to see detailed key/value information): +```bash +cronosd database patch \ + -d blockstore \ + -H 123456 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db \ + --log_level debug +``` + +> **Note**: Debug logs automatically format binary data (like txhashes) as hex strings (e.g., `0x1a2b3c...`) for readability, while text keys (like `tx.height/123/0`) are displayed as-is. + +**Dry run** (preview without making changes): +```bash +cronosd database patch \ + -d blockstore \ + -H 123456 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db \ + --dry-run +``` + +#### 4. Verify and Restart + +```bash +# Check the logs from database patch output +# Look for: "DATABASE PATCH COMPLETED SUCCESSFULLY" + +# Start node +sudo systemctl start cronosd + +# Verify node is working +cronosd status +``` + +### Common Patching Scenarios + +#### Scenario 1: Missing Blocks + +**Problem**: Node missing blocks 5000000-5000100 + +**Solution**: +```bash +cronosd database patch \ + -d blockstore \ + -H 5000000-5000100 \ + -f /mnt/archive-node \ + -p ~/.cronos/data/blockstore.db \ + -s rocksdb \ + -t rocksdb +``` + +#### Scenario 2: Corrupted Block + +**Problem**: Block 3000000 is corrupted + +**Solution**: +```bash +cronosd database patch \ + -d blockstore \ + -H 3000000 \ + -f /backup/cronos \ + -p ~/.cronos/data/blockstore.db +``` + +#### Scenario 3: Backfill Historical Data + +**Problem**: Pruned node needs specific checkpoint heights + +**Solution**: +```bash +cronosd database patch \ + -d blockstore \ + -H 1000000,2000000,3000000,4000000 \ + -f /archive/cronos \ + -p ~/.cronos/data/blockstore.db +``` + +#### Scenario 4: Patch Both Databases Efficiently + +**Problem**: Missing blocks in both blockstore and tx_index + +**Solution** (patch both at once): +```bash +cronosd database patch \ + -d blockstore,tx_index \ + -H 5000000-5000100 \ + -f /mnt/archive-node \ + -p ~/.cronos/data \ + -s rocksdb \ + -t rocksdb +``` + +> **Note**: When patching `tx_index` by height, the command uses a **three-pass approach**: +> 1. **Pass 1**: Patches `tx.height///` keys (with or without `$es$` suffix) and collects transaction metadata (height, tx_index) +> 2. **Pass 2**: Patches CometBFT `` lookup keys +> 3. **Pass 3**: For each transaction, uses a bounded iterator with range `[start, end)` where start is `ethereum_tx.ethereumTxHash////` and end is `start + 1` +> +> This ensures complete transaction index functionality, including support for `eth_getTransactionReceipt` with Ethereum txhashes. Pass 3 uses bounded iteration for optimal database range scans and copies existing event keys from source DB with their exact format (with or without `$es$` suffix). + +### Patch Flags Reference + +| Flag | Short | Required | Default | Description | +|------|-------|----------|---------|-------------| +| `--database` | `-d` | ✅ Yes | - | Database(s) to patch: `blockstore`, `tx_index`, or `blockstore,tx_index` | +| `--height` | `-H` | ✅ Yes | - | Heights: range (10-20), single (100), or multiple (10,20,30) | +| `--source-home` | `-f` | ✅ Yes | - | Source home directory | +| `--target-path` | `-p` | No | source data dir | For single DB: exact path. For multiple: data directory | +| `--source-backend` | `-s` | No | goleveldb | Source database backend | +| `--target-backend` | `-t` | No | rocksdb | Target database backend | +| `--batch-size` | `-b` | No | 10000 | Batch size for writing | + + +### When to Use Which Command + +| Situation | Use Command | Why | +|-----------|-------------|-----| +| Changing backend (goleveldb → rocksdb) | `database migrate` | Full migration | +| Missing a few blocks | `database patch` | Surgical fix | +| Corrupted block data | `database patch` | Replace specific blocks | +| Need entire database on new backend | `database migrate` | Complete migration | +| Backfilling specific heights | `database patch` | Efficient for specific blocks | +| Migrating application.db | `database migrate` | database patch doesn't support it | +| Target DB doesn't exist yet | `database migrate` | Creates new DB | +| Target DB exists, need specific heights | `database patch` | Updates existing | diff --git a/cmd/cronosd/dbmigrate/README.md b/cmd/cronosd/dbmigrate/README.md new file mode 100644 index 0000000000..a8d137c43c --- /dev/null +++ b/cmd/cronosd/dbmigrate/README.md @@ -0,0 +1,1263 @@ +# Database Migration Tools + +This package provides CLI tools for managing Cronos databases under the `database` (or `db`) command group: + +- **`database migrate`** (or `db migrate`): Full database migration between backends +- **`database patch`** (or `db patch`): Patch specific block heights into existing databases + +> **Alias**: You can use `cronosd database` or `cronosd db` interchangeably. +> **Short Flags**: All flags have short alternatives (e.g., `-s`, `-t`, `-d`, `-H`) + +## database migrate: Full Database Migration + +The `database migrate` command is used for migrating entire databases between different backend types (e.g., LevelDB to RocksDB). + +### Features + +- **Multiple Database Support**: Migrate application and/or CometBFT databases +- **Multiple Backend Support**: Migrate between LevelDB and RocksDB +- **Batch Processing**: Configurable batch size for optimal performance +- **Progress Tracking**: Real-time progress reporting with statistics +- **Data Verification**: Optional post-migration verification to ensure data integrity +- **Configurable RocksDB Options**: Use project-specific RocksDB configurations +- **Safe Migration**: Creates migrated databases in temporary locations to avoid data loss + +--- + +## database patch: Patch Specific Heights + +The `database patch` command is used for patching specific block heights from a source database into an existing target database. Unlike `database migrate`, it **updates an existing database** rather than creating a new one. + +### Key Differences + +| Feature | `database migrate` | `database patch` | +|---------|-------------------|------------------| +| **Purpose** | Full database migration | Patch specific heights | +| **Target** | Creates new database | Updates existing database | +| **Height Filter** | Not supported | Required | +| **Supported DBs** | All databases | blockstore, tx_index only | +| **Use Case** | Moving entire database | Adding/fixing specific blocks | +| **Key Format** | All backends | String-encoded heights (CometBFT) | + +### Use Cases + +- **Adding missing blocks** to an existing database +- **Backfilling specific heights** from an archive node +- **Fixing corrupted blocks** by patching from backup +- **Selective data recovery** without full resync + +### Quick Example + +```bash +# Patch a single missing block (with short flags) +cronosd database patch \ + -d blockstore \ + -H 123456 \ + -f ~/.cronos-archive \ + -p ~/.cronos/data/blockstore.db + +# Patch a range of blocks +cronosd db patch \ + -d blockstore \ + -H 1000000-2000000 \ + -f ~/backup/cronos \ + -p ~/.cronos/data/blockstore.db + +# Patch both blockstore and tx_index at once +cronosd db patch \ + -d blockstore,tx_index \ + -H 1000000-2000000 \ + -f ~/backup/cronos \ + -p ~/.cronos/data + +# Patch specific heights +cronosd database patch \ + --database tx_index \ + --height 100000,200000,300000 \ + --source-home ~/.cronos-old \ + --target-path ~/.cronos/data/tx_index.db +``` + +--- + +## Supported Databases + +### Application Database +- **application.db** - Chain state (accounts, contracts, balances, etc.) + +### CometBFT Databases +- **blockstore.db** - Block data (headers, commits, evidence) +- **state.db** - Latest state (validator sets, consensus params) +- **tx_index.db** - Transaction indexing for lookups +- **evidence.db** - Misbehavior evidence + +Use the `--db-type` flag to select which databases to migrate: +- `app` (default): Application database only +- `cometbft`: CometBFT databases only +- `all`: Both application and CometBFT databases + +## Usage + +### Basic Migration + +#### Migrate Application Database Only +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type app \ + --home ~/.cronos +``` + +#### Migrate CometBFT Databases Only +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type cometbft \ + --home ~/.cronos +``` + +#### Migrate All Databases +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --home ~/.cronos +``` + +### Migration with Verification + +Enable verification to ensure data integrity: + +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --verify \ + --home ~/.cronos +``` + +### Migration to Different Location + +Migrate to a different directory: + +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --target-home /mnt/new-storage \ + --home ~/.cronos +``` + +### Custom Batch Size + +Adjust batch size for performance tuning: + +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --batch-size 50000 \ + --home ~/.cronos +``` + + +## Command-Line Flags (migrate) + +| Flag | Description | Default | +|------|-------------|---------| +| `--source-backend` (`-s`) | Source database backend type (`goleveldb`, `rocksdb`) | goleveldb | +| `--target-backend` (`-t`) | Target database backend type (`goleveldb`, `rocksdb`) | rocksdb | +| `--db-type` (`-y`) | Database type to migrate (`app`, `cometbft`, `all`) | app | +| `--databases` (`-d`) | Comma-separated list of specific databases (e.g., `blockstore,tx_index`). Valid: `application`, `blockstore`, `state`, `tx_index`, `evidence`. Takes precedence over `--db-type` | (empty) | +| `--target-home` (`-o`) | Target home directory (if different from source) | Same as `--home` | +| `--batch-size` (`-b`) | Number of key-value pairs to process in each batch | 10000 | +| `--verify` (`-v`) | Verify migration by comparing source and target databases | true | +| `--home` | Node home directory | ~/.cronos | + +**Note:** The `migrate` command performs **full database migration** without height filtering. For selective height-based operations, use the `database patch` command instead. + +## Migration Process + +The migration tool follows these steps: + +1. **Opens Source Database** - Opens the source database in read-only mode +2. **Creates Target Database** - Creates a new database with `.migrate-temp` suffix +3. **Counts Keys** - Counts total keys for progress reporting +4. **Migrates Data** - Copies all key-value pairs in batches +5. **Verifies Data** (optional) - Compares source and target to ensure integrity +6. **Reports Statistics** - Displays migration statistics and next steps + +## Important Notes + +### Before Migration + +1. **Backup Your Data** - Always backup your database before migration +2. **Stop Your Node** - Ensure the node is not running during migration +3. **Check Disk Space** - Ensure sufficient disk space for the new database +4. **Verify Requirements** - For RocksDB migration, ensure RocksDB is compiled (build with `-tags rocksdb`) + +### After Migration + +The migrated databases are created with a temporary suffix to prevent accidental overwrites: + +```text +Application Database: + Original: ~/.cronos/data/application.db + Migrated: ~/.cronos/data/application.migrate-temp.db + +CometBFT Databases: + Original: ~/.cronos/data/blockstore.db + Migrated: ~/.cronos/data/blockstore.migrate-temp.db + (same pattern for state, tx_index, evidence) +``` + +**Manual Steps Required:** + +1. Verify the migration was successful +2. Replace the original databases with the migrated ones + + **Option A: Using the swap script (recommended):** + ```bash + # Preview changes + ./cmd/cronosd/dbmigrate/swap-migrated-db.sh \ + --home ~/.cronos \ + --db-type all \ + --dry-run + + # Perform swap with automatic backup + ./cmd/cronosd/dbmigrate/swap-migrated-db.sh \ + --home ~/.cronos \ + --db-type all + ``` + + **Option B: Manual replacement:** + ```bash + cd ~/.cronos/data + + # For application database + mv application.db application.db.backup + mv application.migrate-temp.db application.db + + # For CometBFT databases (if migrated) + for db in blockstore state tx_index evidence; do + if [ -d "${db}.migrate-temp.db" ]; then + mv ${db}.db ${db}.db.backup + mv ${db}.migrate-temp.db ${db}.db + fi + done + ``` + +3. Update configuration files: + - `app.toml`: Set `app-db-backend` to new backend type + - `config.toml`: Set `db_backend` to new backend type (if CometBFT databases were migrated) +4. Restart your node + +## Examples + +### Example 1: Basic LevelDB to RocksDB Migration + +```bash +# Stop the node +systemctl stop cronosd + +# Backup the database +cp -r ~/.cronos/data/application.db ~/.cronos/data/application.db.backup-$(date +%Y%m%d) + +# Run migration +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --verify \ + --home ~/.cronos + +# Replace the database +cd ~/.cronos/data +mv application.db application.db.old +mv application.migrate-temp.db application.db + +# Update app.toml +# Change: app-db-backend = "rocksdb" + +# Restart the node +systemctl start cronosd +``` + +### Example 2: Migrate All Databases (with Swap Script) + +For a complete migration of all node databases using the automated swap script: + +```bash +# Stop the node +systemctl stop cronosd + +# Run migration +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --verify \ + --home ~/.cronos + +# Use the swap script to replace databases (includes automatic backup) +./cmd/cronosd/dbmigrate/swap-migrated-db.sh \ + --home ~/.cronos \ + --db-type all + +# Update config files +# Edit app.toml: app-db-backend = "rocksdb" +# Edit config.toml: db_backend = "rocksdb" + +# Restart the node +systemctl start cronosd +``` + +### Example 2b: Migrate All Databases (Manual Method) + +For a complete migration with manual database replacement: + +```bash +# Stop the node +systemctl stop cronosd + +# Backup all databases +cd ~/.cronos/data +for db in application blockstore state tx_index evidence; do + if [ -d "${db}.db" ]; then + cp -r ${db}.db ${db}.db.backup-$(date +%Y%m%d) + fi +done + +# Run migration +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --verify \ + --home ~/.cronos + +# Replace the databases +cd ~/.cronos/data +mkdir -p backups +for db in application blockstore state tx_index evidence; do + if [ -d "${db}.db" ]; then + mv ${db}.db backups/ + mv ${db}.migrate-temp.db ${db}.db + fi +done + +# Update config files +# Edit app.toml: app-db-backend = "rocksdb" +# Edit config.toml: db_backend = "rocksdb" + +# Restart the node +systemctl start cronosd +``` + +### Example 3: Migration with Custom Batch Size + +For slower disks or limited memory, reduce batch size: + +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --batch-size 1000 \ + --verify \ + --home ~/.cronos +``` + +### Example 4: Migrate Specific Databases + +Migrate only the databases you need: + +```bash +# Migrate only transaction indexing and block storage +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --databases tx_index,blockstore \ + --verify \ + --home ~/.cronos + +# Manually replace the databases +cd ~/.cronos/data +mv tx_index.db tx_index.db.backup +mv tx_index.migrate-temp.db tx_index.db +mv blockstore.db blockstore.db.backup +mv blockstore.migrate-temp.db blockstore.db + +# Update config.toml: db_backend = "rocksdb" +``` + +### Example 5: Large Database Migration + +For very large databases, disable verification for faster migration: + +```bash +cronosd database migrate \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --db-type all \ + --batch-size 50000 \ + --verify=false \ + --home ~/.cronos +``` + +## Performance Considerations + +### Batch Size + +- **Small Batch (1000-5000)**: Better for limited memory, slower overall +- **Medium Batch (10000-20000)**: Balanced performance (default: 10000) +- **Large Batch (50000+)**: Faster migration, requires more memory + +### Verification + +- **Enabled**: Ensures data integrity but doubles migration time +- **Disabled**: Faster migration but no automatic verification +- **Recommendation**: Enable for production systems, disable for testing + +### Disk I/O + +- Migration speed is primarily limited by disk I/O +- SSDs provide significantly better performance than HDDs +- Consider migration during low-traffic periods + +## Troubleshooting + +### Migration Fails with "file not found" + +Ensure the source database exists and the path is correct: + +```bash +ls -la ~/.cronos/data/application.db +``` + +### RocksDB Build Error + +RocksDB requires native libraries. Build with RocksDB support: + +```bash +# From project root with RocksDB support +COSMOS_BUILD_OPTIONS=rocksdb make build + +# Or with specific tags +go build -tags rocksdb -o ./cronosd ./cmd/cronosd +``` + +Note: RocksDB requires native C++ libraries to be installed on your system. On macOS, install via `brew install rocksdb`. On Ubuntu/Debian, install via `apt-get install librocksdb-dev`. For other systems, see the [RocksDB installation guide](https://github.com/facebook/rocksdb/blob/main/INSTALL.md). + +### Verification Fails + +If verification fails, check: +1. Source database wasn't modified during migration +2. Sufficient disk space for target database +3. No I/O errors in logs + +### Out of Memory + +Reduce batch size: + +```bash +cronosd database migrate --batch-size 1000 ... +``` + +## Testing + +Run tests: + +```bash +# Unit tests (no RocksDB required) +go test -v ./cmd/cronosd/dbmigrate/... -short + +# All tests including RocksDB +go test -v -tags rocksdb ./cmd/cronosd/dbmigrate/... + +# Large database tests +go test -v ./cmd/cronosd/dbmigrate/... +``` + +## Architecture + +### Package Structure + +```text +cmd/cronosd/dbmigrate/ +├── migrate.go # Core migration logic +├── migrate_rocksdb.go # RocksDB-specific functions (with build tag) +├── migrate_no_rocksdb.go # RocksDB stubs (without build tag) +├── patch.go # Patching logic for specific heights +├── height_filter.go # Height-based filtering and iterators +├── migrate_basic_test.go # Tests without RocksDB +├── migrate_test.go # General migration tests +├── migrate_dbname_test.go # Database name-specific tests +├── migrate_rocksdb_test.go # RocksDB-specific tests (build tag) +├── patch_test.go # Patching tests +├── height_parse_test.go # Height parsing tests +├── height_filter_test.go # Height filtering tests +├── swap-migrated-db.sh # Script to swap databases after migration +├── README.md # Full documentation +└── QUICKSTART.md # Quick start guide +``` + +### Build Tags + +The package uses build tags to conditionally compile RocksDB support: + +- **Without RocksDB**: Basic functionality, LevelDB migrations +- **With RocksDB** (`-tags rocksdb`): Full RocksDB support + +## API + +### MigrateOptions + +```go +type MigrateOptions struct { + SourceHome string // Source home directory + TargetHome string // Target home directory + SourceBackend dbm.BackendType // Source database backend + TargetBackend dbm.BackendType // Target database backend + BatchSize int // Batch size for processing + Logger log.Logger // Logger for progress reporting + RocksDBOptions interface{} // RocksDB options (if applicable) + Verify bool // Enable post-migration verification + DBName string // Database name (application, blockstore, state, tx_index, evidence) +} +``` + +### MigrationStats + +```go +type MigrationStats struct { + TotalKeys atomic.Int64 // Total number of keys + ProcessedKeys atomic.Int64 // Number of keys processed + ErrorCount atomic.Int64 // Number of errors encountered + StartTime time.Time // Migration start time + EndTime time.Time // Migration end time +} +``` + +## Height Filtering Feature + +### Overview + +**IMPORTANT**: Height-based filtering is **ONLY supported** by the `database patch` command, not `database migrate`. + +- **`database migrate`**: Full database migration between backends (processes entire database, no filtering) +- **`database patch`**: Selective patching of specific heights to existing database (supports height filtering) + +The `database patch` command supports height-based filtering for `blockstore` and `tx_index` databases, allowing you to: + +- Patch only specific block heights to an existing database +- Efficiently process ranges without scanning entire database +- Handle single blocks or multiple specific heights + +### Height Specification Format + +The `--height` flag supports three formats: + +1. **Range**: `1000000-2000000` - Continuous range (inclusive) +2. **Single**: `123456` - One specific height +3. **Multiple**: `100000,200000,300000` - Comma-separated heights + +### Bounded Iterator Optimization + +Height filtering uses **bounded database iterators** for maximum efficiency: + +#### Traditional Approach (Inefficient) + +```text +Open iterator for entire database +For each key: + Extract height + If height in range: + Process key + Else: + Skip key +``` + +- Reads ALL keys from disk +- Filters at application level +- Slow for large databases with small ranges + +#### Bounded Iterator Approach (Efficient) + +```text +Calculate start_key for start_height +Calculate end_key for end_height +Open iterator with bounds [start_key, end_key) +For each key: + Process key (all keys are in range) +``` +- Only reads relevant keys from disk +- Database-level filtering +- Performance scales with range size, not total DB size + +### Performance Comparison + +Example: Patching heights 1M-1.1M from a 5M block database + +| Approach | Keys Read | Disk I/O | Time | +|----------|-----------|----------|------| +| **Full Scan + Filter** | 5,000,000 | All blocks | ~2 hours | +| **Bounded Iterator** | 100,000 | Only range | ~3 minutes | +| **Improvement** | **50x fewer** | **98% less** | **40x faster** | + +### CometBFT Key Formats + +#### Blockstore Keys + +**Cronos CometBFT uses STRING-ENCODED heights in blockstore keys:** + +```text +H: - Block metadata (height as string) +P:: - Block parts (height as string, part as number) +C: - Commit at height (height as string) +SC: - Seen commit (height as string) +EC: - Extended commit (height as string, ABCI 2.0) +BH: - Block header by hash (no height, migrated via H: keys) +BS:H - Block store height (metadata, no height encoding) +``` + +Example keys for height 38307809: + +```text +H:38307809 # Block metadata +P:38307809:0 # Block parts (part 0) +C:38307809 # Commit +SC:38307809 # Seen commit +EC:38307809 # Extended commit (ABCI 2.0, if present) +BH:0362b5c81d... # Block header by hash (auto-migrated with H: keys) +``` + +> **Important**: Unlike standard CometBFT, Cronos uses **ASCII string-encoded heights**, not binary encoding. + +#### TX Index Keys + +Transaction index has two types of keys: + +**1. Height-indexed keys:** + +```text +tx.height///$es$0 +tx.height/// (without $es$ suffix) +``` + +- **Key format**: Height (twice) and transaction index, optionally with event sequence suffix +- **Value**: The transaction hash (txhash) + +**2. Direct hash lookup keys (CometBFT):** + +```text + +``` + +- **Key format**: The CometBFT transaction hash itself +- **Value**: Transaction result data (protobuf-encoded) + +**3. Event-indexed keys (Ethereum):** + +```text +ethereum_tx.ethereumTxHash/0x//$es$ +ethereum_tx.ethereumTxHash/0x// (without $es$ suffix) +``` +- **Key format**: Event key + Ethereum txhash (hex, with 0x) + height + tx index, optionally with event sequence + - `ethereum_tx.ethereumTxHash`: Event attribute key + - `0x`: Ethereum txhash (hex, with 0x prefix) + - ``: Block height + - ``: Transaction index within block + - `$es$`: Event sequence separator and number (optional) +- **Value**: CometBFT transaction hash (allows lookup by Ethereum txhash) +- **Purpose**: Enables `eth_getTransactionReceipt` by Ethereum txhash + +**Important**: When patching by height, all three key types are automatically patched using a three-pass approach: + +#### Pass 1: Height-indexed keys + +- Iterator reads `tx.height///` keys within the height range (with or without `$es$` suffix) +- Patches these keys to target database +- Collects CometBFT txhashes from the values +- **Extracts Ethereum txhashes** from transaction result events + +#### Pass 2: CometBFT txhash lookup keys + +- For each collected CometBFT txhash, reads the `` key from source +- Patches the txhash keys to target database + +#### Pass 3: Ethereum event-indexed keys +- For each transaction from Pass 1, creates a bounded iterator with specific start/end keys +- Start: `ethereum_tx.ethereumTxHash/0x//` +- End: `start + 1` (exclusive upper bound) +- Iterates only through event keys for that specific transaction (matches keys with or without `$es$` suffix) +- Patches all matching event keys to target database +- **Critical for `eth_getTransactionReceipt` to work correctly** +- **Performance**: Uses bounded iteration for optimal database range scans + +This ensures all tx_index keys (including event-indexed keys) are properly patched. + +Example: + +```text +# Pass 1: Height-indexed key (from iterator) +tx.height/1000000/1000000/0$es$0 → value: + +# Pass 2: CometBFT direct lookup key (read individually) + → value: + +# Pass 3: Ethereum event-indexed key (searched from source DB) +ethereum_tx.ethereumTxHash/0xa1b2c3d4.../1000000/0$es$0 → value: +``` + +> **Note**: Pass 3 is only performed for transactions that contain `ethereum_tx` events. Non-EVM transactions (e.g., bank transfers, staking) will not have Ethereum txhashes. + +### Implementation Details + +#### Blockstore Bounded Iterators + +**CRITICAL**: CometBFT uses **string-encoded decimal heights** (e.g., "H:100", "H:20"), which **do NOT sort lexicographically by numeric value**: +- "H:20" > "H:150" (lexically) +- "H:9" > "H:10000" (lexically) + +**Solution**: We use **prefix-only iterators** with **Go-level numeric filtering** (Strategy B): + +```go +// H: prefix - create prefix-only iterator +start := []byte("H:") +end := []byte("I:") // Next prefix in ASCII +iterator1 := db.Iterator(start, end) + +// For each key from iterator: +// 1. Extract height numerically from key (e.g., parse "H:12345" -> 12345) +// 2. Check if height is within range using shouldIncludeKey() +// 3. Only process keys that pass the numeric filter + +// ... similar for P:, C:, SC:, and EC: prefixes +``` + +This strategy trades some iteration efficiency for correctness, scanning all keys with each prefix but filtering at the application level. + +> **Note**: Metadata keys like `BS:H` are NOT included when using height filtering (they don't have height encoding). + +**BH: Key Patching**: Block header by hash (`BH:`) keys don't contain height information. During **patching** (not full migration), when an `H:` key is patched, the block hash is extracted from its value and used to look up and patch the corresponding `BH:` key automatically. For full migrations, BH: keys are included in the complete database scan. + +#### TX Index Bounded Iterator + +**CRITICAL**: tx_index keys use format `tx.height/{height}/{hash}` where height is a **decimal string** (not zero-padded). Like blockstore, decimal strings **do NOT sort lexicographically by numeric value**: +- "tx.height/20/" > "tx.height/150/" (lexically) +- "tx.height/9/" > "tx.height/10000/" (lexically) + +**Solution**: We use a **prefix-only iterator** with **Go-level numeric filtering** (Strategy B): + +```go +// Create prefix-only iterator for tx.height namespace +start := []byte("tx.height/") +end := []byte("tx.height/~") // '~' is ASCII 126, after all digits +iterator := db.Iterator(start, end) + +// For each key from iterator: +// 1. Extract height numerically from key (e.g., parse "tx.height/12345/..." -> 12345) +// 2. Check if height is within range using shouldIncludeKey() +// 3. Only process keys that pass the numeric filter +``` + +#### Specific Heights Handling + +For specific heights (e.g., `100,200,300`): + +1. **Create encompassing range iterator**: From min(100) to max(300) +2. **Filter at application level**: Check if extracted height is in list +3. **Still efficient**: Only reads 100-300 range, not entire database + +```go +// Create iterator for overall range +minHeight := 100 +maxHeight := 300 +iterator := db.Iterator(makeKey(minHeight), makeKey(maxHeight+1)) + +// Filter to specific heights +for ; iterator.Valid(); iterator.Next() { + height := extractHeight(iterator.Key()) + if height == 100 || height == 200 || height == 300 { + process(iterator.Key(), iterator.Value()) + } +} +``` + +--- + +## database patch Command (Detailed Documentation) + +### Overview + +The `database patch` command patches specific block heights from a source database into an **existing** target database. + +**Key characteristics**: +- Target database MUST already exist +- Height specification is REQUIRED +- Only supports `blockstore` and `tx_index` +- Updates existing database (overwrites existing keys) + +### When to Use patch vs migrate + +| Scenario | Command | Reason | +|----------|---------|--------| +| **Changing database backend** | migrate | Creates new database with all data | +| **Missing a few blocks** | patch | Surgical fix, efficient for small ranges | +| **Corrupted block data** | patch | Replace specific bad blocks | +| **Entire database migration** | migrate | Handles all databases, includes verification | +| **Backfilling specific heights** | patch | Efficient for non-continuous heights | +| **Migrating application.db** | migrate | patch only supports blockstore/tx_index | +| **Target doesn't exist** | migrate | Creates new database | +| **Target exists, need additions** | patch | Updates existing database | + +## Command-Line Flags (patch) + +### Required Flags + +| Flag | Description | +|------|-------------| +| `--database` (`-d`) | Database name: `blockstore`, `tx_index`, or `blockstore,tx_index` | +| `--height` (`-H`) | Height specification: range (`1000-2000`), single (`123456`), or multiple (`100,200,300`) | +| `--source-home` (`-f`) | Source node home directory | + +### Optional Flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--target-path` (`-p`) | For single DB: exact path (e.g., `~/.cronos/data/blockstore.db`)
For multiple DBs: data directory (e.g., `~/.cronos/data`) | Source home data directory | +| `--source-backend` (`-s`) | Source database backend type (`goleveldb`, `rocksdb`) | goleveldb | +| `--target-backend` (`-t`) | Target database backend type (`goleveldb`, `rocksdb`) | rocksdb | +| `--batch-size` (`-b`) | Number of key-value pairs to process in each batch | 10000 | +| `--dry-run` | Simulate patching without making changes | false | +| `--log_level` | Log level (`info`, `debug`, etc.) | info | + +**Dry-Run Mode**: When using `--dry-run`, the patch command will: +- Simulate the entire patching process without writing any data +- Log all keys that would be patched (with `--log_level debug`) +- For blockstore patches, also discover and report BH: (block header by hash) keys that would be patched +- Report the total number of operations that would be performed + +#### Debug Logging + +When using `--log_level debug`, the patch command will log detailed information about each key-value pair being patched: + +```bash +# Enable debug logging to see detailed patching information +cronosd database patch \ + --database blockstore \ + --height 5000000 \ + --source-home ~/.cronos-archive \ + --target-path ~/.cronos/data/blockstore.db \ + --log_level debug +``` + +**Debug Output Includes**: +- **Key**: The full database key (up to 80 characters) + - Text keys: Displayed as-is (e.g., `tx.height/123/0`) + - Binary keys: Displayed as hex (e.g., `0x1a2b3c...` for txhashes) +- **Key Size**: Size in bytes of the key +- **Value Preview**: Preview of the value (up to 100 bytes) + - Text values: Displayed as-is + - Binary values: Displayed as hex (e.g., `0x0a8f01...`) +- **Value Size**: Total size in bytes of the value +- **Batch Information**: Current batch count and progress + +**Example Debug Output**: + +For blockstore keys (text): + +```text +DBG Patched key to target database key=C:5000000 key_size=9 value_preview=0x0a8f01... value_size=143 batch_count=1 +DBG Patched key to target database key=P:5000000:0 key_size=13 value_preview=0x0a4d0a... value_size=77 batch_count=2 +``` + +For tx_index keys: + +```text +# Pass 1: Height-indexed keys +DBG Patched tx.height key key=tx.height/5000000/5000000/0$es$0 +DBG Collected ethereum txhash eth_txhash=0xa1b2c3d4... height=5000000 tx_index=0 + +# Pass 2: CometBFT txhash keys (binary) +DBG Patched txhash key txhash=0x1a2b3c4d5e6f7890abcdef1234567890abcdef1234567890abcdef1234567890 + +# Pass 3: Ethereum event-indexed keys (searched from source DB) +DBG Found ethereum event key in source event_key=ethereum_tx.ethereumTxHash/0xa1b2c3d4.../5000000/0$es$0 +DBG Patched ethereum event key event_key=ethereum_tx.ethereumTxHash/0xa1b2c3d4.../5000000/0$es$0 +``` + +### Detailed Examples + +#### Example 1: Single Missing Block + +**Scenario**: Your node is missing block 5,000,000 due to a network issue. + +```bash +# 1. Stop the node +sudo systemctl stop cronosd + +# 2. Backup +cp -r ~/.cronos/data/blockstore.db ~/.cronos/data/blockstore.db.backup + +# 3. Patch the block +cronosd database patch \ + --database blockstore \ + --height 5000000 \ + --source-home /mnt/archive-node \ + --target-path ~/.cronos/data/blockstore.db \ + --source-backend rocksdb \ + --target-backend rocksdb + +# 4. Restart +sudo systemctl start cronosd +``` + +#### Example 2: Range of Missing Blocks + +**Scenario**: Network partition caused missing blocks 1,000,000 to 1,001,000. + +```bash +cronosd database patch \ + --database blockstore \ + --height 1000000-1001000 \ + --source-home ~/backup/cronos \ + --target-path ~/.cronos/data/blockstore.db +``` + +#### Example 3: Multiple Checkpoint Heights + +**Scenario**: Pruned node needs specific governance proposal heights. + +```bash +cronosd database patch \ + --database blockstore \ + --height 1000000,2000000,3000000,4000000,5000000 \ + --source-home /archive/cronos \ + --target-path ~/.cronos/data/blockstore.db +``` + +#### Example 4: Cross-Backend Patching + +**Scenario**: Patch from goleveldb backup to rocksdb production. + +```bash +cronosd database patch \ + --database blockstore \ + --height 4500000-4600000 \ + --source-home /backup/cronos-goleveldb \ + --target-path /production/cronos/data/blockstore.db \ + --source-backend goleveldb \ + --target-backend rocksdb \ + --batch-size 5000 +``` + +#### Example 5: TX Index Patching + +**Scenario**: Rebuild transaction index for specific heights. + +```bash +cronosd database patch \ + --database tx_index \ + --height 3000000-3100000 \ + --source-home ~/.cronos-archive \ + --target-path ~/.cronos/data/tx_index.db +``` + +#### Example 6: Patch Both Databases at Once + +**Scenario**: Missing blocks in both blockstore and tx_index (most efficient). + +```bash +cronosd database patch \ + --database blockstore,tx_index \ + --height 5000000-5000100 \ + --source-home ~/.cronos-archive \ + --target-path ~/.cronos/data \ + --source-backend rocksdb \ + --target-backend rocksdb +``` + +**Note**: When patching multiple databases, `--target-path` should be the data directory. The command will automatically append the database name (e.g., `blockstore.db`, `tx_index.db`). + +### Safety and Best Practices + +#### Always Backup First + +```bash +# Timestamp your backups +TIMESTAMP=$(date +%Y%m%d-%H%M%S) + +# Backup the target database +cp -r ~/.cronos/data/blockstore.db \ + ~/.cronos/data/blockstore.db.backup-$TIMESTAMP + +# Verify backup +du -sh ~/.cronos/data/blockstore.db* +``` + +#### Stop the Node + +Never patch while the node is running: + +```bash +# Stop the node +sudo systemctl stop cronosd + +# Verify it's stopped +ps aux | grep cronosd + +# Wait for graceful shutdown +sleep 5 +``` + +#### Verify Source Has the Data + +Before patching, verify the source has the heights you need: + +```bash +# For RocksDB +ldb --db=/source/blockstore.db scan --from=H: --max_keys=10 + +# For LevelDB +# Use leveldb tools or open the database programmatically +``` + +#### Monitor Progress + +The `database patch` command logs progress every 5 seconds: + +```text +INFO Patching progress processed=5000 total=10000 progress=50.00% errors=0 +INFO Patching progress processed=10000 total=10000 progress=100.00% errors=0 +INFO Database patch completed +``` + +#### Verify After Patching + +```bash +# Start the node +sudo systemctl start cronosd + +# Check node status +cronosd status + +# Verify block heights +cronosd query block + +# Check logs for errors +journalctl -u cronosd -f +``` + +### Error Handling + +#### Common Errors and Solutions + +#### 1. "target database does not exist" + +```text +Error: target database does not exist: /path/to/blockstore.db +``` + +**Solution**: Create the target database first or use `database migrate` to initialize it: + +```bash +# Option 1: Use db migrate to create empty database +cronosd database migrate --db-type cometbft --home ~/.cronos + +# Option 2: Copy from another node +cp -r /other-node/data/blockstore.db ~/.cronos/data/ +``` + +#### 2. "height range is required for patching" + +```text +Error: height range is required for patching +``` + +**Solution**: Always specify `--height` flag: + +```bash +cronosd database patch --height 123456 ... +``` + +#### 3. "database X does not support height-based patching" + +```text +Error: database application does not support height-based patching +``` + +**Solution**: Use `database migrate` for non-height-encoded databases: + +```bash +# For application, state, evidence databases +cronosd database migrate --db-type app ... +``` + +#### 4. "No keys found in source database for specified heights" + +```text +WARN No keys found in source database for specified heights +``` + +**Possible causes**: +- Source database doesn't have those heights (pruned) +- Wrong database name specified +- Incorrect source-home path + +**Solution**: Verify source database content and paths. + +#### 5. "Failed to open source database" + +```text +Error: failed to open source database:
+``` + +**Solutions**: +- Check source-home path is correct +- Verify database backend type matches +- Ensure database isn't corrupted +- Check file permissions + +### Performance Tuning + +#### Batch Size + +Adjust `--batch-size` based on your system: + +| System | Recommended Batch Size | Reasoning | +|--------|------------------------|-----------| +| **HDD** | 5,000 | Slower I/O, smaller batches | +| **SSD** | 10,000 (default) | Good balance | +| **NVMe** | 20,000 | Fast I/O, larger batches | +| **Low Memory** | 1,000 | Reduce memory usage | + +```bash +# For fast NVMe +cronosd database patch --batch-size 20000 ... + +# For slow HDD +cronosd database patch --batch-size 5000 ... +``` + + +### Advanced Usage + +#### Patching Multiple Databases + +##### Option 1: Patch both at once (recommended) + +```bash +# Patch both databases in a single command +cronosd database patch \ + --database blockstore,tx_index \ + --height 1000000-2000000 \ + --source-home ~/archive \ + --target-path ~/.cronos/data +``` + +**Benefits**: + +- Single command execution +- Consistent height range across databases +- Aggregated statistics +- Faster overall (no command overhead between runs) + +##### Option 2: Patch separately + +```bash +# Patch blockstore +cronosd database patch \ + --database blockstore \ + --height 1000000-2000000 \ + --source-home ~/archive \ + --target-path ~/.cronos/data/blockstore.db + +# Patch tx_index for same range +cronosd database patch \ + --database tx_index \ + --height 1000000-2000000 \ + --source-home ~/archive \ + --target-path ~/.cronos/data/tx_index.db +``` + + +### Implementation Architecture + +#### Core Components + +```text +cmd/cronosd/cmd/patch_db.go + └─> PatchDBCmd() # CLI command definition + └─> dbmigrate.PatchDatabase() # Core patching logic + +cmd/cronosd/dbmigrate/patch.go + ├─> PatchDatabase() # Main entry point + ├─> patchDataWithHeightFilter() # Router for database types + ├─> patchBlockstoreData() # Blockstore-specific patching + ├─> patchTxIndexData() # TX index-specific patching + └─> patchWithIterator() # Generic iterator processing + +cmd/cronosd/dbmigrate/height_filter.go + ├─> ParseHeightFlag() # Parse height specification + ├─> getBlockstoreIterators() # Get bounded iterators + ├─> getTxIndexIterator() # Get bounded iterator + └─> extractHeightFrom*Key() # Extract height from keys +``` + +#### Data Flow + +```text +1. Parse CLI flags +2. Validate inputs (target exists, height specified, etc.) +3. Open source database (read-only) +4. Open target database (read-write) +5. Count keys to patch (using bounded iterators) +6. For each bounded iterator: + a. Read key-value pairs + b. Filter if specific heights + c. Write to target in batches + d. Log progress +7. Flush if RocksDB +8. Close databases +9. Report statistics +``` + + +### Limitations + +#### 1. Application-Level Filtering for Specific Heights + +Specific heights use encompassing range iterator + application filter. + +**Impact**: Less efficient than continuous ranges, but still much better than full scan. + +#### 2. No Cross-Version Support + +Patching between different Cronos versions may fail if database formats differ. + +**Mitigation**: Use matching versions for source and target nodes. + +#### 3. No Rollback on Failure + +If patching fails midway, there's no automatic rollback. + +**Mitigation**: Always backup before patching. Can re-run db patch to complete. + +#### 4. Limited Database Support + +Only `blockstore` and `tx_index` supported. + +**Reason**: These are the only databases with height-encoded keys. Use `database migrate` for others. + + + + +## License + +This tool is part of the Cronos project and follows the same license. + diff --git a/cmd/cronosd/dbmigrate/height_filter.go b/cmd/cronosd/dbmigrate/height_filter.go new file mode 100644 index 0000000000..d8df731704 --- /dev/null +++ b/cmd/cronosd/dbmigrate/height_filter.go @@ -0,0 +1,476 @@ +package dbmigrate + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + dbm "github.com/cosmos/cosmos-db" +) + +// Database name constants +const ( + DBNameBlockstore = "blockstore" + DBNameTxIndex = "tx_index" +) + +// HeightRange represents block heights to migrate +// Can be a continuous range or specific heights +type HeightRange struct { + Start int64 // inclusive, 0 means from beginning (only used for ranges) + End int64 // inclusive, 0 means to end (only used for ranges) + SpecificHeights []int64 // specific heights to migrate (if set, Start/End are ignored) +} + +// IsWithinRange checks if a height is within the specified range or in specific heights +func (hr HeightRange) IsWithinRange(height int64) bool { + // If specific heights are set, check if height is in the list + if len(hr.SpecificHeights) > 0 { + for _, h := range hr.SpecificHeights { + if h == height { + return true + } + } + return false + } + + // Otherwise use range check + if hr.Start > 0 && height < hr.Start { + return false + } + if hr.End > 0 && height > hr.End { + return false + } + return true +} + +// IsEmpty returns true if no height range or specific heights are specified +func (hr HeightRange) IsEmpty() bool { + return hr.Start == 0 && hr.End == 0 && len(hr.SpecificHeights) == 0 +} + +// HasSpecificHeights returns true if specific heights are specified (not a range) +func (hr HeightRange) HasSpecificHeights() bool { + return len(hr.SpecificHeights) > 0 +} + +// String returns a human-readable representation of the height range +func (hr HeightRange) String() string { + if hr.IsEmpty() { + return "all heights" + } + + // Specific heights + if len(hr.SpecificHeights) > 0 { + if len(hr.SpecificHeights) == 1 { + return fmt.Sprintf("height %d", hr.SpecificHeights[0]) + } + if len(hr.SpecificHeights) <= 5 { + // Show all heights if 5 or fewer + heightStrs := make([]string, len(hr.SpecificHeights)) + for i, h := range hr.SpecificHeights { + heightStrs[i] = fmt.Sprintf("%d", h) + } + return fmt.Sprintf("heights %s", strings.Join(heightStrs, ", ")) + } + // Show count if more than 5 + return fmt.Sprintf("%d specific heights", len(hr.SpecificHeights)) + } + + // Range + if hr.Start > 0 && hr.End > 0 { + return fmt.Sprintf("heights %d to %d", hr.Start, hr.End) + } + if hr.Start > 0 { + return fmt.Sprintf("heights from %d", hr.Start) + } + if hr.End > 0 { + return fmt.Sprintf("heights up to %d", hr.End) + } + return "all heights" +} + +// Validate checks if the height range is valid +func (hr HeightRange) Validate() error { + // Validate specific heights + if len(hr.SpecificHeights) > 0 { + for _, h := range hr.SpecificHeights { + if h < 0 { + return fmt.Errorf("height cannot be negative: %d", h) + } + } + return nil + } + + // Validate range + if hr.Start < 0 { + return fmt.Errorf("start height cannot be negative: %d", hr.Start) + } + if hr.End < 0 { + return fmt.Errorf("end height cannot be negative: %d", hr.End) + } + if hr.End > 0 && hr.Start > hr.End { + return fmt.Errorf("start height (%d) cannot be greater than end height (%d)", hr.Start, hr.End) + } + return nil +} + +// ParseHeightFlag parses the --height flag value +// Supports: +// - Range: "10000-20000" +// - Single height: "123456" +// - Multiple heights: "123456,234567,999999" +func ParseHeightFlag(heightStr string) (HeightRange, error) { + if heightStr == "" { + return HeightRange{}, nil + } + + // Check if it's a range (contains '-') + if bytes.IndexByte([]byte(heightStr), '-') >= 0 { + return parseHeightRange(heightStr) + } + + // Check if it contains commas (multiple heights) + if bytes.IndexByte([]byte(heightStr), ',') >= 0 { + return parseSpecificHeights(heightStr) + } + + // Single height + height, err := parseInt64(heightStr) + if err != nil { + return HeightRange{}, fmt.Errorf("invalid height value: %w", err) + } + if height < 0 { + return HeightRange{}, fmt.Errorf("height cannot be negative: %d", height) + } + + return HeightRange{ + SpecificHeights: []int64{height}, + }, nil +} + +// parseHeightRange parses a range like "10000-20000" +func parseHeightRange(rangeStr string) (HeightRange, error) { + parts := splitString(rangeStr, '-') + if len(parts) != 2 { + return HeightRange{}, fmt.Errorf("invalid range format, expected 'start-end', got: %s", rangeStr) + } + + start, err := parseInt64(trimSpace(parts[0])) + if err != nil { + return HeightRange{}, fmt.Errorf("invalid start height: %w", err) + } + + end, err := parseInt64(trimSpace(parts[1])) + if err != nil { + return HeightRange{}, fmt.Errorf("invalid end height: %w", err) + } + + if start < 0 || end < 0 { + return HeightRange{}, fmt.Errorf("heights cannot be negative: %d-%d", start, end) + } + + if start > end { + return HeightRange{}, fmt.Errorf("start height (%d) cannot be greater than end height (%d)", start, end) + } + + return HeightRange{ + Start: start, + End: end, + }, nil +} + +// parseSpecificHeights parses comma-separated heights like "123456,234567,999999" +func parseSpecificHeights(heightsStr string) (HeightRange, error) { + parts := splitString(heightsStr, ',') + heights := make([]int64, 0, len(parts)) + + for _, part := range parts { + part = trimSpace(part) + if part == "" { + continue + } + + height, err := parseInt64(part) + if err != nil { + return HeightRange{}, fmt.Errorf("invalid height value '%s': %w", part, err) + } + + if height < 0 { + return HeightRange{}, fmt.Errorf("height cannot be negative: %d", height) + } + + heights = append(heights, height) + } + + if len(heights) == 0 { + return HeightRange{}, fmt.Errorf("no valid heights specified") + } + + return HeightRange{ + SpecificHeights: heights, + }, nil +} + +// Helper functions for parsing + +func parseInt64(s string) (int64, error) { + return strconv.ParseInt(strings.TrimSpace(s), 10, 64) +} + +func splitString(s string, sep byte) []string { + return strings.Split(s, string(sep)) +} + +func trimSpace(s string) string { + return strings.TrimSpace(s) +} + +// extractHeightFromBlockstoreKey extracts block height from CometBFT blockstore keys +// CometBFT blockstore key formats (string-encoded): +// - "H:" + height (as string) - block metadata +// - "P:" + height (as string) + ":" + part - block parts +// - "C:" + height (as string) - block commit +// - "SC:" + height (as string) - seen commit +// - "EC:" + height (as string) - extended commit (ABCI 2.0) +// - "BH:" + hash (as hex string) - block header by hash +// - "BS:H" - block store height (metadata) +func extractHeightFromBlockstoreKey(key []byte) (int64, bool) { + if len(key) < 3 { + return 0, false + } + + keyStr := string(key) + + // Check for different key prefixes + switch { + case bytes.HasPrefix(key, []byte("H:")): + // Block meta: "H:" + height (string) + heightStr := keyStr[2:] + var height int64 + _, err := fmt.Sscanf(heightStr, "%d", &height) + if err == nil { + return height, true + } + return 0, false + + case bytes.HasPrefix(key, []byte("P:")): + // Block parts: "P:" + height (string) + ":" + part + // Extract height between "P:" and next ":" + start := 2 + end := start + for end < len(keyStr) && keyStr[end] != ':' { + end++ + } + if end > start { + heightStr := keyStr[start:end] + var height int64 + _, err := fmt.Sscanf(heightStr, "%d", &height) + if err == nil { + return height, true + } + } + return 0, false + + case bytes.HasPrefix(key, []byte("C:")): + // Block commit: "C:" + height (string) + heightStr := keyStr[2:] + var height int64 + _, err := fmt.Sscanf(heightStr, "%d", &height) + if err == nil { + return height, true + } + return 0, false + + case bytes.HasPrefix(key, []byte("SC:")): + // Seen commit: "SC:" + height (string) + heightStr := keyStr[3:] + var height int64 + _, err := fmt.Sscanf(heightStr, "%d", &height) + if err == nil { + return height, true + } + return 0, false + + case bytes.HasPrefix(key, []byte("EC:")): + // Extended commit: "EC:" + height (string) - ABCI 2.0 + heightStr := keyStr[3:] + var height int64 + _, err := fmt.Sscanf(heightStr, "%d", &height) + if err == nil { + return height, true + } + return 0, false + + case bytes.HasPrefix(key, []byte("BH:")): + // Block header by hash - no height information + return 0, false + + default: + // Other keys (like "BS:H" for metadata) don't have height, include them + return 0, false + } +} + +// extractHeightFromTxIndexKey extracts height from transaction index keys +// CometBFT tx_index key formats: +// - "tx.height/" + height (as string) + "/" + hash - transaction by height +// - Other index keys may have height in different positions +func extractHeightFromTxIndexKey(key []byte) (int64, bool) { + keyStr := string(key) + + // Look for "tx.height/" prefix + if bytes.HasPrefix(key, []byte("tx.height/")) { + // Format: "tx.height/{height}/{hash}" + // Extract height which comes after "tx.height/" and before next "/" + start := len("tx.height/") + if len(keyStr) <= start { + return 0, false + } + + // Find the next "/" after the height + end := start + for end < len(keyStr) && keyStr[end] != '/' { + end++ + } + + if end > start { + heightStr := keyStr[start:end] + var height int64 + _, err := fmt.Sscanf(heightStr, "%d", &height) + if err == nil { + return height, true + } + } + } + + // For other tx_index keys, check if they contain height information + // Some keys might have height encoded differently + // For now, include all keys that don't match known patterns + return 0, false +} + +// shouldIncludeKey determines if a key should be included based on database type and height range +func shouldIncludeKey(key []byte, dbName string, heightRange HeightRange) bool { + // If no height range specified, include all keys + if heightRange.IsEmpty() { + return true + } + + var height int64 + var hasHeight bool + + switch dbName { + case DBNameBlockstore: + height, hasHeight = extractHeightFromBlockstoreKey(key) + case DBNameTxIndex: + height, hasHeight = extractHeightFromTxIndexKey(key) + default: + // For other databases, height filtering is not supported + return true + } + + // If key doesn't have height information, include it (likely metadata) + if !hasHeight { + return true + } + + // Check if height is within range + return heightRange.IsWithinRange(height) +} + +// getBlockstoreIterators creates prefix-only iterators for blockstore database +// Returns a slice of iterators, one for each key prefix (H:, P:, C:, SC:, EC:) +func getBlockstoreIterators(db dbm.DB, heightRange HeightRange) ([]dbm.Iterator, error) { + if heightRange.IsEmpty() { + // No height filtering, return full iterator + itr, err := db.Iterator(nil, nil) + if err != nil { + return nil, err + } + return []dbm.Iterator{itr}, nil + } + + var iterators []dbm.Iterator + prefixes := []string{"H:", "P:", "C:", "SC:", "EC:"} + + for _, prefix := range prefixes { + start := []byte(prefix) + end := []byte(prefix) + end[len(end)-1]++ + + itr, err := db.Iterator(start, end) + if err != nil { + for _, it := range iterators { + it.Close() + } + return nil, fmt.Errorf("failed to create iterator for prefix %s: %w", prefix, err) + } + iterators = append(iterators, itr) + } + + return iterators, nil +} + +// getTxIndexIterator creates a prefix-only iterator for tx_index database +func getTxIndexIterator(db dbm.DB, heightRange HeightRange) (dbm.Iterator, error) { + if heightRange.IsEmpty() { + // No height filtering, return full iterator + return db.Iterator(nil, nil) + } + + start := []byte("tx.height/") + end := []byte("tx.height/~") // '~' is ASCII 126, after all digits and '/' + + return db.Iterator(start, end) +} + +// extractBlockHashFromMetadata attempts to extract the block hash from H: (block metadata) key value +// The block hash is typically stored in the BlockMeta protobuf structure +// Returns the hash bytes and true if successful, nil and false otherwise +func extractBlockHashFromMetadata(value []byte) ([]byte, bool) { + // BlockMeta is a protobuf structure. The hash is typically near the beginning + // after the block_id field. We look for a field with tag 1 (BlockID) which contains + // the hash field (tag 1 within BlockID). + // + // Protobuf wire format for nested messages: + // - Field 1 (BlockID): tag=(1<<3)|2=0x0a, length-delimited + // - Inside BlockID, Field 1 (Hash): tag=(1<<3)|2=0x0a, length-delimited + // - Hash is typically 32 bytes for SHA256 + // + // This is a simplified extraction that looks for the pattern: + // 0x0a 0x0a + + if len(value) < 35 { // Minimum: 1+1+1+1+32 bytes + return nil, false + } + + // Look for the BlockID field (tag 0x0a) + for i := 0; i < len(value)-34; i++ { + if value[i] == 0x0a { // Field 1, wire type 2 (length-delimited) + blockIDLen := int(value[i+1]) + if i+2+blockIDLen > len(value) { + continue + } + + // Look for Hash field within BlockID (tag 0x0a) + if value[i+2] == 0x0a { + hashLen := int(value[i+3]) + // Typical hash lengths: 32 (SHA256), 20 (RIPEMD160) + if hashLen >= 20 && hashLen <= 64 && i+4+hashLen <= len(value) { + hash := make([]byte, hashLen) + copy(hash, value[i+4:i+4+hashLen]) + return hash, true + } + } + } + } + + return nil, false +} + +// supportsHeightFiltering returns true if the database supports height-based filtering +func supportsHeightFiltering(dbName string) bool { + return dbName == DBNameBlockstore || dbName == DBNameTxIndex +} diff --git a/cmd/cronosd/dbmigrate/height_filter_test.go b/cmd/cronosd/dbmigrate/height_filter_test.go new file mode 100644 index 0000000000..5fc514084f --- /dev/null +++ b/cmd/cronosd/dbmigrate/height_filter_test.go @@ -0,0 +1,534 @@ +package dbmigrate + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHeightRange_IsWithinRange(t *testing.T) { + tests := []struct { + name string + hr HeightRange + height int64 + want bool + }{ + { + name: "empty range includes all", + hr: HeightRange{Start: 0, End: 0}, + height: 1000, + want: true, + }, + { + name: "within range", + hr: HeightRange{Start: 100, End: 200}, + height: 150, + want: true, + }, + { + name: "at start boundary", + hr: HeightRange{Start: 100, End: 200}, + height: 100, + want: true, + }, + { + name: "at end boundary", + hr: HeightRange{Start: 100, End: 200}, + height: 200, + want: true, + }, + { + name: "below start", + hr: HeightRange{Start: 100, End: 200}, + height: 99, + want: false, + }, + { + name: "above end", + hr: HeightRange{Start: 100, End: 200}, + height: 201, + want: false, + }, + { + name: "only start specified - within", + hr: HeightRange{Start: 1000, End: 0}, + height: 2000, + want: true, + }, + { + name: "only start specified - below", + hr: HeightRange{Start: 1000, End: 0}, + height: 999, + want: false, + }, + { + name: "only end specified - within", + hr: HeightRange{Start: 0, End: 1000}, + height: 500, + want: true, + }, + { + name: "only end specified - above", + hr: HeightRange{Start: 0, End: 1000}, + height: 1001, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.IsWithinRange(tt.height) + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_IsEmpty(t *testing.T) { + tests := []struct { + name string + hr HeightRange + want bool + }{ + { + name: "empty range", + hr: HeightRange{Start: 0, End: 0}, + want: true, + }, + { + name: "only start specified", + hr: HeightRange{Start: 100, End: 0}, + want: false, + }, + { + name: "only end specified", + hr: HeightRange{Start: 0, End: 200}, + want: false, + }, + { + name: "both specified", + hr: HeightRange{Start: 100, End: 200}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.IsEmpty() + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_String(t *testing.T) { + tests := []struct { + name string + hr HeightRange + want string + }{ + { + name: "empty range", + hr: HeightRange{Start: 0, End: 0}, + want: "all heights", + }, + { + name: "both start and end", + hr: HeightRange{Start: 100, End: 200}, + want: "heights 100 to 200", + }, + { + name: "only start", + hr: HeightRange{Start: 1000, End: 0}, + want: "heights from 1000", + }, + { + name: "only end", + hr: HeightRange{Start: 0, End: 2000}, + want: "heights up to 2000", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.String() + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_Validate(t *testing.T) { + tests := []struct { + name string + hr HeightRange + wantErr bool + }{ + { + name: "valid range", + hr: HeightRange{Start: 100, End: 200}, + wantErr: false, + }, + { + name: "valid empty range", + hr: HeightRange{Start: 0, End: 0}, + wantErr: false, + }, + { + name: "valid only start", + hr: HeightRange{Start: 100, End: 0}, + wantErr: false, + }, + { + name: "valid only end", + hr: HeightRange{Start: 0, End: 200}, + wantErr: false, + }, + { + name: "negative start", + hr: HeightRange{Start: -1, End: 200}, + wantErr: true, + }, + { + name: "negative end", + hr: HeightRange{Start: 100, End: -1}, + wantErr: true, + }, + { + name: "start greater than end", + hr: HeightRange{Start: 200, End: 100}, + wantErr: true, + }, + { + name: "start equals end", + hr: HeightRange{Start: 100, End: 100}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.hr.Validate() + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestExtractHeightFromBlockstoreKey(t *testing.T) { + tests := []struct { + name string + key []byte + wantHeight int64 + wantOK bool + }{ + { + name: "block meta key H:", + key: makeBlockstoreKey("H:", 1000), + wantHeight: 1000, + wantOK: true, + }, + { + name: "block parts key P:", + key: makeBlockstoreKey("P:", 2000), + wantHeight: 2000, + wantOK: true, + }, + { + name: "block commit key C:", + key: makeBlockstoreKey("C:", 3000), + wantHeight: 3000, + wantOK: true, + }, + { + name: "seen commit key SC:", + key: makeSeenCommitKey(4000), + wantHeight: 4000, + wantOK: true, + }, + { + name: "extended commit key EC: (ABCI 2.0)", + key: []byte("EC:5000"), + wantHeight: 5000, + wantOK: true, + }, + { + name: "metadata key BS:H", + key: []byte("BS:H"), + wantHeight: 0, + wantOK: false, + }, + { + name: "too short key", + key: []byte("H:"), + wantHeight: 0, + wantOK: false, + }, + { + name: "unknown prefix", + key: []byte("XYZ:12345678"), + wantHeight: 0, + wantOK: false, + }, + { + name: "empty key", + key: []byte{}, + wantHeight: 0, + wantOK: false, + }, + { + name: "height 0", + key: makeBlockstoreKey("H:", 0), + wantHeight: 0, + wantOK: true, + }, + { + name: "large height", + key: makeBlockstoreKey("H:", 10000000), + wantHeight: 10000000, + wantOK: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHeight, gotOK := extractHeightFromBlockstoreKey(tt.key) + require.Equal(t, tt.wantOK, gotOK) + if gotOK { + require.Equal(t, tt.wantHeight, gotHeight) + } + }) + } +} + +func TestExtractHeightFromTxIndexKey(t *testing.T) { + tests := []struct { + name string + key []byte + wantHeight int64 + wantOK bool + }{ + { + name: "tx.height key", + key: []byte("tx.height/1000/hash123"), + wantHeight: 1000, + wantOK: true, + }, + { + name: "tx.height key with long height", + key: []byte("tx.height/9999999/abcdef"), + wantHeight: 9999999, + wantOK: true, + }, + { + name: "tx.height key height 0", + key: []byte("tx.height/0/hash"), + wantHeight: 0, + wantOK: true, + }, + { + name: "tx.height prefix only", + key: []byte("tx.height/"), + wantHeight: 0, + wantOK: false, + }, + { + name: "non-height key", + key: []byte("tx.hash/abcdef"), + wantHeight: 0, + wantOK: false, + }, + { + name: "empty key", + key: []byte{}, + wantHeight: 0, + wantOK: false, + }, + { + name: "malformed tx.height key", + key: []byte("tx.height/abc/hash"), + wantHeight: 0, + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHeight, gotOK := extractHeightFromTxIndexKey(tt.key) + require.Equal(t, tt.wantOK, gotOK) + if gotOK { + require.Equal(t, tt.wantHeight, gotHeight) + } + }) + } +} + +func TestShouldIncludeKey(t *testing.T) { + tests := []struct { + name string + key []byte + dbName string + heightRange HeightRange + want bool + }{ + { + name: "blockstore - within range", + key: makeBlockstoreKey("H:", 1500), + dbName: "blockstore", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: true, + }, + { + name: "blockstore - below range", + key: makeBlockstoreKey("H:", 500), + dbName: "blockstore", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: false, + }, + { + name: "blockstore - above range", + key: makeBlockstoreKey("H:", 2500), + dbName: "blockstore", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: false, + }, + { + name: "blockstore - metadata key always included", + key: []byte("BS:H"), + dbName: "blockstore", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: true, + }, + { + name: "blockstore - empty range includes all", + key: makeBlockstoreKey("H:", 500), + dbName: "blockstore", + heightRange: HeightRange{Start: 0, End: 0}, + want: true, + }, + { + name: "tx_index - within range", + key: []byte("tx.height/1500/hash"), + dbName: "tx_index", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: true, + }, + { + name: "tx_index - below range", + key: []byte("tx.height/500/hash"), + dbName: "tx_index", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: false, + }, + { + name: "tx_index - non-height key always included", + key: []byte("tx.hash/abcdef"), + dbName: "tx_index", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: true, + }, + { + name: "application db - ignores height range", + key: []byte("some_app_key"), + dbName: "application", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: true, + }, + { + name: "state db - ignores height range", + key: []byte("some_state_key"), + dbName: "state", + heightRange: HeightRange{Start: 1000, End: 2000}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := shouldIncludeKey(tt.key, tt.dbName, tt.heightRange) + require.Equal(t, tt.want, got) + }) + } +} + +// Helper functions for tests + +// makeBlockstoreKey creates a CometBFT blockstore key with the given prefix and height +func makeBlockstoreKey(prefix string, height int64) []byte { + // String-encoded format + if prefix == "P:" { + // Block parts: "P:" + height + ":" + part + return []byte(fmt.Sprintf("%s%d:0", prefix, height)) + } + // For other prefixes: prefix + height + return []byte(fmt.Sprintf("%s%d", prefix, height)) +} + +// makeSeenCommitKey creates a seen commit key with the given height +func makeSeenCommitKey(height int64) []byte { + // String-encoded format: "SC:" + height + return []byte(fmt.Sprintf("SC:%d", height)) +} + +func TestExtractBlockHashFromMetadata(t *testing.T) { + tests := []struct { + name string + value []byte + wantOK bool + wantLen int + }{ + { + name: "valid BlockMeta with hash", + // Minimal protobuf-like structure: 0x0a (BlockID field) + len + 0x0a (Hash field) + hashlen + hash + value: []byte{ + 0x0a, 0x22, // Field 1 (BlockID), length 34 + 0x0a, 0x20, // Field 1 (Hash), length 32 + // 32-byte hash + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + // Additional fields (ignored) + 0x12, 0x00, + }, + wantOK: true, + wantLen: 32, + }, + { + name: "too short value", + value: []byte{0x0a, 0x22, 0x0a, 0x20}, + wantOK: false, + wantLen: 0, + }, + { + name: "empty value", + value: []byte{}, + wantOK: false, + wantLen: 0, + }, + { + name: "value without BlockID field", + value: []byte{ + 0x12, 0x10, // Wrong field tag + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + }, + wantOK: false, + wantLen: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hash, ok := extractBlockHashFromMetadata(tt.value) + require.Equal(t, tt.wantOK, ok) + if ok { + require.Equal(t, tt.wantLen, len(hash)) + require.NotNil(t, hash) + } else { + require.Nil(t, hash) + } + }) + } +} diff --git a/cmd/cronosd/dbmigrate/height_parse_test.go b/cmd/cronosd/dbmigrate/height_parse_test.go new file mode 100644 index 0000000000..797646310d --- /dev/null +++ b/cmd/cronosd/dbmigrate/height_parse_test.go @@ -0,0 +1,312 @@ +package dbmigrate + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseHeightFlag(t *testing.T) { + tests := []struct { + name string + input string + want HeightRange + wantErr bool + errContains string + }{ + { + name: "empty string", + input: "", + want: HeightRange{}, + }, + { + name: "single height", + input: "123456", + want: HeightRange{SpecificHeights: []int64{123456}}, + }, + { + name: "range", + input: "10000-20000", + want: HeightRange{Start: 10000, End: 20000}, + }, + { + name: "range with spaces", + input: "10000 - 20000", + want: HeightRange{Start: 10000, End: 20000}, + }, + { + name: "multiple heights", + input: "123456,234567,999999", + want: HeightRange{SpecificHeights: []int64{123456, 234567, 999999}}, + }, + { + name: "multiple heights with spaces", + input: "123456, 234567, 999999", + want: HeightRange{SpecificHeights: []int64{123456, 234567, 999999}}, + }, + { + name: "two heights", + input: "100000,200000", + want: HeightRange{SpecificHeights: []int64{100000, 200000}}, + }, + { + name: "negative single height", + input: "-123", + wantErr: true, + // parsed as range with empty start, error is "invalid start height" + }, + { + name: "negative range start", + input: "-100-200", + wantErr: true, + // multiple dashes cause "invalid range format" + }, + { + name: "negative range end", + input: "100--200", + wantErr: true, + // multiple dashes cause "invalid range format" + }, + { + name: "invalid range - start > end", + input: "20000-10000", + wantErr: true, + errContains: "greater than", + }, + { + name: "invalid format", + input: "abc", + wantErr: true, + errContains: "invalid", + }, + { + name: "invalid range format - too many parts", + input: "10-20-30", + wantErr: true, + errContains: "invalid range format", + }, + { + name: "empty with commas", + input: ",,,", + wantErr: true, + errContains: "no valid heights", + }, + { + name: "mixed valid and empty heights", + input: "123456,,234567", + want: HeightRange{SpecificHeights: []int64{123456, 234567}}, + }, + { + name: "invalid height in list", + input: "123456,abc,234567", + wantErr: true, + errContains: "invalid", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseHeightFlag(tt.input) + if tt.wantErr { + require.Error(t, err) + if tt.errContains != "" { + require.Contains(t, err.Error(), tt.errContains) + } + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_IsWithinRange_SpecificHeights(t *testing.T) { + tests := []struct { + name string + hr HeightRange + height int64 + want bool + }{ + { + name: "single height - match", + hr: HeightRange{SpecificHeights: []int64{123456}}, + height: 123456, + want: true, + }, + { + name: "single height - no match", + hr: HeightRange{SpecificHeights: []int64{123456}}, + height: 123457, + want: false, + }, + { + name: "multiple heights - first match", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300}}, + height: 100, + want: true, + }, + { + name: "multiple heights - middle match", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300}}, + height: 200, + want: true, + }, + { + name: "multiple heights - last match", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300}}, + height: 300, + want: true, + }, + { + name: "multiple heights - no match", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300}}, + height: 150, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.IsWithinRange(tt.height) + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_String_SpecificHeights(t *testing.T) { + tests := []struct { + name string + hr HeightRange + want string + }{ + { + name: "single height", + hr: HeightRange{SpecificHeights: []int64{123456}}, + want: "height 123456", + }, + { + name: "two heights", + hr: HeightRange{SpecificHeights: []int64{100, 200}}, + want: "heights 100, 200", + }, + { + name: "five heights", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300, 400, 500}}, + want: "heights 100, 200, 300, 400, 500", + }, + { + name: "many heights (shows count)", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300, 400, 500, 600}}, + want: "6 specific heights", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.String() + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_HasSpecificHeights(t *testing.T) { + tests := []struct { + name string + hr HeightRange + want bool + }{ + { + name: "empty", + hr: HeightRange{}, + want: false, + }, + { + name: "range only", + hr: HeightRange{Start: 100, End: 200}, + want: false, + }, + { + name: "specific heights", + hr: HeightRange{SpecificHeights: []int64{100}}, + want: true, + }, + { + name: "both (specific takes precedence)", + hr: HeightRange{Start: 100, End: 200, SpecificHeights: []int64{150}}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.HasSpecificHeights() + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_IsEmpty_WithSpecificHeights(t *testing.T) { + tests := []struct { + name string + hr HeightRange + want bool + }{ + { + name: "completely empty", + hr: HeightRange{}, + want: true, + }, + { + name: "has specific heights", + hr: HeightRange{SpecificHeights: []int64{100}}, + want: false, + }, + { + name: "has range", + hr: HeightRange{Start: 100, End: 200}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.hr.IsEmpty() + require.Equal(t, tt.want, got) + }) + } +} + +func TestHeightRange_Validate_SpecificHeights(t *testing.T) { + tests := []struct { + name string + hr HeightRange + wantErr bool + }{ + { + name: "valid specific heights", + hr: HeightRange{SpecificHeights: []int64{100, 200, 300}}, + wantErr: false, + }, + { + name: "specific height with negative", + hr: HeightRange{SpecificHeights: []int64{100, -200, 300}}, + wantErr: true, + }, + { + name: "specific height zero (valid)", + hr: HeightRange{SpecificHeights: []int64{0, 100}}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.hr.Validate() + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/cmd/cronosd/dbmigrate/migrate.go b/cmd/cronosd/dbmigrate/migrate.go new file mode 100644 index 0000000000..95c32b3d49 --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate.go @@ -0,0 +1,500 @@ +package dbmigrate + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "time" + + dbm "github.com/cosmos/cosmos-db" + + "cosmossdk.io/log" +) + +const ( + // DefaultBatchSize is the number of key-value pairs to process in a single batch + DefaultBatchSize = 10000 + // DefaultWorkers is the number of concurrent workers for migration + DefaultWorkers = 4 +) + +// MigrateOptions holds the configuration for database migration +type MigrateOptions struct { + // SourceHome is the home directory containing the source database + SourceHome string + // TargetHome is the home directory for the target database (if empty, uses SourceHome) + TargetHome string + // SourceBackend is the source database backend type + SourceBackend dbm.BackendType + // TargetBackend is the target database backend type + TargetBackend dbm.BackendType + // BatchSize is the number of key-value pairs to process in a single batch + BatchSize int + // Logger for progress reporting + Logger log.Logger + // RocksDBOptions for creating RocksDB (only used when target is RocksDB) + // This is interface{} to avoid importing grocksdb when rocksdb tag is not used + RocksDBOptions interface{} + // Verify enables post-migration verification + Verify bool + // DBName is the name of the database to migrate (e.g., "application", "blockstore", "state") + DBName string +} + +// MigrationStats tracks migration progress and statistics +type MigrationStats struct { + TotalKeys atomic.Int64 + ProcessedKeys atomic.Int64 + ErrorCount atomic.Int64 + StartTime time.Time + EndTime time.Time +} + +// Progress returns the current progress as a percentage +func (s *MigrationStats) Progress() float64 { + total := s.TotalKeys.Load() + if total == 0 { + return 0 + } + return float64(s.ProcessedKeys.Load()) / float64(total) * 100 +} + +// Duration returns the time elapsed since start +func (s *MigrationStats) Duration() time.Duration { + if s.EndTime.IsZero() { + return time.Since(s.StartTime) + } + return s.EndTime.Sub(s.StartTime) +} + +// Migrate performs database migration from source backend to target backend +func Migrate(opts MigrateOptions) (*MigrationStats, error) { + if opts.BatchSize <= 0 { + opts.BatchSize = DefaultBatchSize + } + if opts.TargetHome == "" { + opts.TargetHome = opts.SourceHome + } + if opts.Logger == nil { + opts.Logger = log.NewNopLogger() + } + + stats := &MigrationStats{ + StartTime: time.Now(), + } + + // Default to "application" if DBName is not specified + if opts.DBName == "" { + opts.DBName = "application" + } + + opts.Logger.Info("Starting database migration", + "database", opts.DBName, + "source_backend", opts.SourceBackend, + "target_backend", opts.TargetBackend, + "source_home", opts.SourceHome, + "target_home", opts.TargetHome, + ) + + // Open source database in read-only mode + sourceDataDir := filepath.Join(opts.SourceHome, "data") + sourceDB, err := dbm.NewDB(opts.DBName, opts.SourceBackend, sourceDataDir) + if err != nil { + return stats, fmt.Errorf("failed to open source database: %w", err) + } + sourceDBClosed := false + defer func() { + if !sourceDBClosed { + sourceDB.Close() + } + }() + + // Create target database + targetDataDir := filepath.Join(opts.TargetHome, "data") + + // For migration, we need to ensure we don't accidentally overwrite an existing DB + // Unified path format for all backends: .migrate-temp.db + tempTargetDir := filepath.Join(targetDataDir, opts.DBName+".migrate-temp.db") + finalTargetDir := filepath.Join(targetDataDir, opts.DBName+DbExtension) + + var targetDB dbm.DB + if opts.TargetBackend == dbm.RocksDBBackend { + // RocksDB: we specify the exact directory path + // RocksDB needs the parent directory to exist + if err := os.MkdirAll(targetDataDir, 0o755); err != nil { + return stats, fmt.Errorf("failed to create target data directory: %w", err) + } + targetDB, err = openRocksDBForMigration(tempTargetDir, opts.RocksDBOptions) + } else { + // LevelDB/others: dbm.NewDB appends .db to the name, so we pass the name without .db + targetDB, err = dbm.NewDB(opts.DBName+".migrate-temp", opts.TargetBackend, targetDataDir) + } + if err != nil { + return stats, fmt.Errorf("failed to create target database: %w", err) + } + targetDBClosed := false + defer func() { + if !targetDBClosed && targetDB != nil { + targetDB.Close() + } + }() + + // Count total keys first for progress reporting + opts.Logger.Info("Counting total keys...") + totalKeys, err := countKeys(sourceDB) + if err != nil { + return stats, fmt.Errorf("failed to count keys: %w", err) + } + opts.Logger.Info("Total keys to migrate", "count", totalKeys) + + stats.TotalKeys.Store(totalKeys) + + // Perform the full database migration + if err := migrateData(sourceDB, targetDB, opts, stats); err != nil { + return stats, fmt.Errorf("migration failed: %w", err) + } + + // Flush memtable to SST files for RocksDB + if opts.TargetBackend == dbm.RocksDBBackend { + opts.Logger.Info("Flushing RocksDB memtable to SST files...") + if err := flushRocksDB(targetDB); err != nil { + return stats, fmt.Errorf("failed to flush RocksDB: %w", err) + } + opts.Logger.Info("Flush completed") + } + + // Close databases before verification to release locks + // This prevents "resource temporarily unavailable" errors + if err := targetDB.Close(); err != nil { + opts.Logger.Error("Warning: failed to close target database", "error", err) + } + targetDBClosed = true + + if err := sourceDB.Close(); err != nil { + opts.Logger.Error("Warning: failed to close source database", "error", err) + } + sourceDBClosed = true + + stats.EndTime = time.Now() + opts.Logger.Info("Migration completed", + "total_keys", stats.TotalKeys.Load(), + "processed_keys", stats.ProcessedKeys.Load(), + "errors", stats.ErrorCount.Load(), + "duration", stats.Duration(), + ) + + // Verification step if requested + if opts.Verify { + opts.Logger.Info("Starting verification...") + if err := verifyMigration(sourceDataDir, tempTargetDir, opts); err != nil { + return stats, fmt.Errorf("verification failed: %w", err) + } + opts.Logger.Info("Verification completed successfully") + } + + opts.Logger.Info("Migration process completed", + "temp_location", tempTargetDir, + "target_location", finalTargetDir, + "note", "Please backup your source database and manually rename the temp directory to replace the original", + ) + + return stats, nil +} + +// countKeys counts the total number of keys in the database +func countKeys(db dbm.DB) (int64, error) { + itr, err := db.Iterator(nil, nil) + if err != nil { + return 0, err + } + defer itr.Close() + + var count int64 + for ; itr.Valid(); itr.Next() { + count++ + } + return count, itr.Error() +} + +// migrateData performs the actual data migration +func migrateData(sourceDB, targetDB dbm.DB, opts MigrateOptions, stats *MigrationStats) error { + itr, err := sourceDB.Iterator(nil, nil) + if err != nil { + return err + } + defer itr.Close() + + return migrateWithIterator(itr, sourceDB, targetDB, opts, stats) +} + +// migrateWithIterator migrates data from a single iterator +func migrateWithIterator(itr dbm.Iterator, sourceDB, targetDB dbm.DB, opts MigrateOptions, stats *MigrationStats) error { + batch := targetDB.NewBatch() + defer batch.Close() + + batchCount := 0 + lastProgressReport := time.Now() + + for ; itr.Valid(); itr.Next() { + key := itr.Key() + value := itr.Value() + + // Make copies since the iterator might reuse the slices + keyCopy := make([]byte, len(key)) + valueCopy := make([]byte, len(value)) + copy(keyCopy, key) + copy(valueCopy, value) + + if err := batch.Set(keyCopy, valueCopy); err != nil { + opts.Logger.Error("Failed to add key to batch", "error", err) + stats.ErrorCount.Add(1) + continue + } + + batchCount++ + stats.ProcessedKeys.Add(1) + + // Write batch when it reaches the configured size + if batchCount >= opts.BatchSize { + if err := batch.Write(); err != nil { + return fmt.Errorf("failed to write batch: %w", err) + } + batch.Close() + batch = targetDB.NewBatch() + batchCount = 0 + } + + // Report progress every second + if time.Since(lastProgressReport) >= time.Second { + opts.Logger.Info("Migration progress", + "progress", fmt.Sprintf("%.2f%%", stats.Progress()), + "processed", stats.ProcessedKeys.Load(), + "total", stats.TotalKeys.Load(), + "errors", stats.ErrorCount.Load(), + ) + lastProgressReport = time.Now() + } + } + + // Write any remaining items in the batch + if batchCount > 0 { + if err := batch.Write(); err != nil { + return fmt.Errorf("failed to write final batch: %w", err) + } + } + + return itr.Error() +} + +// openDBWithRetry attempts to open a database with exponential backoff retry logic. +// This handles OS-level file lock delays that can occur after database closure. +func openDBWithRetry(dbName string, backend dbm.BackendType, dir string, maxRetries int, initialDelay time.Duration, logger log.Logger) (dbm.DB, error) { + var db dbm.DB + var err error + delay := initialDelay + + for attempt := 0; attempt < maxRetries; attempt++ { + db, err = dbm.NewDB(dbName, backend, dir) + if err == nil { + return db, nil + } + + if attempt < maxRetries-1 { + logger.Info("Failed to open database, retrying...", + "attempt", attempt+1, + "max_retries", maxRetries, + "delay", delay, + "error", err, + ) + time.Sleep(delay) + delay *= 2 // Exponential backoff + } + } + + return nil, fmt.Errorf("failed to open database after %d attempts: %w", maxRetries, err) +} + +// openRocksDBWithRetry attempts to open a RocksDB database with exponential backoff retry logic. +func openRocksDBWithRetry(dir string, maxRetries int, initialDelay time.Duration, logger log.Logger) (dbm.DB, error) { + var db dbm.DB + var err error + delay := initialDelay + + for attempt := 0; attempt < maxRetries; attempt++ { + db, err = openRocksDBForRead(dir) + if err == nil { + return db, nil + } + + if attempt < maxRetries-1 { + logger.Info("Failed to open RocksDB, retrying...", + "attempt", attempt+1, + "max_retries", maxRetries, + "delay", delay, + "error", err, + ) + time.Sleep(delay) + delay *= 2 // Exponential backoff + } + } + + return nil, fmt.Errorf("failed to open RocksDB after %d attempts: %w", maxRetries, err) +} + +// verifyMigration compares source and target databases to ensure data integrity +func verifyMigration(sourceDir, targetDir string, opts MigrateOptions) error { + // Determine database name from the directory path + // Extract the database name from sourceDir (e.g., "blockstore" from "/path/to/blockstore.db") + dbName := opts.DBName + if dbName == "" { + dbName = "application" + } + + // Reopen databases for verification with retry logic to handle OS-level file lock delays + // that can occur after database closure. Use exponential backoff: 50ms, 100ms, 200ms, 400ms, 800ms + const maxRetries = 5 + const initialDelay = 50 * time.Millisecond + + sourceDB, err := openDBWithRetry(dbName, opts.SourceBackend, sourceDir, maxRetries, initialDelay, opts.Logger) + if err != nil { + return fmt.Errorf("failed to open source database for verification: %w", err) + } + defer sourceDB.Close() + + var targetDB dbm.DB + if opts.TargetBackend == dbm.RocksDBBackend { + targetDB, err = openRocksDBWithRetry(targetDir, maxRetries, initialDelay, opts.Logger) + } else { + targetDB, err = openDBWithRetry(dbName+".migrate-temp", opts.TargetBackend, filepath.Dir(targetDir), maxRetries, initialDelay, opts.Logger) + } + if err != nil { + return fmt.Errorf("failed to open target database for verification: %w", err) + } + defer targetDB.Close() + + var verifiedKeys int64 + var mismatchCount int64 + lastProgressReport := time.Now() + + // Phase 1: Verify all keys in source exist in target and match + sourceItr, err := sourceDB.Iterator(nil, nil) + if err != nil { + return err + } + defer sourceItr.Close() + + for ; sourceItr.Valid(); sourceItr.Next() { + key := sourceItr.Key() + sourceValue := sourceItr.Value() + + targetValue, err := targetDB.Get(key) + if err != nil { + opts.Logger.Error("Failed to get key from target database", "key", fmt.Sprintf("%x", key), "error", err) + mismatchCount++ + continue + } + + if targetValue == nil { + opts.Logger.Error("Key missing in target database", "key", fmt.Sprintf("%x", key)) + mismatchCount++ + continue + } + + // Use bytes.Equal for efficient comparison + if !bytes.Equal(sourceValue, targetValue) { + opts.Logger.Error("Value mismatch", + "key", fmt.Sprintf("%x", key), + "source_len", len(sourceValue), + "target_len", len(targetValue), + ) + mismatchCount++ + } + + verifiedKeys++ + + // Report progress every second + if time.Since(lastProgressReport) >= time.Second { + opts.Logger.Info("Verification progress", + "verified", verifiedKeys, + "mismatches", mismatchCount, + ) + lastProgressReport = time.Now() + } + } + + if err := sourceItr.Error(); err != nil { + return err + } + + // Phase 2: Verify target doesn't have extra keys (iterate target, check against source) + opts.Logger.Info("Starting second verification phase (checking for extra keys in target)...") + targetItr, err := targetDB.Iterator(nil, nil) + if err != nil { + return fmt.Errorf("failed to create target iterator: %w", err) + } + defer targetItr.Close() + + var targetKeys int64 + lastProgressReport = time.Now() + + for ; targetItr.Valid(); targetItr.Next() { + key := targetItr.Key() + targetKeys++ + + // Check if this key exists in source + sourceValue, err := sourceDB.Get(key) + if err != nil { + opts.Logger.Error("Failed to get key from source database during reverse verification", + "key", fmt.Sprintf("%x", key), + "error", err, + ) + mismatchCount++ + continue + } + + // If key doesn't exist in source (Get returns nil for non-existent keys) + if sourceValue == nil { + opts.Logger.Error("Extra key found in target that doesn't exist in source", + "key", fmt.Sprintf("%x", key), + ) + mismatchCount++ + } + + // Report progress every second + if time.Since(lastProgressReport) >= time.Second { + opts.Logger.Info("Reverse verification progress", + "target_keys_checked", targetKeys, + "mismatches", mismatchCount, + ) + lastProgressReport = time.Now() + } + } + + if err := targetItr.Error(); err != nil { + return fmt.Errorf("error during target iteration: %w", err) + } + + // Compare key counts + if targetKeys != verifiedKeys { + opts.Logger.Error("Key count mismatch", + "source_keys", verifiedKeys, + "target_keys", targetKeys, + "difference", targetKeys-verifiedKeys, + ) + mismatchCount++ + } + + if mismatchCount > 0 { + return fmt.Errorf("verification failed: %d mismatches found", mismatchCount) + } + + opts.Logger.Info("Verification summary", + "verified_keys", verifiedKeys, + "target_keys", targetKeys, + "mismatches", mismatchCount, + ) + + return nil +} diff --git a/cmd/cronosd/dbmigrate/migrate_basic_test.go b/cmd/cronosd/dbmigrate/migrate_basic_test.go new file mode 100644 index 0000000000..df4f161439 --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate_basic_test.go @@ -0,0 +1,349 @@ +//go:build !rocksdb +// +build !rocksdb + +package dbmigrate + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + dbm "github.com/cosmos/cosmos-db" + "github.com/stretchr/testify/require" + + "cosmossdk.io/log" +) + +// setupBasicTestDB creates a test database with sample data (no RocksDB) +func setupBasicTestDB(t *testing.T, backend dbm.BackendType, numKeys int) (string, dbm.DB) { + t.Helper() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + db, err := dbm.NewDB("application", backend, dataDir) + require.NoError(t, err) + + // Populate with test data + for i := 0; i < numKeys; i++ { + key := []byte(fmt.Sprintf("key-%06d", i)) + value := []byte(fmt.Sprintf("value-%06d-data-for-testing-migration", i)) + err := db.Set(key, value) + require.NoError(t, err) + } + + return tempDir, db +} + +// TestCountKeys tests the key counting functionality +func TestCountKeys(t *testing.T) { + tests := []struct { + name string + backend dbm.BackendType + numKeys int + }{ + { + name: "leveldb with 100 keys", + backend: dbm.GoLevelDBBackend, + numKeys: 100, + }, + { + name: "leveldb with 0 keys", + backend: dbm.GoLevelDBBackend, + numKeys: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, db := setupBasicTestDB(t, tt.backend, tt.numKeys) + defer db.Close() + + count, err := countKeys(db) + require.NoError(t, err) + require.Equal(t, int64(tt.numKeys), count) + }) + } +} + +// TestMigrateLevelDBToLevelDB tests basic migration functionality +func TestMigrateLevelDBToLevelDB(t *testing.T) { + numKeys := 100 + + // Setup source database + sourceDir, sourceDB := setupBasicTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) +} + +// TestMigrationStats tests the statistics tracking +func TestMigrationStats(t *testing.T) { + stats := &MigrationStats{} + + // Test initial state + require.Equal(t, int64(0), stats.TotalKeys.Load()) + require.Equal(t, int64(0), stats.ProcessedKeys.Load()) + require.Equal(t, float64(0), stats.Progress()) + + // Test with some values + stats.TotalKeys.Store(100) + stats.ProcessedKeys.Store(50) + require.Equal(t, float64(50), stats.Progress()) + + stats.ProcessedKeys.Store(100) + require.Equal(t, float64(100), stats.Progress()) +} + +// TestMigrateLargeDatabase tests migration with a larger dataset +func TestMigrateLargeDatabase(t *testing.T) { + if testing.Short() { + t.Skip("Skipping large database test in short mode") + } + + numKeys := 10000 + + // Setup source database + sourceDir, sourceDB := setupBasicTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration with smaller batch size + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, // Use LevelDB for verification to work + BatchSize: 100, + Logger: log.NewTestLogger(t), + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) +} + +// TestMigrateEmptyDatabase tests migration of an empty database +func TestMigrateEmptyDatabase(t *testing.T) { + // Setup empty source database + sourceDir, sourceDB := setupBasicTestDB(t, dbm.GoLevelDBBackend, 0) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(0), stats.TotalKeys.Load()) + require.Equal(t, int64(0), stats.ProcessedKeys.Load()) +} + +// TestMigrationWithoutVerification tests migration without verification +func TestMigrationWithoutVerification(t *testing.T) { + numKeys := 100 + + // Setup source database + sourceDir, sourceDB := setupBasicTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration without verification + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) +} + +// TestMigrationBatchSizes tests migration with different batch sizes +func TestMigrationBatchSizes(t *testing.T) { + numKeys := 150 + batchSizes := []int{1, 10, 50, 100, 200} + + for _, batchSize := range batchSizes { + t.Run(fmt.Sprintf("batch_size_%d", batchSize), func(t *testing.T) { + // Setup source database + sourceDir, sourceDB := setupBasicTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: batchSize, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + }) + } +} + +// TestMigrateSpecialKeys tests migration with special key patterns +func TestMigrateSpecialKeys(t *testing.T) { + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + db, err := dbm.NewDB("application", dbm.GoLevelDBBackend, dataDir) + require.NoError(t, err) + + // Add keys with special patterns + specialKeys := [][]byte{ + []byte("\x00"), // null byte + []byte("\x00\x00\x00"), // multiple null bytes + []byte("key with spaces"), // spaces + []byte("key\nwith\nnewlines"), // newlines + []byte("🔑emoji-key"), // unicode + make([]byte, 1024), // large key + } + + for i, key := range specialKeys { + if len(key) > 0 { // Skip empty key if not supported + value := []byte(fmt.Sprintf("value-%d", i)) + err := db.Set(key, value) + require.NoError(t, err) + } + } + db.Close() + + // Now migrate + targetDir := t.TempDir() + opts := MigrateOptions{ + SourceHome: tempDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 2, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Greater(t, stats.ProcessedKeys.Load(), int64(0)) +} + +// TestMigrationPathCorrectness verifies that logged paths match actual database locations +// Unified path format for all backends: .migrate-temp.db +func TestMigrationPathCorrectness(t *testing.T) { + tests := []struct { + name string + backend dbm.BackendType + expectedSuffix string + }{ + { + name: "LevelDB uses unified .migrate-temp.db format", + backend: dbm.GoLevelDBBackend, + expectedSuffix: ".migrate-temp.db", + }, + // Note: RocksDB also uses .migrate-temp.db but requires rocksdb build tag to test + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup source database + sourceDir, sourceDB := setupBasicTestDB(t, tt.backend, 10) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: tt.backend, + TargetBackend: tt.backend, + DBName: "application", + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + + // Verify the actual database directory exists + targetDataDir := filepath.Join(targetDir, "data") + expectedPath := filepath.Join(targetDataDir, "application"+tt.expectedSuffix) + + // Check that the directory exists + info, err := os.Stat(expectedPath) + require.NoError(t, err, "Database directory should exist at expected path: %s", expectedPath) + require.True(t, info.IsDir(), "Expected path should be a directory") + + // Verify we can open the database at this path + db, err := dbm.NewDB("application.migrate-temp", tt.backend, targetDataDir) + require.NoError(t, err, "Should be able to open database at the expected path") + defer db.Close() + + // Verify it has the correct data + count, err := countKeys(db) + require.NoError(t, err) + require.Equal(t, int64(10), count, "Database should contain all migrated keys") + }) + } +} diff --git a/cmd/cronosd/dbmigrate/migrate_dbname_test.go b/cmd/cronosd/dbmigrate/migrate_dbname_test.go new file mode 100644 index 0000000000..ee4319d08b --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate_dbname_test.go @@ -0,0 +1,343 @@ +//go:build !rocksdb +// +build !rocksdb + +package dbmigrate + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + dbm "github.com/cosmos/cosmos-db" + "github.com/stretchr/testify/require" + + "cosmossdk.io/log" +) + +// setupTestDBWithName creates a test database with a specific name +func setupTestDBWithName(t *testing.T, backend dbm.BackendType, dbName string, numKeys int) (string, dbm.DB) { + t.Helper() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + db, err := dbm.NewDB(dbName, backend, dataDir) + require.NoError(t, err) + + // Populate with test data + for i := 0; i < numKeys; i++ { + key := []byte(fmt.Sprintf("key-%s-%06d", dbName, i)) + value := []byte(fmt.Sprintf("value-%s-%06d-data", dbName, i)) + err := db.Set(key, value) + require.NoError(t, err) + } + + return tempDir, db +} + +// TestMigrateWithDBName tests migration with specific database names +func TestMigrateWithDBName(t *testing.T) { + dbNames := []string{"application", "blockstore", "state", "tx_index", "evidence"} + + for _, dbName := range dbNames { + t.Run(dbName, func(t *testing.T) { + numKeys := 50 + + // Setup source database with specific name + sourceDir, sourceDB := setupTestDBWithName(t, dbm.GoLevelDBBackend, dbName, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration with explicit DBName + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: true, + DBName: dbName, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) + + // Verify duration is positive + require.Positive(t, stats.Duration()) + }) + } +} + +// TestMigrateMultipleDatabases tests migrating multiple databases sequentially +func TestMigrateMultipleDatabases(t *testing.T) { + dbNames := []string{"blockstore", "tx_index"} + numKeys := 100 + + // Setup source databases + sourceDir := t.TempDir() + dataDir := filepath.Join(sourceDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + // Create multiple source databases + for _, dbName := range dbNames { + db, err := dbm.NewDB(dbName, dbm.GoLevelDBBackend, dataDir) + require.NoError(t, err) + + // Populate with test data + for i := 0; i < numKeys; i++ { + key := []byte(fmt.Sprintf("key-%s-%06d", dbName, i)) + value := []byte(fmt.Sprintf("value-%s-%06d", dbName, i)) + err := db.Set(key, value) + require.NoError(t, err) + } + db.Close() + } + + // Create target directory + targetDir := t.TempDir() + + // Migrate each database + var totalProcessed int64 + for _, dbName := range dbNames { + t.Run("migrate_"+dbName, func(t *testing.T) { + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 20, + Logger: log.NewTestLogger(t), + Verify: true, + DBName: dbName, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) + + totalProcessed += stats.ProcessedKeys.Load() + }) + } + + // Verify total keys migrated + expectedTotal := int64(numKeys * len(dbNames)) + require.Equal(t, expectedTotal, totalProcessed) +} + +// TestMigrateWithDefaultDBName tests that migration defaults to "application" when DBName is not set +func TestMigrateWithDefaultDBName(t *testing.T) { + numKeys := 50 + + // Setup source database with "application" name + sourceDir, sourceDB := setupTestDBWithName(t, dbm.GoLevelDBBackend, "application", numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration without specifying DBName (should default to "application") + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: true, + // DBName is intentionally not set + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) +} + +// TestMigrateCometBFTDatabases tests migrating all CometBFT databases +func TestMigrateCometBFTDatabases(t *testing.T) { + cometbftDBs := []string{"blockstore", "state", "tx_index", "evidence"} + numKeys := 25 + + // Setup source databases + sourceDir := t.TempDir() + dataDir := filepath.Join(sourceDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + // Create CometBFT databases + for _, dbName := range cometbftDBs { + db, err := dbm.NewDB(dbName, dbm.GoLevelDBBackend, dataDir) + require.NoError(t, err) + + // Add some data specific to each database + for i := 0; i < numKeys; i++ { + key := []byte(fmt.Sprintf("%s-key-%d", dbName, i)) + value := []byte(fmt.Sprintf("%s-value-%d", dbName, i)) + err := db.Set(key, value) + require.NoError(t, err) + } + db.Close() + } + + // Create target directory + targetDir := t.TempDir() + + // Migrate each CometBFT database + for _, dbName := range cometbftDBs { + t.Run(dbName, func(t *testing.T) { + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + DBName: dbName, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + }) + } +} + +// TestMigrateEmptyDatabaseWithName tests migration of an empty database with a specific name +func TestMigrateEmptyDatabaseWithName(t *testing.T) { + dbName := "empty_db" + + // Create an empty database + sourceDir, sourceDB := setupTestDBWithName(t, dbm.GoLevelDBBackend, dbName, 0) + sourceDB.Close() + + targetDir := t.TempDir() + + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + DBName: dbName, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(0), stats.TotalKeys.Load()) + require.Equal(t, int64(0), stats.ProcessedKeys.Load()) +} + +// TestMigrateDifferentDBNames tests migrating databases with different names to ensure isolation +func TestMigrateDifferentDBNames(t *testing.T) { + numKeys := 30 + db1Name := "db_one" + db2Name := "db_two" + + // Setup source directory with two different databases + sourceDir := t.TempDir() + dataDir := filepath.Join(sourceDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + // Create first database + db1, err := dbm.NewDB(db1Name, dbm.GoLevelDBBackend, dataDir) + require.NoError(t, err) + for i := 0; i < numKeys; i++ { + err := db1.Set([]byte(fmt.Sprintf("db1-key-%d", i)), []byte("db1-value")) + require.NoError(t, err) + } + db1.Close() + + // Create second database with different data + db2, err := dbm.NewDB(db2Name, dbm.GoLevelDBBackend, dataDir) + require.NoError(t, err) + for i := 0; i < numKeys*2; i++ { // Different number of keys + err := db2.Set([]byte(fmt.Sprintf("db2-key-%d", i)), []byte("db2-value")) + require.NoError(t, err) + } + db2.Close() + + targetDir := t.TempDir() + + // Migrate first database + opts1 := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + DBName: db1Name, + } + + stats1, err := Migrate(opts1) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats1.TotalKeys.Load()) + + // Migrate second database + opts2 := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + DBName: db2Name, + } + + stats2, err := Migrate(opts2) + require.NoError(t, err) + require.Equal(t, int64(numKeys*2), stats2.TotalKeys.Load()) + + // Verify both databases were migrated separately + require.NotEqual(t, stats1.TotalKeys.Load(), stats2.TotalKeys.Load(), "databases should have different key counts") +} + +// TestMigrateDBNameWithSpecialCharacters tests database names with underscores +func TestMigrateDBNameWithSpecialCharacters(t *testing.T) { + dbName := "tx_index" // Contains underscore + numKeys := 40 + + sourceDir, sourceDB := setupTestDBWithName(t, dbm.GoLevelDBBackend, dbName, numKeys) + sourceDB.Close() + + targetDir := t.TempDir() + + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 15, + Logger: log.NewNopLogger(), + Verify: false, + DBName: dbName, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) +} diff --git a/cmd/cronosd/dbmigrate/migrate_no_rocksdb.go b/cmd/cronosd/dbmigrate/migrate_no_rocksdb.go new file mode 100644 index 0000000000..82334c5193 --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate_no_rocksdb.go @@ -0,0 +1,32 @@ +//go:build !rocksdb +// +build !rocksdb + +package dbmigrate + +import ( + "fmt" + + dbm "github.com/cosmos/cosmos-db" +) + +// PrepareRocksDBOptions returns nil when RocksDB is not enabled +func PrepareRocksDBOptions() interface{} { + return nil +} + +// openRocksDBForMigration is a stub that returns an error when rocksdb is not available +func openRocksDBForMigration(dir string, opts interface{}) (dbm.DB, error) { + return nil, fmt.Errorf("rocksdb support not enabled, rebuild with -tags rocksdb") +} + +// openRocksDBForRead is a stub that returns an error when rocksdb is not available +func openRocksDBForRead(dir string) (dbm.DB, error) { + return nil, fmt.Errorf("rocksdb support not enabled, rebuild with -tags rocksdb") +} + +// flushRocksDB is a stub that returns an error when rocksdb is not available +func flushRocksDB(db dbm.DB) error { + // This should never be called since migrate.go checks TargetBackend == RocksDBBackend + // But we need the stub for compilation + return fmt.Errorf("rocksdb support not enabled, rebuild with -tags rocksdb") +} diff --git a/cmd/cronosd/dbmigrate/migrate_rocksdb.go b/cmd/cronosd/dbmigrate/migrate_rocksdb.go new file mode 100644 index 0000000000..da195b8694 --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate_rocksdb.go @@ -0,0 +1,92 @@ +//go:build rocksdb +// +build rocksdb + +package dbmigrate + +import ( + dbm "github.com/cosmos/cosmos-db" + "github.com/crypto-org-chain/cronos/cmd/cronosd/opendb" + "github.com/linxGnu/grocksdb" +) + +// PrepareRocksDBOptions returns RocksDB options for migration +func PrepareRocksDBOptions() interface{} { + return opendb.NewRocksdbOptions(nil, false) +} + +// openRocksDBForMigration opens a RocksDB database for migration (write mode) +func openRocksDBForMigration(dir string, optsInterface interface{}) (dbm.DB, error) { + var opts *grocksdb.Options + var createdOpts bool + + // Type assert from interface{} to *grocksdb.Options + if optsInterface != nil { + var ok bool + opts, ok = optsInterface.(*grocksdb.Options) + if !ok { + // If type assertion fails, use default options + opts = nil + } + } + // Handle nil opts by creating default options + if opts == nil { + opts = grocksdb.NewDefaultOptions() + opts.SetCreateIfMissing(true) + opts.SetLevelCompactionDynamicLevelBytes(true) + createdOpts = true // Track that we created these options + } + + // Ensure we clean up options we created after opening the database + // Options are copied internally by RocksDB, so they can be destroyed after OpenDb + if createdOpts { + defer opts.Destroy() + } + + ro := grocksdb.NewDefaultReadOptions() + wo := grocksdb.NewDefaultWriteOptions() + woSync := grocksdb.NewDefaultWriteOptions() + woSync.SetSync(true) + + db, err := grocksdb.OpenDb(opts, dir) + if err != nil { + // Clean up read/write options on error + ro.Destroy() + wo.Destroy() + woSync.Destroy() + return nil, err + } + + // Note: ro, wo, woSync are NOT destroyed here - they're needed for database operations + // and will be cleaned up when the database is closed + return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil +} + +// openRocksDBForRead opens a RocksDB database in read-only mode +func openRocksDBForRead(dir string) (dbm.DB, error) { + opts := grocksdb.NewDefaultOptions() + defer opts.Destroy() + db, err := grocksdb.OpenDbForReadOnly(opts, dir, false) + if err != nil { + return nil, err + } + + ro := grocksdb.NewDefaultReadOptions() + wo := grocksdb.NewDefaultWriteOptions() + woSync := grocksdb.NewDefaultWriteOptions() + woSync.SetSync(true) + + return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil +} + +// flushRocksDB explicitly flushes the memtable to SST files +func flushRocksDB(db dbm.DB) error { + // Type assert to get the underlying RocksDB instance + if rocksDB, ok := db.(*dbm.RocksDB); ok { + opts := grocksdb.NewDefaultFlushOptions() + defer opts.Destroy() + opts.SetWait(true) // Wait for flush to complete + + return rocksDB.DB().Flush(opts) + } + return nil // Not a RocksDB instance, nothing to flush +} diff --git a/cmd/cronosd/dbmigrate/migrate_rocksdb_test.go b/cmd/cronosd/dbmigrate/migrate_rocksdb_test.go new file mode 100644 index 0000000000..1c91c53a7a --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate_rocksdb_test.go @@ -0,0 +1,338 @@ +//go:build rocksdb +// +build rocksdb + +package dbmigrate + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + dbm "github.com/cosmos/cosmos-db" + "github.com/linxGnu/grocksdb" + "github.com/stretchr/testify/require" + + "cosmossdk.io/log" +) + +// newRocksDBOptions creates RocksDB options similar to the app configuration +func newRocksDBOptions() *grocksdb.Options { + opts := grocksdb.NewDefaultOptions() + opts.SetCreateIfMissing(true) + opts.SetLevelCompactionDynamicLevelBytes(true) + opts.IncreaseParallelism(runtime.NumCPU()) + opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) + opts.SetTargetFileSizeMultiplier(2) + + // block based table options + bbto := grocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(grocksdb.NewLRUCache(64 << 20)) // 64MB is ample for tests + bbto.SetFilterPolicy(grocksdb.NewRibbonHybridFilterPolicy(9.9, 1)) + bbto.SetIndexType(grocksdb.KTwoLevelIndexSearchIndexType) + bbto.SetPartitionFilters(true) + bbto.SetOptimizeFiltersForMemory(true) + bbto.SetCacheIndexAndFilterBlocks(true) + bbto.SetPinTopLevelIndexAndFilter(true) + bbto.SetPinL0FilterAndIndexBlocksInCache(true) + bbto.SetDataBlockIndexType(grocksdb.KDataBlockIndexTypeBinarySearchAndHash) + opts.SetBlockBasedTableFactory(bbto) + opts.SetOptimizeFiltersForHits(true) + + return opts +} + +// setupRocksDB creates a test RocksDB database with sample data +func setupRocksDB(t *testing.T, numKeys int) (string, dbm.DB) { + t.Helper() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + opts := newRocksDBOptions() + t.Cleanup(func() { opts.Destroy() }) + rocksDir := filepath.Join(dataDir, "application.db") + rawDB, err := grocksdb.OpenDb(opts, rocksDir) + require.NoError(t, err) + + ro := grocksdb.NewDefaultReadOptions() + t.Cleanup(func() { ro.Destroy() }) + wo := grocksdb.NewDefaultWriteOptions() + t.Cleanup(func() { wo.Destroy() }) + woSync := grocksdb.NewDefaultWriteOptions() + t.Cleanup(func() { woSync.Destroy() }) + woSync.SetSync(true) + db := dbm.NewRocksDBWithRawDB(rawDB, ro, wo, woSync) + + // Populate with test data + for i := 0; i < numKeys; i++ { + key := []byte(fmt.Sprintf("key-%06d", i)) + value := []byte(fmt.Sprintf("value-%06d-data-for-testing-rocksdb-migration", i)) + err := db.Set(key, value) + require.NoError(t, err) + } + + return tempDir, db +} + +// TestMigrateLevelDBToRocksDB tests migration from LevelDB to RocksDB +func TestMigrateLevelDBToRocksDB(t *testing.T) { + numKeys := 1000 + + // Setup source database with LevelDB + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, numKeys) + + // Store expected key-value pairs + expectedData := make(map[string]string) + for i := 0; i < numKeys; i++ { + key := fmt.Sprintf("key-%06d", i) + value := fmt.Sprintf("value-%06d-data-for-testing-migration", i) + expectedData[key] = value + } + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + rocksOpts := newRocksDBOptions() + defer rocksOpts.Destroy() + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.RocksDBBackend, + BatchSize: 100, + Logger: log.NewTestLogger(t), + RocksDBOptions: rocksOpts, + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) + + // Verify the migrated data by opening the target database + // Unified path format: application.migrate-temp.db + targetDBPath := filepath.Join(targetDir, "data", "application.migrate-temp.db") + targetDB, err := openRocksDBForRead(targetDBPath) + require.NoError(t, err) + defer targetDB.Close() + + // Check a few random keys + for i := 0; i < 10; i++ { + key := []byte(fmt.Sprintf("key-%06d", i)) + value, err := targetDB.Get(key) + require.NoError(t, err) + expectedValue := []byte(expectedData[string(key)]) + require.Equal(t, expectedValue, value) + } +} + +// TestMigrateRocksDBToLevelDB tests migration from RocksDB to LevelDB +func TestMigrateRocksDBToLevelDB(t *testing.T) { + numKeys := 500 + + // Setup source database with RocksDB + sourceDir, sourceDB := setupRocksDB(t, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + rocksOpts := newRocksDBOptions() + defer rocksOpts.Destroy() + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.RocksDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 50, + Logger: log.NewTestLogger(t), + RocksDBOptions: rocksOpts, + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) +} + +// TestMigrateRocksDBToRocksDB tests migration between RocksDB instances +func TestMigrateRocksDBToRocksDB(t *testing.T) { + numKeys := 300 + + // Setup source database with RocksDB + sourceDir, sourceDB := setupRocksDB(t, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration (useful for compaction or options change) + rocksOpts := newRocksDBOptions() + defer rocksOpts.Destroy() + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.RocksDBBackend, + TargetBackend: dbm.RocksDBBackend, + BatchSize: 100, + Logger: log.NewTestLogger(t), + RocksDBOptions: rocksOpts, + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) +} + +// TestMigrateRocksDBLargeDataset tests RocksDB migration with a large dataset +func TestMigrateRocksDBLargeDataset(t *testing.T) { + if testing.Short() { + t.Skip("Skipping large dataset test in short mode") + } + + numKeys := 50000 + + // Setup source database with RocksDB + sourceDir, sourceDB := setupRocksDB(t, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + rocksOpts := newRocksDBOptions() + defer rocksOpts.Destroy() + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.RocksDBBackend, + TargetBackend: dbm.RocksDBBackend, + BatchSize: 1000, + Logger: log.NewTestLogger(t), + RocksDBOptions: rocksOpts, + Verify: false, // Skip verification for speed + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) + + t.Logf("Migrated %d keys in %s", numKeys, stats.Duration()) +} + +// TestMigrateRocksDBWithDifferentOptions tests migration with custom RocksDB options +func TestMigrateRocksDBWithDifferentOptions(t *testing.T) { + numKeys := 100 + + // Setup source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Create custom RocksDB options with different settings + customOpts := grocksdb.NewDefaultOptions() + defer customOpts.Destroy() + customOpts.SetCreateIfMissing(true) + customOpts.SetLevelCompactionDynamicLevelBytes(true) + // Different compression + customOpts.SetCompression(grocksdb.SnappyCompression) + + // Perform migration with custom options + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.RocksDBBackend, + BatchSize: 50, + Logger: log.NewTestLogger(t), + RocksDBOptions: customOpts, + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) +} + +// TestMigrateRocksDBDataIntegrity tests that data integrity is maintained during migration +func TestMigrateRocksDBDataIntegrity(t *testing.T) { + numKeys := 1000 + + // Setup source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, numKeys) + + // Read all source data before closing + sourceData := make(map[string][]byte) + itr, err := sourceDB.Iterator(nil, nil) + require.NoError(t, err) + for ; itr.Valid(); itr.Next() { + key := make([]byte, len(itr.Key())) + value := make([]byte, len(itr.Value())) + copy(key, itr.Key()) + copy(value, itr.Value()) + sourceData[string(key)] = value + } + require.NoError(t, itr.Error()) + itr.Close() + sourceDB.Close() + + // Perform migration + targetDir := t.TempDir() + rocksOpts := newRocksDBOptions() + defer rocksOpts.Destroy() + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.RocksDBBackend, + BatchSize: 100, + Logger: log.NewNopLogger(), + RocksDBOptions: rocksOpts, + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + + // Open target database and verify all data + // Unified path format: application.migrate-temp.db + targetDBPath := filepath.Join(targetDir, "data", "application.migrate-temp.db") + targetDB, err := openRocksDBForRead(targetDBPath) + require.NoError(t, err) + defer targetDB.Close() + + // Verify every key + verifiedCount := 0 + for key, expectedValue := range sourceData { + actualValue, err := targetDB.Get([]byte(key)) + require.NoError(t, err, "Failed to get key: %s", key) + require.Equal(t, expectedValue, actualValue, "Value mismatch for key: %s", key) + verifiedCount++ + } + + require.Equal(t, len(sourceData), verifiedCount) + t.Logf("Verified %d keys successfully", verifiedCount) +} diff --git a/cmd/cronosd/dbmigrate/migrate_test.go b/cmd/cronosd/dbmigrate/migrate_test.go new file mode 100644 index 0000000000..128908c6a7 --- /dev/null +++ b/cmd/cronosd/dbmigrate/migrate_test.go @@ -0,0 +1,423 @@ +//go:build rocksdb +// +build rocksdb + +package dbmigrate + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + dbm "github.com/cosmos/cosmos-db" + "github.com/linxGnu/grocksdb" + "github.com/stretchr/testify/require" + + "cosmossdk.io/log" +) + +// setupTestDB creates a test database with sample data +func setupTestDB(t *testing.T, backend dbm.BackendType, numKeys int) (string, dbm.DB) { + t.Helper() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + var db dbm.DB + if backend == dbm.RocksDBBackend { + opts := grocksdb.NewDefaultOptions() + defer opts.Destroy() + opts.SetCreateIfMissing(true) + rocksDir := filepath.Join(dataDir, "application.db") + rawDB, err := grocksdb.OpenDb(opts, rocksDir) + require.NoError(t, err) + + ro := grocksdb.NewDefaultReadOptions() + defer ro.Destroy() + wo := grocksdb.NewDefaultWriteOptions() + defer wo.Destroy() + woSync := grocksdb.NewDefaultWriteOptions() + defer woSync.Destroy() + woSync.SetSync(true) + db = dbm.NewRocksDBWithRawDB(rawDB, ro, wo, woSync) + } else { + db, err = dbm.NewDB("application", backend, dataDir) + require.NoError(t, err) + } + + // Populate with test data + for i := 0; i < numKeys; i++ { + key := []byte(fmt.Sprintf("key-%06d", i)) + value := []byte(fmt.Sprintf("value-%06d-data-for-testing-migration", i)) + err := db.Set(key, value) + require.NoError(t, err) + } + + return tempDir, db +} + +// TestCountKeys tests the key counting functionality +func TestCountKeys(t *testing.T) { + tests := []struct { + name string + backend dbm.BackendType + numKeys int + }{ + { + name: "leveldb with 100 keys", + backend: dbm.GoLevelDBBackend, + numKeys: 100, + }, + { + name: "leveldb with 0 keys", + backend: dbm.GoLevelDBBackend, + numKeys: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, db := setupTestDB(t, tt.backend, tt.numKeys) + defer db.Close() + + count, err := countKeys(db) + require.NoError(t, err) + require.Equal(t, int64(tt.numKeys), count) + }) + } +} + +// TestMigrationStats tests the statistics tracking +func TestMigrationStats(t *testing.T) { + stats := &MigrationStats{} + + // Test initial state + require.Equal(t, int64(0), stats.TotalKeys.Load()) + require.Equal(t, int64(0), stats.ProcessedKeys.Load()) + require.Equal(t, float64(0), stats.Progress()) + + // Test with some values + stats.TotalKeys.Store(100) + stats.ProcessedKeys.Store(50) + require.Equal(t, float64(50), stats.Progress()) + + stats.ProcessedKeys.Store(100) + require.Equal(t, float64(100), stats.Progress()) +} + +// TestMigrateLargeDatabase tests migration with a larger dataset +func TestMigrateLargeDatabase(t *testing.T) { + if testing.Short() { + t.Skip("Skipping large database test in short mode") + } + + numKeys := 10000 + + // Setup source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration with smaller batch size + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 100, + Logger: log.NewTestLogger(t), + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + require.Equal(t, int64(0), stats.ErrorCount.Load()) +} + +// TestMigrateEmptyDatabase tests migration of an empty database +func TestMigrateEmptyDatabase(t *testing.T) { + // Setup empty source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, 0) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: true, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(0), stats.TotalKeys.Load()) + require.Equal(t, int64(0), stats.ProcessedKeys.Load()) +} + +// TestMigrationWithoutVerification tests migration without verification +func TestMigrationWithoutVerification(t *testing.T) { + numKeys := 100 + + // Setup source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration without verification + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 10, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.NotNil(t, stats) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) +} + +// TestMigrationBatchSizes tests migration with different batch sizes +func TestMigrationBatchSizes(t *testing.T) { + numKeys := 150 + batchSizes := []int{1, 10, 50, 100, 200} + + for _, batchSize := range batchSizes { + t.Run(fmt.Sprintf("batch_size_%d", batchSize), func(t *testing.T) { + // Setup source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, numKeys) + sourceDB.Close() + + // Create target directory + targetDir := t.TempDir() + + // Perform migration + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: batchSize, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + require.NoError(t, err) + require.Equal(t, int64(numKeys), stats.TotalKeys.Load()) + require.Equal(t, int64(numKeys), stats.ProcessedKeys.Load()) + }) + } +} + +// TestVerifyMigration tests the verification functionality +func TestVerifyMigration(t *testing.T) { + tests := []struct { + name string + numKeys int + setupMismatch func(sourceDB, targetDB dbm.DB) error + expectError bool + }{ + { + name: "identical databases should pass verification", + numKeys: 50, + setupMismatch: nil, + expectError: false, + }, + { + name: "value mismatch should fail verification", + numKeys: 50, + setupMismatch: func(sourceDB, targetDB dbm.DB) error { + // Change a value in target + return targetDB.Set([]byte("key-000010"), []byte("different-value")) + }, + expectError: true, + }, + { + name: "extra key in target should fail verification", + numKeys: 50, + setupMismatch: func(sourceDB, targetDB dbm.DB) error { + // Add an extra key to target that doesn't exist in source + return targetDB.Set([]byte("extra-key-in-target"), []byte("extra-value")) + }, + expectError: true, + }, + { + name: "missing key in target should fail verification", + numKeys: 50, + setupMismatch: func(sourceDB, targetDB dbm.DB) error { + // Delete a key from target + return targetDB.Delete([]byte("key-000010")) + }, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup source database + sourceDir, sourceDB := setupTestDB(t, dbm.GoLevelDBBackend, tt.numKeys) + defer sourceDB.Close() + + // Setup target database by copying data from source + targetDir := t.TempDir() + targetDataDir := filepath.Join(targetDir, "data") + err := os.MkdirAll(targetDataDir, 0o755) + require.NoError(t, err) + + targetDB, err := dbm.NewDB("application.migrate-temp", dbm.GoLevelDBBackend, targetDataDir) + require.NoError(t, err) + + // Copy all data from source to target + itr, err := sourceDB.Iterator(nil, nil) + require.NoError(t, err) + defer itr.Close() + for ; itr.Valid(); itr.Next() { + err := targetDB.Set(itr.Key(), itr.Value()) + require.NoError(t, err) + } + + // Apply mismatch if specified + if tt.setupMismatch != nil { + err := tt.setupMismatch(sourceDB, targetDB) + require.NoError(t, err) + } + + // Close databases before verification + sourceDB.Close() + targetDB.Close() + + opts := MigrateOptions{ + SourceHome: sourceDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + DBName: "application", + Logger: log.NewNopLogger(), + } + + // Perform verification + err = verifyMigration( + filepath.Join(sourceDir, "data"), + filepath.Join(targetDataDir, "application.migrate-temp.db"), + opts, + ) + + if tt.expectError { + require.Error(t, err, "expected verification to fail but it passed") + } else { + require.NoError(t, err, "expected verification to pass but it failed") + } + }) + } +} + +// TestMigrateSpecialKeys tests migration with special key patterns +func TestMigrateSpecialKeys(t *testing.T) { + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "data") + err := os.MkdirAll(dataDir, 0o755) + require.NoError(t, err) + + db, err := dbm.NewDB("application", dbm.GoLevelDBBackend, dataDir) + require.NoError(t, err) + + // Add keys with special patterns + type keyValuePair struct { + key []byte + value []byte + } + + specialKeys := [][]byte{ + []byte(""), // empty key (may not be supported) + []byte("\x00"), // null byte + []byte("\x00\x00\x00"), // multiple null bytes + []byte("key with spaces"), // spaces + []byte("key\nwith\nnewlines"), // newlines + []byte("🔑emoji-key"), // unicode + make([]byte, 1024), // large key + } + + // Track successfully written keys + var expectedKeys []keyValuePair + + for i, key := range specialKeys { + value := []byte(fmt.Sprintf("value-%d", i)) + err := db.Set(key, value) + if err != nil { + // Only skip empty key if explicitly unsupported + if len(key) == 0 { + t.Logf("Skipping empty key (unsupported): %v", err) + continue + } + // Any other key failure is unexpected and should fail the test + require.NoError(t, err, "unexpected error setting key at index %d", i) + } + + // Record successfully written key + expectedKeys = append(expectedKeys, keyValuePair{ + key: key, + value: value, + }) + t.Logf("Successfully wrote key %d: len=%d", i, len(key)) + } + db.Close() + + require.Greater(t, len(expectedKeys), 0, "no keys were successfully written to source DB") + + // Now migrate + targetDir := t.TempDir() + opts := MigrateOptions{ + SourceHome: tempDir, + TargetHome: targetDir, + SourceBackend: dbm.GoLevelDBBackend, + TargetBackend: dbm.GoLevelDBBackend, + BatchSize: 2, + Logger: log.NewNopLogger(), + Verify: false, + } + + stats, err := Migrate(opts) + + // Assert no migration errors + require.NoError(t, err, "migration should complete without error") + require.Equal(t, int64(0), stats.ErrorCount.Load(), "migration should have zero errors") + + // Assert the number of migrated keys equals the number written + require.Equal(t, int64(len(expectedKeys)), stats.ProcessedKeys.Load(), + "number of migrated keys should equal number of keys written") + + // Open target DB and verify each expected key + targetDataDir := filepath.Join(targetDir, "data") + targetDB, err := dbm.NewDB("application.migrate-temp", dbm.GoLevelDBBackend, targetDataDir) + require.NoError(t, err) + defer targetDB.Close() + + for i, pair := range expectedKeys { + gotValue, err := targetDB.Get(pair.key) + require.NoError(t, err, "failed to get key %d from target DB", i) + require.NotNil(t, gotValue, "key %d should exist in target DB", i) + require.Equal(t, pair.value, gotValue, + "value for key %d should match expected value", i) + t.Logf("Verified key %d: value matches", i) + } +} diff --git a/cmd/cronosd/dbmigrate/patch.go b/cmd/cronosd/dbmigrate/patch.go new file mode 100644 index 0000000000..69332e9820 --- /dev/null +++ b/cmd/cronosd/dbmigrate/patch.go @@ -0,0 +1,1341 @@ +package dbmigrate + +import ( + "bufio" + "bytes" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + abci "github.com/cometbft/cometbft/abci/types" + dbm "github.com/cosmos/cosmos-db" + "github.com/cosmos/gogoproto/proto" + + "cosmossdk.io/log" +) + +const ( + DbExtension = ".db" +) + +// EthTxInfo stores information needed to search for event-indexed keys in source DB +type EthTxInfo struct { + Height int64 // Block height + TxIndex int64 // Transaction index within block +} + +// ConflictResolution represents how to handle key conflicts +type ConflictResolution int + +const ( + // ConflictAsk prompts user for each conflict + ConflictAsk ConflictResolution = iota + // ConflictSkip skips conflicting keys + ConflictSkip + // ConflictReplace replaces conflicting keys + ConflictReplace + // ConflictReplaceAll replaces all conflicting keys without asking + ConflictReplaceAll +) + +// PatchOptions contains options for patching databases +type PatchOptions struct { + SourceHome string // Source home directory + TargetPath string // Target database path (exact path to patch) + SourceBackend dbm.BackendType // Source backend type + TargetBackend dbm.BackendType // Target backend type + BatchSize int // Batch size for writing + Logger log.Logger // Logger + RocksDBOptions interface{} // RocksDB specific options + DBName string // Database name (blockstore, tx_index, etc.) + HeightRange HeightRange // Height range/specific heights to patch + ConflictStrategy ConflictResolution // How to handle key conflicts + SkipConflictChecks bool // Skip checking for conflicts (faster, overwrites all) + DryRun bool // If true, simulate operation without writing +} + +// validatePatchOptions validates the patch options +func validatePatchOptions(opts PatchOptions) error { + if opts.Logger == nil { + return fmt.Errorf("logger is required") + } + if opts.HeightRange.IsEmpty() { + return fmt.Errorf("height range is required for patching") + } + if !supportsHeightFiltering(opts.DBName) { + return fmt.Errorf("database %s does not support height-based patching (only blockstore and tx_index supported)", opts.DBName) + } + + // Construct and validate source database path + sourceDBPath := filepath.Join(opts.SourceHome, "data", opts.DBName+DbExtension) + if _, err := os.Stat(sourceDBPath); os.IsNotExist(err) { + return fmt.Errorf("source database does not exist: %s", sourceDBPath) + } + + // Validate target exists + if _, err := os.Stat(opts.TargetPath); os.IsNotExist(err) { + return fmt.Errorf("target database does not exist: %s (use db migrate to create new databases)", opts.TargetPath) + } + + return nil +} + +// openSourceDatabase opens the source database for reading +func openSourceDatabase(opts PatchOptions) (dbm.DB, string, error) { + sourceDBPath := filepath.Join(opts.SourceHome, "data", opts.DBName+DbExtension) + sourceDir := filepath.Dir(sourceDBPath) + sourceName := filepath.Base(sourceDBPath) + if len(sourceName) > len(DbExtension) && sourceName[len(sourceName)-len(DbExtension):] == DbExtension { + sourceName = sourceName[:len(sourceName)-len(DbExtension)] + } + + sourceDB, err := dbm.NewDB(sourceName, opts.SourceBackend, sourceDir) + if err != nil { + return nil, "", fmt.Errorf("failed to open source database: %w", err) + } + return sourceDB, sourceDBPath, nil +} + +// openTargetDatabase opens the target database for patching +func openTargetDatabase(opts PatchOptions) (dbm.DB, error) { + var targetDB dbm.DB + var err error + + if opts.TargetBackend == dbm.RocksDBBackend { + targetDB, err = openRocksDBForMigration(opts.TargetPath, opts.RocksDBOptions) + } else { + targetDir := filepath.Dir(opts.TargetPath) + targetName := filepath.Base(opts.TargetPath) + targetName = strings.TrimSuffix(targetName, DbExtension) + targetDB, err = dbm.NewDB(targetName, opts.TargetBackend, targetDir) + } + if err != nil { + return nil, fmt.Errorf("failed to open target database: %w", err) + } + return targetDB, nil +} + +// PatchDatabase patches specific heights from source to target database +func PatchDatabase(opts PatchOptions) (*MigrationStats, error) { + // Validate options + if err := validatePatchOptions(opts); err != nil { + return nil, err + } + + logger := opts.Logger + stats := &MigrationStats{ + StartTime: time.Now(), + } + + // Log dry-run mode if enabled + if opts.DryRun { + logger.Info("DRY RUN MODE - No changes will be made") + if opts.DBName == DBNameBlockstore { + logger.Info("Note: Blockstore patching will also discover and patch corresponding BH: (block header by hash) keys") + } + } + + // Open source database + sourceDB, sourceDBPath, err := openSourceDatabase(opts) + if err != nil { + return stats, err + } + defer sourceDB.Close() + + // Open target database + targetDB, err := openTargetDatabase(opts) + if err != nil { + return stats, err + } + defer targetDB.Close() + + logger.Info("Opening databases for patching", + "source_db", sourceDBPath, + "source_backend", opts.SourceBackend, + "target_db", opts.TargetPath, + "target_backend", opts.TargetBackend, + "height_range", opts.HeightRange.String(), + "dry_run", opts.DryRun, + ) + + // Count keys to patch + totalKeys, err := countKeysForPatch(sourceDB, opts.DBName, opts.HeightRange, logger) + if err != nil { + return stats, fmt.Errorf("failed to count keys: %w", err) + } + stats.TotalKeys.Store(totalKeys) + + if totalKeys == 0 { + logger.Info("No keys found in source database for specified heights", + "database", opts.DBName, + "height_range", opts.HeightRange.String(), + ) + return stats, nil + } + + logger.Info("Starting database patch", + "database", opts.DBName, + "total_keys", totalKeys, + "height_range", opts.HeightRange.String(), + "batch_size", opts.BatchSize, + ) + + // Perform the patch operation + if err := patchDataWithHeightFilter(sourceDB, targetDB, opts, stats); err != nil { + return stats, fmt.Errorf("failed to patch data: %w", err) + } + + // Flush RocksDB if needed + if opts.TargetBackend == dbm.RocksDBBackend { + if err := flushRocksDB(targetDB); err != nil { + logger.Info("Failed to flush RocksDB", "error", err) + } + } + + stats.EndTime = time.Now() + return stats, nil +} + +// countKeysForPatch counts the number of keys to patch based on height range +func countKeysForPatch(db dbm.DB, dbName string, heightRange HeightRange, logger log.Logger) (int64, error) { + var totalCount int64 + + // If we have specific heights, we need to filter while counting + needsFiltering := heightRange.HasSpecificHeights() + + switch dbName { + case DBNameBlockstore: + // For blockstore, count keys from all prefixes + iterators, err := getBlockstoreIterators(db, heightRange) + if err != nil { + return 0, fmt.Errorf("failed to get blockstore iterators: %w", err) + } + + keysSeen := 0 + for iterIdx, it := range iterators { + logger.Debug("Counting keys from blockstore iterator", "iterator_index", iterIdx) + for ; it.Valid(); it.Next() { + keysSeen++ + // Log first few keys to understand the format + if keysSeen <= 5 { + height, hasHeight := extractHeightFromBlockstoreKey(it.Key()) + logger.Debug("Blockstore key found", + "key_prefix", string(it.Key()[:min(10, len(it.Key()))]), + "key_hex", fmt.Sprintf("%x", it.Key()[:min(20, len(it.Key()))]), + "has_height", hasHeight, + "height", height, + "in_range", !needsFiltering || (hasHeight && heightRange.IsWithinRange(height))) + } + if needsFiltering { + // Extract height and check if it's in our specific list + height, hasHeight := extractHeightFromBlockstoreKey(it.Key()) + if hasHeight && !heightRange.IsWithinRange(height) { + continue + } + } + totalCount++ + } + it.Close() + } + logger.Debug("Total keys seen in blockstore", "total_seen", keysSeen, "total_counted", totalCount) + + case DBNameTxIndex: + // For tx_index + it, err := getTxIndexIterator(db, heightRange) + if err != nil { + return 0, fmt.Errorf("failed to get tx_index iterator: %w", err) + } + defer it.Close() + + for ; it.Valid(); it.Next() { + if needsFiltering { + // Extract height and check if it's in our specific list + height, hasHeight := extractHeightFromTxIndexKey(it.Key()) + if hasHeight && !heightRange.IsWithinRange(height) { + continue + } + } + totalCount++ + } + + default: + return 0, fmt.Errorf("unsupported database for height filtering: %s", dbName) + } + + return totalCount, nil +} + +// patchDataWithHeightFilter patches data using height-filtered iterators +func patchDataWithHeightFilter(sourceDB, targetDB dbm.DB, opts PatchOptions, stats *MigrationStats) error { + switch opts.DBName { + case DBNameBlockstore: + return patchBlockstoreData(sourceDB, targetDB, opts, stats) + case DBNameTxIndex: + return patchTxIndexData(sourceDB, targetDB, opts, stats) + default: + return fmt.Errorf("unsupported database for height filtering: %s", opts.DBName) + } +} + +// patchBlockstoreData patches blockstore data +func patchBlockstoreData(sourceDB, targetDB dbm.DB, opts PatchOptions, stats *MigrationStats) error { + // Get bounded iterators for all blockstore prefixes + iterators, err := getBlockstoreIterators(sourceDB, opts.HeightRange) + if err != nil { + return fmt.Errorf("failed to get blockstore iterators: %w", err) + } + + opts.Logger.Info("Patching blockstore data", + "height_range", opts.HeightRange.String(), + "iterator_count", len(iterators), + ) + + // Process each iterator + for idx, it := range iterators { + opts.Logger.Debug("Processing blockstore iterator", "index", idx) + if err := patchWithIterator(it, sourceDB, targetDB, opts, stats); err != nil { + return fmt.Errorf("failed to patch with iterator %d: %w", idx, err) + } + } + + return nil +} + +// extractHeightAndTxIndexFromKey extracts height and txIndex from a tx.height key +// Returns (height, txIndex, success) +func extractHeightAndTxIndexFromKey(key []byte, logger log.Logger) (int64, int64, bool) { + keyStr := string(key) + if !bytes.HasPrefix(key, []byte("tx.height/")) { + return 0, 0, false + } + + // Format: "tx.height///$es$0" or "tx.height///" + parts := strings.Split(keyStr[len("tx.height/"):], "/") + if len(parts) < 3 { + return 0, 0, false + } + + // parts[0] = height (first occurrence) + // parts[1] = height (second occurrence, same value) + // parts[2] = txindex$es$0 OR just txindex + var height, txIndex int64 + _, err := fmt.Sscanf(parts[0], "%d", &height) + if err != nil { + logger.Debug("Failed to parse height from tx.height key", "key", keyStr, "error", err) + return 0, 0, false + } + + // Extract txIndex - handle both with and without "$es$" suffix + txIndexStr := parts[2] + if strings.Contains(txIndexStr, "$es$") { + // Key has "$es$" suffix + txIndexStr = strings.Split(txIndexStr, "$es$")[0] + } + _, err = fmt.Sscanf(txIndexStr, "%d", &txIndex) + if err != nil { + logger.Debug("Failed to parse txIndex from tx.height key", "key", keyStr, "error", err) + return 0, 0, false + } + + return height, txIndex, true +} + +// checkTxHeightKeyConflict checks for key conflicts and returns whether to write +// Returns (shouldWrite, newStrategy, skipped) +func checkTxHeightKeyConflict(key, value []byte, targetDB dbm.DB, currentStrategy ConflictResolution, opts PatchOptions, logger log.Logger) (bool, ConflictResolution, bool) { + if opts.SkipConflictChecks { + return true, currentStrategy, false + } + + existingValue, err := targetDB.Get(key) + if err != nil { + logger.Error("Failed to check existing key", "error", err) + return false, currentStrategy, false + } + + // No conflict if key doesn't exist + if existingValue == nil { + return true, currentStrategy, false + } + + // Handle conflict based on strategy + switch currentStrategy { + case ConflictAsk: + decision, newStrategy, err := promptKeyConflict(key, existingValue, value, opts.DBName, opts.HeightRange) + if err != nil { + logger.Error("Failed to get user input", "error", err) + return false, currentStrategy, false + } + if newStrategy != ConflictAsk { + logger.Info("Conflict resolution strategy updated", "strategy", formatStrategy(newStrategy)) + } + return decision, newStrategy, !decision + + case ConflictSkip: + logger.Debug("Skipping existing key", "key", formatKeyPrefix(key, 80)) + return false, currentStrategy, true + + case ConflictReplace, ConflictReplaceAll: + logger.Debug("Replacing existing key", "key", formatKeyPrefix(key, 80)) + return true, currentStrategy, false + } + + return true, currentStrategy, false +} + +// patchTxHeightKeyAndCollect patches a tx.height key and collects txhash info +// Returns true if batch should be written, false if error occurred +func patchTxHeightKeyAndCollect(key, value []byte, sourceDB dbm.DB, batch dbm.Batch, txhashes *[][]byte, ethTxInfos map[string]EthTxInfo, opts PatchOptions, stats *MigrationStats, logger log.Logger) bool { + // Patch the tx.height key + if opts.DryRun { + logger.Debug("[DRY RUN] Would patch tx.height key", + "key", formatKeyPrefix(key, 80), + "value_preview", formatValue(value, 100), + ) + } else { + if err := batch.Set(key, value); err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to set key in batch", "error", err) + return false + } + logger.Debug("Patched tx.height key", "key", formatKeyPrefix(key, 80)) + } + + // Collect CometBFT txhash for later patching (value is the CometBFT txhash) + if len(value) > 0 { + // Make a copy of the value since iterator reuses memory + txhashCopy := make([]byte, len(value)) + copy(txhashCopy, value) + *txhashes = append(*txhashes, txhashCopy) + + // Extract height and txIndex from the key + height, txIndex, ok := extractHeightAndTxIndexFromKey(key, logger) + if ok { + // Try to collect Ethereum txhash for event-indexed keys + collectEthereumTxInfo(sourceDB, txhashCopy, height, txIndex, ethTxInfos, logger) + } + } + + return true +} + +// collectEthereumTxInfo tries to extract Ethereum txhash from a transaction result +// and stores it in ethTxInfos map if found +func collectEthereumTxInfo(sourceDB dbm.DB, txhash []byte, height, txIndex int64, ethTxInfos map[string]EthTxInfo, logger log.Logger) { + // Read the transaction result from source database + txResultValue, err := sourceDB.Get(txhash) + if err != nil || txResultValue == nil { + return + } + + // Extract ethereum txhash from events + ethTxHash, err := extractEthereumTxHash(txResultValue) + if err != nil { + logger.Debug("Failed to extract ethereum txhash", "error", err, "cometbft_txhash", formatKeyPrefix(txhash, 80)) + return + } + + if ethTxHash != "" { + // Store the info for later Ethereum event key patching + ethTxInfos[ethTxHash] = EthTxInfo{ + Height: height, + TxIndex: txIndex, + } + logger.Debug("Collected ethereum txhash", + "eth_txhash", ethTxHash, + "cometbft_txhash", formatKeyPrefix(txhash, 80), + "height", height, + "tx_index", txIndex, + ) + } +} + +// patchTxHeightKeys patches tx.height keys and collects txhashes and ethereum tx info +// Returns (txhashes, ethTxInfos, currentStrategy, error) +func patchTxHeightKeys(it dbm.Iterator, sourceDB, targetDB dbm.DB, opts PatchOptions, stats *MigrationStats) ([][]byte, map[string]EthTxInfo, ConflictResolution, error) { + logger := opts.Logger + txhashes := make([][]byte, 0, 1000) // Pre-allocate for performance + ethTxInfos := make(map[string]EthTxInfo) // eth_txhash (hex) -> EthTxInfo + batch := targetDB.NewBatch() + defer batch.Close() + + batchCount := 0 + processedCount := int64(0) + skippedCount := int64(0) + currentStrategy := opts.ConflictStrategy + + for it.Valid() { + key := it.Key() + value := it.Value() + + // Additional filtering for specific heights (if needed) + if opts.HeightRange.HasSpecificHeights() { + height, hasHeight := extractHeightFromTxIndexKey(key) + if !hasHeight { + it.Next() + continue + } + if !opts.HeightRange.IsWithinRange(height) { + it.Next() + continue + } + } + + // Check for key conflicts + shouldWrite, newStrategy, skipped := checkTxHeightKeyConflict(key, value, targetDB, currentStrategy, opts, logger) + if newStrategy != currentStrategy { + currentStrategy = newStrategy + } + if skipped { + skippedCount++ + } + if !shouldWrite { + it.Next() + continue + } + + // Patch the key and collect txhash info + if !patchTxHeightKeyAndCollect(key, value, sourceDB, batch, &txhashes, ethTxInfos, opts, stats, logger) { + it.Next() + continue + } + + batchCount++ + processedCount++ + + // Write batch when full + if batchCount >= opts.BatchSize { + if !opts.DryRun { + if err := batch.Write(); err != nil { + return nil, nil, currentStrategy, fmt.Errorf("failed to write batch: %w", err) + } + logger.Debug("Wrote batch", "batch_size", batchCount) + batch.Close() + batch = targetDB.NewBatch() + } + stats.ProcessedKeys.Add(int64(batchCount)) + batchCount = 0 + } + + it.Next() + } + + // Write remaining batch + if batchCount > 0 { + if !opts.DryRun { + if err := batch.Write(); err != nil { + return nil, nil, currentStrategy, fmt.Errorf("failed to write final batch: %w", err) + } + logger.Debug("Wrote final batch", "batch_size", batchCount) + } + stats.ProcessedKeys.Add(int64(batchCount)) + } + + if err := it.Error(); err != nil { + return nil, nil, currentStrategy, fmt.Errorf("iterator error: %w", err) + } + + logger.Info("Patched tx.height keys", + "processed", processedCount, + "skipped", skippedCount, + "txhashes_collected", len(txhashes), + "ethereum_txhashes_collected", len(ethTxInfos), + ) + + return txhashes, ethTxInfos, currentStrategy, nil +} + +func patchTxIndexData(sourceDB, targetDB dbm.DB, opts PatchOptions, stats *MigrationStats) error { + logger := opts.Logger + + // Get bounded iterator for tx_index (only iterates over tx.height// keys) + it, err := getTxIndexIterator(sourceDB, opts.HeightRange) + if err != nil { + return fmt.Errorf("failed to get tx_index iterator: %w", err) + } + defer it.Close() + + logger.Info("Patching tx_index data", + "height_range", opts.HeightRange.String(), + ) + + // Step 1: Patch tx.height keys and collect CometBFT txhashes and Ethereum tx info + txhashes, ethTxInfos, currentStrategy, err := patchTxHeightKeys(it, sourceDB, targetDB, opts, stats) + if err != nil { + return err + } + + // Step 2: Patch CometBFT txhash keys + if len(txhashes) > 0 { + logger.Info("Patching CometBFT txhash lookup keys", "count", len(txhashes)) + if err := patchTxHashKeys(sourceDB, targetDB, txhashes, opts, stats, currentStrategy); err != nil { + return fmt.Errorf("failed to patch txhash keys: %w", err) + } + } + + // Step 3: Patch Ethereum event-indexed keys from source database + if len(ethTxInfos) > 0 { + logger.Info("Patching Ethereum event-indexed keys from source database", "count", len(ethTxInfos)) + if err := patchEthereumEventKeysFromSource(sourceDB, targetDB, ethTxInfos, opts, stats, currentStrategy); err != nil { + return fmt.Errorf("failed to patch ethereum event keys: %w", err) + } + } + + return nil +} + +// patchTxHashKeys patches txhash lookup keys from collected txhashes +func patchTxHashKeys(sourceDB, targetDB dbm.DB, txhashes [][]byte, opts PatchOptions, stats *MigrationStats, currentStrategy ConflictResolution) error { + logger := opts.Logger + batch := targetDB.NewBatch() + defer batch.Close() + + batchCount := 0 + processedCount := int64(0) + skippedCount := int64(0) + + for _, txhash := range txhashes { + // Read txhash value from source + txhashValue, err := sourceDB.Get(txhash) + if err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to read txhash from source", "error", err, "txhash", formatKeyPrefix(txhash, 80)) + continue + } + if txhashValue == nil { + logger.Debug("Txhash key not found in source", "txhash", formatKeyPrefix(txhash, 80)) + continue + } + + // Check for conflicts + shouldWrite := true + if !opts.SkipConflictChecks { + existingValue, err := targetDB.Get(txhash) + if err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to check existing txhash", "error", err) + continue + } + + if existingValue != nil { + switch currentStrategy { + case ConflictSkip: + shouldWrite = false + skippedCount++ + logger.Debug("Skipping existing txhash", "txhash", formatKeyPrefix(txhash, 80)) + + case ConflictReplace, ConflictReplaceAll: + shouldWrite = true + logger.Debug("Replacing existing txhash", "txhash", formatKeyPrefix(txhash, 80)) + + case ConflictAsk: + // Use replace strategy for txhash keys to avoid double-prompting + shouldWrite = true + logger.Debug("Patching txhash (using current strategy)", "txhash", formatKeyPrefix(txhash, 80)) + } + } + } + + if shouldWrite { + if opts.DryRun { + logger.Debug("[DRY RUN] Would patch txhash key", + "txhash", formatKeyPrefix(txhash, 80), + "value_preview", formatValue(txhashValue, 100), + ) + } else { + if err := batch.Set(txhash, txhashValue); err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to set txhash in batch", "error", err) + continue + } + logger.Debug("Patched txhash key", "txhash", formatKeyPrefix(txhash, 80)) + } + + batchCount++ + processedCount++ + + // Write batch when full + if batchCount >= opts.BatchSize { + if !opts.DryRun { + if err := batch.Write(); err != nil { + return fmt.Errorf("failed to write txhash batch: %w", err) + } + logger.Debug("Wrote txhash batch", "batch_size", batchCount) + batch.Close() + batch = targetDB.NewBatch() + } + stats.ProcessedKeys.Add(int64(batchCount)) + batchCount = 0 + } + } + } + + // Write remaining batch + if batchCount > 0 { + if !opts.DryRun { + if err := batch.Write(); err != nil { + return fmt.Errorf("failed to write final txhash batch: %w", err) + } + logger.Debug("Wrote final txhash batch", "batch_size", batchCount) + } + stats.ProcessedKeys.Add(int64(batchCount)) + } + + logger.Info("Patched txhash keys", + "processed", processedCount, + "skipped", skippedCount, + ) + + return nil +} + +// extractEthereumTxHash extracts the Ethereum transaction hash from transaction result events +// Returns the eth txhash (with 0x prefix) if found, empty string otherwise +func extractEthereumTxHash(txResultValue []byte) (string, error) { + // Decode the transaction result + var txResult abci.TxResult + if err := proto.Unmarshal(txResultValue, &txResult); err != nil { + return "", fmt.Errorf("failed to unmarshal tx result: %w", err) + } + + // Look for ethereum_tx event with eth_hash attribute + for _, event := range txResult.Result.Events { + if event.Type == "ethereum_tx" { + for _, attr := range event.Attributes { + if attr.Key == "ethereumTxHash" { + // The value is the Ethereum txhash (with or without 0x prefix) + ethHash := attr.Value + // Ensure 0x prefix is present + if len(ethHash) >= 2 && ethHash[:2] != "0x" { + ethHash = "0x" + ethHash + } + // Validate it's a valid hex hash (should be 66 characters: 0x + 64 hex chars) + if len(ethHash) != 66 { + return "", fmt.Errorf("invalid ethereum txhash length: %d", len(ethHash)) + } + // Decode to verify it's valid hex (skip 0x prefix) + if _, err := hex.DecodeString(ethHash[2:]); err != nil { + return "", fmt.Errorf("invalid ethereum txhash hex: %w", err) + } + return ethHash, nil + } + } + } + } + + // No ethereum_tx event found (this is normal for non-EVM transactions) + return "", nil +} + +// incrementBytes increments a byte slice by 1 to create an exclusive upper bound for iterators +// Returns a new byte slice that is the input + 1 +func incrementBytes(b []byte) []byte { + if len(b) == 0 { + return nil + } + + // Create a copy to avoid modifying the original + incremented := make([]byte, len(b)) + copy(incremented, b) + + // Increment from the last byte, carrying over if necessary + for i := len(incremented) - 1; i >= 0; i-- { + if incremented[i] < 0xFF { + incremented[i]++ + return incremented + } + // If byte is 0xFF, set to 0x00 and continue to carry + incremented[i] = 0x00 + } + + // If all bytes were 0xFF, return nil to signal no exclusive upper bound + return nil +} + +// patchEthereumEventKeysFromSource patches ethereum event-indexed keys by searching source DB +// Key format: "ethereum_tx.ethereumTxHash/0x//$es$" +// +// or "ethereum_tx.ethereumTxHash/0x//" (without $es$ suffix) +// +// Value: CometBFT tx hash (allows lookup by Ethereum txhash) +func patchEthereumEventKeysFromSource(sourceDB, targetDB dbm.DB, ethTxInfos map[string]EthTxInfo, opts PatchOptions, stats *MigrationStats, currentStrategy ConflictResolution) error { + logger := opts.Logger + batch := targetDB.NewBatch() + defer batch.Close() + + batchCount := 0 + processedCount := int64(0) + skippedCount := int64(0) + + // For each Ethereum transaction, create a specific prefix and iterate + for ethTxHash, info := range ethTxInfos { + // Create specific prefix for this transaction to minimize iteration range + // Format: ethereum_tx.ethereumTxHash/0x// + // This will match both keys with and without "$es$" suffix + // Note: ethTxHash already includes the 0x prefix + prefix := fmt.Sprintf("ethereum_tx.ethereumTxHash/%s/%d/%d", ethTxHash, info.Height, info.TxIndex) + prefixBytes := []byte(prefix) + + // Create end boundary by incrementing the prefix (exclusive upper bound) + endBytes := incrementBytes(prefixBytes) + + // Create bounded iterator with [start, end) + it, err := sourceDB.Iterator(prefixBytes, endBytes) + if err != nil { + logger.Error("Failed to create iterator for ethereum event keys", "error", err, "eth_txhash", ethTxHash) + stats.ErrorCount.Add(1) + continue + } + + eventKeysFound := 0 + for it.Valid() { + key := it.Key() + value := it.Value() + + // Stop if we're past the prefix + if !bytes.HasPrefix(key, prefixBytes) { + break + } + + eventKeysFound++ + keyStr := string(key) + + logger.Debug("Found ethereum event key in source", + "event_key", keyStr, + "eth_txhash", ethTxHash, + "height", info.Height, + "tx_index", info.TxIndex, + ) + + // Check for conflicts + shouldWrite := true + if !opts.SkipConflictChecks { + existingValue, err := targetDB.Get(key) + if err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to check existing ethereum event key", "error", err) + it.Next() + continue + } + + if existingValue != nil { + switch currentStrategy { + case ConflictSkip: + shouldWrite = false + skippedCount++ + logger.Debug("Skipping existing ethereum event key", + "event_key", keyStr, + ) + + case ConflictReplace, ConflictReplaceAll: + shouldWrite = true + logger.Debug("Replacing existing ethereum event key", + "event_key", keyStr, + ) + + case ConflictAsk: + // Use replace strategy for event keys to avoid excessive prompting + shouldWrite = true + logger.Debug("Patching ethereum event key (using current strategy)", + "event_key", keyStr, + ) + } + } + } + + if shouldWrite { + // Make a copy of the value since iterator reuses memory + valueCopy := make([]byte, len(value)) + copy(valueCopy, value) + + if opts.DryRun { + logger.Debug("[DRY RUN] Would patch ethereum event key", + "event_key", keyStr, + "value_preview", formatKeyPrefix(valueCopy, 80), + ) + } else { + if err := batch.Set(key, valueCopy); err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to set ethereum event key in batch", "error", err) + it.Next() + continue + } + logger.Debug("Patched ethereum event key", + "event_key", keyStr, + "value_preview", formatKeyPrefix(valueCopy, 80), + ) + } + + batchCount++ + processedCount++ + + // Write batch when full + if batchCount >= opts.BatchSize { + if !opts.DryRun { + if err := batch.Write(); err != nil { + it.Close() + return fmt.Errorf("failed to write ethereum event batch: %w", err) + } + logger.Debug("Wrote ethereum event batch", "batch_size", batchCount) + batch.Close() + batch = targetDB.NewBatch() + } + stats.ProcessedKeys.Add(int64(batchCount)) + batchCount = 0 + } + } + + it.Next() + } + + if err := it.Error(); err != nil { + it.Close() + return fmt.Errorf("iterator error for eth_txhash %s: %w", ethTxHash, err) + } + + it.Close() + + if eventKeysFound > 0 { + logger.Debug("Processed event keys for transaction", + "eth_txhash", ethTxHash, + "event_keys_found", eventKeysFound, + ) + } + } + + // Write remaining batch + if batchCount > 0 { + if !opts.DryRun { + if err := batch.Write(); err != nil { + return fmt.Errorf("failed to write final ethereum event batch: %w", err) + } + logger.Debug("Wrote final ethereum event batch", "batch_size", batchCount) + } + stats.ProcessedKeys.Add(int64(batchCount)) + } + + logger.Info("Patched ethereum event keys from source database", + "processed", processedCount, + "skipped", skippedCount, + ) + + return nil +} + +// patchWithIterator patches data from an iterator to target database +// shouldProcessKey checks if a key should be processed based on height filtering +func shouldProcessKey(key []byte, dbName string, heightRange HeightRange) bool { + if !heightRange.HasSpecificHeights() { + return true + } + + // Extract height from key + var height int64 + var hasHeight bool + + switch dbName { + case DBNameBlockstore: + height, hasHeight = extractHeightFromBlockstoreKey(key) + case DBNameTxIndex: + height, hasHeight = extractHeightFromTxIndexKey(key) + default: + return false + } + + if !hasHeight { + return false + } + + return heightRange.IsWithinRange(height) +} + +// handleKeyConflict handles key conflict resolution +// Returns (shouldWrite bool, newStrategy ConflictResolution, skipped bool) +func handleKeyConflict(key, existingValue, newValue []byte, targetDB dbm.DB, currentStrategy ConflictResolution, opts PatchOptions, logger log.Logger) (bool, ConflictResolution, bool) { + if opts.SkipConflictChecks { + return true, currentStrategy, false + } + + // Key doesn't exist, no conflict + if existingValue == nil { + return true, currentStrategy, false + } + + // log the existing value and key + logger.Debug("Existing key", + "key", formatKeyPrefix(key, 80), + "existing_value_preview", formatValue(existingValue, 100), + ) + + // Handle conflict based on strategy + switch currentStrategy { + case ConflictAsk: + decision, newStrategy, err := promptKeyConflict(key, existingValue, newValue, opts.DBName, opts.HeightRange) + if err != nil { + logger.Error("Failed to get user input", "error", err) + return false, currentStrategy, true + } + if newStrategy != ConflictAsk { + logger.Info("Conflict resolution strategy updated", "strategy", formatStrategy(newStrategy)) + } + return decision, newStrategy, !decision + + case ConflictSkip: + logger.Debug("Skipping existing key", + "key", formatKeyPrefix(key, 80), + "existing_value_preview", formatValue(existingValue, 100), + ) + return false, currentStrategy, true + + case ConflictReplace, ConflictReplaceAll: + logger.Debug("Replacing existing key", + "key", formatKeyPrefix(key, 80), + "old_value_preview", formatValue(existingValue, 100), + "new_value_preview", formatValue(newValue, 100), + ) + return true, currentStrategy, false + } + + return true, currentStrategy, false +} + +// patchSingleKey patches a single key-value pair, including BH: key for blockstore H: keys +func patchSingleKey(key, value []byte, sourceDB dbm.DB, batch dbm.Batch, opts PatchOptions, logger log.Logger) error { + if opts.DryRun { + // Debug log for what would be patched + logger.Debug("[DRY RUN] Would patch key", + "key", formatKeyPrefix(key, 80), + "key_size", len(key), + "value_preview", formatValue(value, 100), + "value_size", len(value), + ) + + // For blockstore H: keys, check if corresponding BH: key would be patched + if opts.DBName == DBNameBlockstore && len(key) > 2 && key[0] == 'H' && key[1] == ':' { + if blockHash, ok := extractBlockHashFromMetadata(value); ok { + // Check if BH: key exists in source DB + bhKey := make([]byte, 3+len(blockHash)) + copy(bhKey[0:3], []byte("BH:")) + copy(bhKey[3:], blockHash) + + bhValue, err := sourceDB.Get(bhKey) + if err == nil && bhValue != nil { + logger.Debug("[DRY RUN] Would patch BH: key", + "hash", fmt.Sprintf("%x", blockHash), + "key_size", len(bhKey), + "value_size", len(bhValue), + ) + } + } + } + return nil + } + + // Copy key-value to batch (actual write) + if err := batch.Set(key, value); err != nil { + return fmt.Errorf("failed to set key in batch: %w", err) + } + + // For blockstore H: keys, also patch the corresponding BH: key + if opts.DBName == DBNameBlockstore && len(key) > 2 && key[0] == 'H' && key[1] == ':' { + if blockHash, ok := extractBlockHashFromMetadata(value); ok { + // Construct BH: key + bhKey := make([]byte, 3+len(blockHash)) + copy(bhKey[0:3], []byte("BH:")) + copy(bhKey[3:], blockHash) + + // Try to get the value from source DB + bhValue, err := sourceDB.Get(bhKey) + if err == nil && bhValue != nil { + // Make a copy of the value before adding to batch + bhValueCopy := make([]byte, len(bhValue)) + copy(bhValueCopy, bhValue) + + if err := batch.Set(bhKey, bhValueCopy); err != nil { + logger.Debug("Failed to patch BH: key", "error", err, "hash", fmt.Sprintf("%x", blockHash)) + } else { + logger.Debug("Patched BH: key", "hash", fmt.Sprintf("%x", blockHash)) + } + } + } + } + + // Debug log for each key patched + logger.Debug("Patched key to target database", + "key", formatKeyPrefix(key, 80), + "key_size", len(key), + "value_preview", formatValue(value, 100), + "value_size", len(value), + ) + + return nil +} + +// writeAndResetBatch writes the batch to the database and creates a new batch +func writeAndResetBatch(batch dbm.Batch, targetDB dbm.DB, batchCount int, opts PatchOptions, logger log.Logger) (dbm.Batch, error) { + if opts.DryRun { + logger.Debug("[DRY RUN] Would write batch", "batch_size", batchCount) + return batch, nil + } + + logger.Debug("Writing batch to target database", "batch_size", batchCount) + if err := batch.Write(); err != nil { + return batch, fmt.Errorf("failed to write batch: %w", err) + } + + // Close and create new batch + batch.Close() + return targetDB.NewBatch(), nil +} + +func patchWithIterator(it dbm.Iterator, sourceDB, targetDB dbm.DB, opts PatchOptions, stats *MigrationStats) error { + defer it.Close() + + logger := opts.Logger + batch := targetDB.NewBatch() + defer batch.Close() + + batchCount := 0 + skippedCount := int64(0) + lastLogTime := time.Now() + const logInterval = 5 * time.Second + + // Track current conflict resolution strategy (may change during execution) + currentStrategy := opts.ConflictStrategy + + for ; it.Valid(); it.Next() { + key := it.Key() + value := it.Value() + + // Check if we should process this key (height filtering) + if !shouldProcessKey(key, opts.DBName, opts.HeightRange) { + continue + } + + // Check for key conflicts and get resolution + existingValue, err := targetDB.Get(key) + if err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to check existing key", "error", err) + continue + } + + shouldWrite, newStrategy, skipped := handleKeyConflict(key, existingValue, value, targetDB, currentStrategy, opts, logger) + if newStrategy != currentStrategy { + currentStrategy = newStrategy + } + if skipped { + skippedCount++ + } + if !shouldWrite { + continue + } + + // Patch the key-value pair + if err := patchSingleKey(key, value, sourceDB, batch, opts, logger); err != nil { + stats.ErrorCount.Add(1) + logger.Error("Failed to patch key", "error", err) + continue + } + + batchCount++ + + // Write batch when it reaches the batch size + if batchCount >= opts.BatchSize { + batch, err = writeAndResetBatch(batch, targetDB, batchCount, opts, logger) + if err != nil { + return err + } + stats.ProcessedKeys.Add(int64(batchCount)) + batchCount = 0 + } + + // Periodic logging + if time.Since(lastLogTime) >= logInterval { + progress := float64(stats.ProcessedKeys.Load()) / float64(stats.TotalKeys.Load()) * 100 + logger.Info("Patching progress", + "processed", stats.ProcessedKeys.Load(), + "skipped", skippedCount, + "total", stats.TotalKeys.Load(), + "progress", fmt.Sprintf("%.2f%%", progress), + "errors", stats.ErrorCount.Load(), + ) + lastLogTime = time.Now() + } + } + + // Write remaining batch + if batchCount > 0 { + if opts.DryRun { + logger.Debug("[DRY RUN] Would write final batch", "batch_size", batchCount) + } else { + if err := batch.Write(); err != nil { + return fmt.Errorf("failed to write final batch: %w", err) + } + } + stats.ProcessedKeys.Add(int64(batchCount)) + } + + // Final logging + if skippedCount > 0 { + logger.Info("Skipped conflicting keys", "count", skippedCount) + } + if opts.DryRun { + logger.Info("[DRY RUN] Simulation complete - no changes were made") + } + + if err := it.Error(); err != nil { + return fmt.Errorf("iterator error: %w", err) + } + + return nil +} + +// promptKeyConflict prompts the user to decide what to do with a conflicting key +// Returns: (shouldWrite bool, newStrategy ConflictResolution, error) +func promptKeyConflict(key, existingValue, newValue []byte, dbName string, heightRange HeightRange) (bool, ConflictResolution, error) { + // Extract height if possible for display + var heightStr string + switch dbName { + case DBNameBlockstore: + if height, ok := extractHeightFromBlockstoreKey(key); ok { + heightStr = fmt.Sprintf(" (height: %d)", height) + } + case DBNameTxIndex: + if height, ok := extractHeightFromTxIndexKey(key); ok { + heightStr = fmt.Sprintf(" (height: %d)", height) + } + } + + // Display key information + fmt.Println("\n" + strings.Repeat("=", 80)) + fmt.Println("KEY CONFLICT DETECTED") + fmt.Println(strings.Repeat("=", 80)) + fmt.Printf("Database: %s\n", dbName) + fmt.Printf("Key: %s%s\n", formatKeyPrefix(key, 40), heightStr) + fmt.Printf("Existing size: %d bytes\n", len(existingValue)) + fmt.Printf("New size: %d bytes\n", len(newValue)) + fmt.Println(strings.Repeat("-", 80)) + + // Prompt for action + reader := bufio.NewReader(os.Stdin) + for { + fmt.Print("Action? [(r)eplace, (s)kip, (R)eplace all, (S)kip all]: ") + input, err := reader.ReadString('\n') + if err != nil { + return false, ConflictAsk, fmt.Errorf("failed to read input: %w", err) + } + + input = strings.TrimSpace(input) + inputLower := strings.ToLower(input) + + switch { + case input == "R": + return true, ConflictReplaceAll, nil + case input == "S": + return false, ConflictSkip, nil + case inputLower == "r" || inputLower == "replace": + return true, ConflictAsk, nil + case inputLower == "s" || inputLower == "skip": + return false, ConflictAsk, nil + default: + fmt.Println("Invalid input. Please enter r, s, R, or S.") + } + } +} + +// formatKeyPrefix formats a key for display, truncating if necessary +// Detects binary data (like txhashes) and formats as hex +func formatKeyPrefix(key []byte, maxLen int) string { + if len(key) == 0 { + return "" + } + + // Check if key is mostly printable ASCII (heuristic for text vs binary) + printableCount := 0 + for _, b := range key { + if (b >= 32 && b <= 126) || b == 9 || b == 10 || b == 13 || b == '/' || b == ':' { + printableCount++ + } + } + + // If more than 80% is printable, treat as text (e.g., "tx.height/123/0") + if float64(printableCount)/float64(len(key)) > 0.8 { + if len(key) <= maxLen { + return string(key) + } + return string(key[:maxLen]) + "..." + } + + // Otherwise, format as hex (e.g., txhashes) + hexStr := fmt.Sprintf("%x", key) + if len(hexStr) <= maxLen { + return "0x" + hexStr + } + // Truncate hex string if too long + halfLen := (maxLen - 8) / 2 // Reserve space for "0x" and "..." + if maxLen <= 8 || halfLen <= 0 { + // Not enough space for "0x..."; just truncate what we can + if maxLen <= 2 { + return "0x" + } + // Truncate to maxLen-2 to account for "0x" prefix + truncLen := maxLen - 2 + if truncLen > len(hexStr) { + truncLen = len(hexStr) + } + return "0x" + hexStr[:truncLen] + } + return "0x" + hexStr[:halfLen] + "..." + hexStr[len(hexStr)-halfLen:] +} + +// formatValue formats a value for display +// If the value appears to be binary data, it shows a hex preview +// Otherwise, it shows the string representation +func formatValue(value []byte, maxLen int) string { + if len(value) == 0 { + return "" + } + + // Check if value is mostly printable ASCII (heuristic for text vs binary) + printableCount := 0 + for _, b := range value { + if (b >= 32 && b <= 126) || b == 9 || b == 10 || b == 13 { + printableCount++ + } + } + + // If more than 80% is printable, treat as text + if float64(printableCount)/float64(len(value)) > 0.8 { + if len(value) <= maxLen { + return string(value) + } + return string(value[:maxLen]) + fmt.Sprintf("... (%d more bytes)", len(value)-maxLen) + } + + // Otherwise, show as hex + hexPreview := fmt.Sprintf("%x", value) + if len(hexPreview) <= maxLen { + return "0x" + hexPreview + } + return "0x" + hexPreview[:maxLen] + fmt.Sprintf("... (%d total bytes)", len(value)) +} + +// formatStrategy returns a human-readable string for a conflict resolution strategy +func formatStrategy(strategy ConflictResolution) string { + switch strategy { + case ConflictAsk: + return "ask" + case ConflictSkip: + return "skip all" + case ConflictReplace: + return "replace" + case ConflictReplaceAll: + return "replace all" + default: + return "unknown" + } +} diff --git a/cmd/cronosd/dbmigrate/patch_test.go b/cmd/cronosd/dbmigrate/patch_test.go new file mode 100644 index 0000000000..0c9e35133c --- /dev/null +++ b/cmd/cronosd/dbmigrate/patch_test.go @@ -0,0 +1,114 @@ +//go:build !rocksdb +// +build !rocksdb + +package dbmigrate + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIncrementBytes tests the byte slice increment helper +func TestIncrementBytes(t *testing.T) { + tests := []struct { + name string + input []byte + expected []byte + }{ + { + name: "simple_increment", + input: []byte{0x01, 0x02, 0x03}, + expected: []byte{0x01, 0x02, 0x04}, + }, + { + name: "carry_over", + input: []byte{0x01, 0x02, 0xFF}, + expected: []byte{0x01, 0x03, 0x00}, + }, + { + name: "all_ff", + input: []byte{0xFF, 0xFF, 0xFF}, + expected: nil, // Returns nil to signal no upper bound for iterators + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := incrementBytes(tt.input) + require.Equal(t, tt.expected, result) + }) + } +} + +// TestFormatKeyPrefix tests the key prefix formatting helper +func TestFormatKeyPrefix(t *testing.T) { + tests := []struct { + name string + input []byte + maxLen int + contains string + }{ + { + name: "ascii_text", + input: []byte("test-key-123"), + maxLen: 20, + contains: "test-key-123", + }, + { + name: "binary_data", + input: []byte{0x01, 0x02, 0xFF, 0xFE}, + maxLen: 20, + contains: "0x", + }, + { + name: "truncated", + input: []byte("this is a very long key that should be truncated"), + maxLen: 10, + contains: "...", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := formatKeyPrefix(tt.input, tt.maxLen) + require.Contains(t, result, tt.contains) + }) + } +} + +// TestFormatValue tests the value formatting helper +func TestFormatValue(t *testing.T) { + tests := []struct { + name string + input []byte + maxLen int + contains string + }{ + { + name: "ascii_text", + input: []byte("test value"), + maxLen: 20, + contains: "test value", + }, + { + name: "binary_data", + input: []byte{0x01, 0x02, 0xFF, 0xFE}, + maxLen: 20, + contains: "0x", + }, + { + name: "empty_value", + input: []byte{}, + maxLen: 20, + contains: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := formatValue(tt.input, tt.maxLen) + require.Contains(t, result, tt.contains) + }) + } +} diff --git a/cmd/cronosd/dbmigrate/swap-migrated-db.sh b/cmd/cronosd/dbmigrate/swap-migrated-db.sh new file mode 100755 index 0000000000..34174013cc --- /dev/null +++ b/cmd/cronosd/dbmigrate/swap-migrated-db.sh @@ -0,0 +1,352 @@ +#!/bin/bash + +# Database Migration Swap Script +# This script replaces original databases with migrated ones and backs up the originals + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to validate backup suffix +validate_backup_suffix() { + local suffix="$1" + local sanitized + + # Remove any characters not in the safe set [A-Za-z0-9._-] + sanitized=$(echo "$suffix" | tr -cd 'A-Za-z0-9._-') + + # Check if empty after sanitization + if [[ -z "$sanitized" ]]; then + echo -e "${RED}Error: BACKUP_SUFFIX is empty or contains only invalid characters${NC}" >&2 + echo -e "${RED}Allowed characters: A-Z, a-z, 0-9, period (.), underscore (_), hyphen (-)${NC}" >&2 + exit 1 + fi + + # Check if sanitized version differs from original (contains disallowed characters) + if [[ "$suffix" != "$sanitized" ]]; then + echo -e "${RED}Error: BACKUP_SUFFIX contains invalid characters: '$suffix'${NC}" >&2 + echo -e "${RED}Allowed characters: A-Z, a-z, 0-9, period (.), underscore (_), hyphen (-)${NC}" >&2 + echo -e "${RED}Sanitized version would be: '$sanitized'${NC}" >&2 + exit 1 + fi + + # Return success if validation passed + return 0 +} + +# Default values +HOME_DIR="$HOME/.cronos" +DB_TYPE="app" +BACKUP_SUFFIX="backup-$(date +%Y%m%d-%H%M%S)" + +# Validate default BACKUP_SUFFIX immediately after construction +validate_backup_suffix "$BACKUP_SUFFIX" + +DRY_RUN=false + +# Usage function +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Swap migrated databases with originals and create backups. + +OPTIONS: + --home DIR Node home directory (default: ~/.cronos) + --db-type TYPE Database type: app, cometbft, or all (default: app) + --backup-suffix STR Backup suffix (default: backup-YYYYMMDD-HHMMSS) + --dry-run Show what would be done without doing it + -h, --help Show this help message + +EXAMPLES: + # Swap application database + $0 --home ~/.cronos --db-type app + + # Swap all CometBFT databases + $0 --db-type cometbft + + # Swap all databases with custom backup name + $0 --db-type all --backup-suffix before-rocksdb + + # Preview changes without executing + $0 --db-type all --dry-run + +EOF + exit 1 +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --home) + HOME_DIR="$2" + shift 2 + ;; + --db-type) + DB_TYPE="$2" + shift 2 + ;; + --backup-suffix) + BACKUP_SUFFIX="$2" + # Validate user-provided BACKUP_SUFFIX immediately + validate_backup_suffix "$BACKUP_SUFFIX" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + ;; + *) + echo -e "${RED}Error: Unknown option $1${NC}" + usage + ;; + esac +done + +# Validate db-type +if [[ "$DB_TYPE" != "app" && "$DB_TYPE" != "cometbft" && "$DB_TYPE" != "all" ]]; then + echo -e "${RED}Error: Invalid db-type '$DB_TYPE'. Must be: app, cometbft, or all${NC}" + exit 1 +fi + +# Validate home directory +if [[ ! -d "$HOME_DIR" ]]; then + echo -e "${RED}Error: Home directory does not exist: $HOME_DIR${NC}" + exit 1 +fi + +DATA_DIR="$HOME_DIR/data" +if [[ ! -d "$DATA_DIR" ]]; then + echo -e "${RED}Error: Data directory does not exist: $DATA_DIR${NC}" + exit 1 +fi + +# Determine which databases to swap +declare -a DB_NAMES +case "$DB_TYPE" in + app) + DB_NAMES=("application") + ;; + cometbft) + DB_NAMES=("blockstore" "state" "tx_index" "evidence") + ;; + all) + DB_NAMES=("application" "blockstore" "state" "tx_index" "evidence") + ;; +esac + +# Function to print colored output +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to get directory size +get_size() { + if [[ -d "$1" ]]; then + du -sh "$1" 2>/dev/null | awk '{print $1}' + else + echo "N/A" + fi +} + +# Check for migrated databases +print_info "Checking for migrated databases..." +FOUND_MIGRATED=false +declare -a AVAILABLE_DBS + +for db_name in "${DB_NAMES[@]}"; do + # Unified path format for all backends: application.migrate-temp.db + migrated_db="$DATA_DIR/${db_name}.migrate-temp.db" + + if [[ -d "$migrated_db" ]]; then + FOUND_MIGRATED=true + AVAILABLE_DBS+=("$db_name") + print_info " ✓ Found: ${db_name}.migrate-temp.db ($(get_size "$migrated_db"))" + else + print_warning " ✗ Not found: ${db_name}.migrate-temp.db" + fi +done + +if [[ "$FOUND_MIGRATED" == false ]]; then + print_error "No migrated databases found in $DATA_DIR" + print_info "Run the migration first: cronosd db migrate --db-type $DB_TYPE" + exit 1 +fi + +echo "" +print_info "Database type: $DB_TYPE" +print_info "Home directory: $HOME_DIR" +print_info "Data directory: $DATA_DIR" +print_info "Backup suffix: $BACKUP_SUFFIX" +if [[ "$DRY_RUN" == true ]]; then + print_warning "DRY RUN MODE - No changes will be made" +fi + +# Create backup directory (skip in dry run to avoid side effects) +BACKUP_DIR="$DATA_DIR/backups-$BACKUP_SUFFIX" +if [[ "$DRY_RUN" == false ]]; then + if ! mkdir -p "$BACKUP_DIR"; then + print_error "Failed to create backup directory: $BACKUP_DIR" + exit 1 + fi +fi + +# Initialize counters +SUCCESS_COUNT=0 + +echo "" +echo "================================================================================" +echo "MIGRATION SWAP PLAN" +echo "================================================================================" + +for db_name in "${AVAILABLE_DBS[@]}"; do + original_db="$DATA_DIR/${db_name}.db" + migrated_db="$DATA_DIR/${db_name}.migrate-temp.db" + backup_db="$BACKUP_DIR/${db_name}.db" + + echo "" + echo "Database: $db_name" + echo " Original: $original_db ($(get_size "$original_db"))" + echo " Migrated: $migrated_db ($(get_size "$migrated_db"))" + echo " Backup: $backup_db" +done + +echo "" +echo "================================================================================" + +# Confirmation for non-dry-run +if [[ "$DRY_RUN" == false ]]; then + echo "" + print_warning "This will:" + echo " 1. Move original databases to: $BACKUP_DIR" + echo " 2. Replace with migrated databases" + echo "" + read -p "Continue? (yes/no): " -r + echo "" + if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then + print_info "Aborted by user" + exit 0 + fi +fi + +echo "" +print_info "Starting database swap..." +echo "" + +# Perform the swap +for db_name in "${AVAILABLE_DBS[@]}"; do + echo "" + print_info "Processing: $db_name" + + original_db="$DATA_DIR/${db_name}.db" + migrated_db="$DATA_DIR/${db_name}.migrate-temp.db" + backup_db="$BACKUP_DIR/${db_name}.db" + + # Check if original exists + if [[ ! -d "$original_db" ]]; then + print_warning " Original database not found, skipping backup: $original_db" + ORIGINAL_EXISTS=false + else + ORIGINAL_EXISTS=true + fi + + # Move original to backup if it exists + if [[ "$ORIGINAL_EXISTS" == true ]]; then + if [[ "$DRY_RUN" == false ]]; then + print_info " Moving original to backup..." + mv "$original_db" "$backup_db" + print_success " ✓ Moved to backup: $original_db → $backup_db" + else + print_info " [DRY RUN] Would move to backup: $original_db → $backup_db" + fi + fi + + # Move migrated to original location + if [[ "$DRY_RUN" == false ]]; then + print_info " Installing migrated database..." + mv "$migrated_db" "$original_db" + print_success " ✓ Moved: $migrated_db → $original_db" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + else + print_info " [DRY RUN] Would move: $migrated_db → $original_db" + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + fi +done + +echo "" +echo "================================================================================" +if [[ "$DRY_RUN" == false ]]; then + echo -e "${GREEN}DATABASE SWAP COMPLETED SUCCESSFULLY${NC}" +else + echo -e "${YELLOW}DRY RUN COMPLETED${NC}" +fi +echo "================================================================================" + +if [[ "$DRY_RUN" == false ]]; then +echo "" +echo "Summary:" +echo " Databases swapped: $SUCCESS_COUNT" +echo " Backups location: $BACKUP_DIR" +echo "" +echo "Note: Original databases were moved (not copied) to backup location." +echo " This is faster and saves disk space." +echo "" +echo "Next steps:" +echo " 1. Update your configuration files:" + + if [[ "$DB_TYPE" == "app" || "$DB_TYPE" == "all" ]]; then + echo " - Edit ~/.cronos/config/app.toml" + echo " Change: app-db-backend = \"rocksdb\" # or your target backend" + fi + + if [[ "$DB_TYPE" == "cometbft" || "$DB_TYPE" == "all" ]]; then + echo " - Edit ~/.cronos/config/config.toml" + echo " Change: db_backend = \"rocksdb\" # or your target backend" + fi + + echo "" + echo " 2. Start your node:" + echo " systemctl start cronosd" + echo " # or" + echo " cronosd start --home $HOME_DIR" + echo "" + echo " 3. Monitor the logs to ensure everything works correctly" + echo "" + echo " 4. If everything works, you can remove the backups:" + echo " rm -rf $BACKUP_DIR" + echo "" +else + echo "" + echo "This was a dry run. No changes were made." + echo "Run without --dry-run to perform the actual swap." + echo "" +fi + +# List data directory +echo "" +print_info "Current data directory contents:" +ls -lh "$DATA_DIR" | grep -E "^d" | awk '{print " " $9 " (" $5 ")"}' + +echo "" +print_success "Script completed" +