go/oasis-node/cmd/storage: Bootstrap off-chain checkpoint sync

Author: martintomazic

go/oasis-node/cmd/storage/checkpoint.go

@@ -8,13 +8,18 @@ import (
 	"path/filepath"
 
 	cmtCfg "github.com/cometbft/cometbft/config"
-	cmtNode "github.com/cometbft/cometbft/node"
+	cmtProtoState "github.com/cometbft/cometbft/proto/tendermint/state"
+	cmtProto "github.com/cometbft/cometbft/proto/tendermint/types"
+	cmtState "github.com/cometbft/cometbft/state"
+	"github.com/cometbft/cometbft/store"
+	"github.com/cometbft/cometbft/types"
+	cmttypes "github.com/cometbft/cometbft/types"
+	"github.com/cometbft/cometbft/version"
+	"github.com/cosmos/gogoproto/proto"
 	"github.com/spf13/cobra"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
-	"github.com/oasisprotocol/oasis-core/go/config"
-	cmtCommon "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/common"
-	cmtDB "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/db"
+	"github.com/oasisprotocol/oasis-core/go/common/cbor"
 	cmdCommon "github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common"
 	"github.com/oasisprotocol/oasis-core/go/storage/mkvs/checkpoint"
 	"github.com/oasisprotocol/oasis-core/go/storage/mkvs/db/api"
@@ -24,8 +29,15 @@ import (
 const (
 	consensusSubdir = "consensus"
 	runtimesSubdir  = "runtimes"
+
+	consensusMetaFilename = "bootstrap.cbor"
 )
 
+type bootstrapMeta struct {
+	State  []byte `json:"state"`
+	Commit []byte `json:"commit"`
+}
+
 func newCheckpointCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "checkpoint",
@@ -72,7 +84,13 @@ func newCreateCmd() *cobra.Command {
 				defer close()
 
 				consensusOutDir := filepath.Join(outDir, consensusSubdir)
-				return createConsensusCheckpoint(cmd.Context(), ndb, height, consensusOutDir)
+				if err := createConsensusCheckpoint(cmd.Context(), ndb, height, consensusOutDir); err != nil {
+					return err
+				}
+				if err := writeConsensusBootstrap(cmd.Context(), cmdCommon.DataDir(), ndb, height, consensusOutDir); err != nil {
+					return fmt.Errorf("failed to write bootstrap metadata: %w", err)
+				}
+				return nil
 			}
 
 			createRuntimeCps := func() error {
@@ -182,8 +200,8 @@ func createRuntimeCheckpoints(ctx context.Context, ndb api.NodeDB, round uint64,
 	if err != nil {
 		return fmt.Errorf("failed to get roots for round %d: %w", round, err)
 	}
-	if lenRoots := len(roots); lenRoots != 2 {
-		return fmt.Errorf("unexpected number of roots: got %d, want %d", lenRoots, 2)
+	if lenRoots := len(roots); 0 == lenRoots || lenRoots > 2 { // Empty IO root is implicitly present.
+		return fmt.Errorf("unexpected number of roots: got %d", lenRoots)
 	}
 	return createCheckpoints(ctx, ndb, roots, outputDir)
 }
@@ -204,6 +222,67 @@ func createCheckpoints(ctx context.Context, ndb api.NodeDB, roots []node.Root, o
 	return nil
 }
 
+func writeConsensusBootstrap(ctx context.Context, dataDir string, ndb api.NodeDB, height uint64, outputDir string) error {
+
+	stateStore, err := openConsensusStatestore(dataDir)
+	if err != nil {
+		return fmt.Errorf("failed to open cometbft state store: %w", err)
+	}
+	defer stateStore.Close()
+
+	blockStore, err := openConsensusBlockstore(dataDir)
+	if err != nil {
+		return fmt.Errorf("failed to open consensus blockstore: %w", err)
+	}
+	defer blockStore.Close()
+
+	state, err := State(ctx, height, stateStore, blockStore)
+	if err != nil {
+		return fmt.Errorf("failed to load consensus state at height %d: %w", height, err)
+	}
+	statePB, err := state.ToProto()
+	if err != nil {
+		return fmt.Errorf("failed to convert consensus state to proto: %w", err)
+	}
+	stateBytes, err := proto.Marshal(statePB)
+	if err != nil {
+		return fmt.Errorf("failed to marshal consensus state: %w", err)
+	}
+
+	commit, err := Commit(ctx, blockStore, height)
+	if err != nil {
+		return fmt.Errorf("failed to load consensus commit at height %d: %w", height, err)
+	}
+	commitBytes, err := proto.Marshal(commit.ToProto())
+	if err != nil {
+		return fmt.Errorf("failed to marshal consensus commit: %w", err)
+	}
+
+	meta := bootstrapMeta{
+		State:  stateBytes,
+		Commit: commitBytes,
+	}
+	if err := os.WriteFile(filepath.Join(outputDir, consensusMetaFilename), cbor.Marshal(meta), 0o600); err != nil {
+		return fmt.Errorf("failed to write bootstrap metadata: %w", err)
+	}
+
+	return nil
+}
+
+func readConsensusBootstrap(inputDir string) (bootstrapMeta, error) {
+	data, err := os.ReadFile(filepath.Join(inputDir, consensusMetaFilename))
+	if err != nil {
+		return bootstrapMeta{}, err
+	}
+
+	var meta bootstrapMeta
+	if err := cbor.Unmarshal(data, &meta); err != nil {
+		return bootstrapMeta{}, fmt.Errorf("failed to decode bootstrap metadata: %w", err)
+	}
+
+	return meta, nil
+}
+
 func restoreConsensusCp(ctx context.Context, dataDir, inputDir string) error {
 	ndb, close, err := openConsensusNodeDB(cmdCommon.DataDir())
 	if err != nil {
@@ -233,31 +312,12 @@ func restoreConsensusCp(ctx context.Context, dataDir, inputDir string) error {
 		return fmt.Errorf("failed to restore checkpoint: %w", err)
 	}
 
-	// TODO: This is just to show how CometBFT is meant to synchronize block and state stores
-	// when the state sync is done offline (aka import checkpoint). Obviously we use our custom
-	// genesis/doc provider and light clients. In theory this shows we could make bootstrap of
-	// oasis node from the checkpoints completely trustless!
-	//
-	// In practice I plan to write our own version of `BootstrapUntrustedState`, where the node
-	// creating checkpoints, so also dump bootstrap metadata.
-	cmtConfig := cmtCfg.DefaultConfig()
-	cmtConfig.SetRoot(filepath.Join(cmdCommon.DataDir(), cmtCommon.StateDir))
-	dbProvider, err := cmtDB.Provider()
+	meta, err := readConsensusBootstrap(inputDir)
 	if err != nil {
-		return fmt.Errorf("failed to obtain db provider: %w", err)
-	}
-	cmtConfig.Genesis = config.GlobalConfig.Genesis.File
-	cmtConfig.StateSync.RPCServers = config.GlobalConfig.Consensus.Providers
-	cmtConfig.StateSync.TrustPeriod = config.GlobalConfig.Consensus.LightClient.Trust.Period
-	cmtConfig.StateSync.TrustHeight = int64(config.GlobalConfig.Consensus.LightClient.Trust.Height)
-	cmtConfig.StateSync.TrustHash = config.GlobalConfig.Consensus.LightClient.Trust.Hash
-
-	root := cps[0].Root
-	if err := cmtNode.BootstrapState(ctx, cmtConfig, dbProvider, root.Version, root.Hash[:]); err != nil {
-		return fmt.Errorf("failed to bootstrap cometbft dbs from the state DB: %w", err)
+		return fmt.Errorf("failed to read bootstrap metadata: %w", err)
 	}
 
-	return nil
+	return bootstrapTrustedState(ctx, dataDir, meta)
 }
 
 func restoreRuntimeCps(ctx context.Context, inputDir, namespace string) error {
@@ -322,3 +382,167 @@ func restoreCheckpoints(ctx context.Context, provider checkpoint.ChunkProvider,
 
 	return nil
 }
+
+// bootstrapTrustedState synchronizes the cometbft databases after the state sync
+// has been performed offline.
+//
+// It is expected that the block store and state store are empty at the time the
+// function is called.
+//
+// Adapted from https://github.com/oasisprotocol/cometbft/blob/08e22df73d354512fc27bd0c5731b3dcf1f8fef7/node/node.go#L198.
+func bootstrapTrustedState(ctx context.Context, dataDir string, meta bootstrapMeta) error {
+	stateDB, err := openConsensusStateDB(dataDir)
+	if err != nil {
+		return fmt.Errorf("failed to open cometbft state store: %w", err)
+	}
+	defer stateDB.Close()
+
+	blockStore, err := openConsensusBlockstore(dataDir)
+	if err != nil {
+		return fmt.Errorf("failed to open consensus blockstore: %w", err)
+	}
+	defer blockStore.Close()
+
+	if !blockStore.IsEmpty() {
+		return fmt.Errorf("blockstore not empty, trying to initialize non empty state")
+	}
+
+	stateStore := cmtState.NewBootstrapStore(stateDB, cmtState.StoreOptions{
+		DiscardABCIResponses: cmtCfg.DefaultConfig().Storage.DiscardABCIResponses,
+	})
+	defer stateStore.Close()
+
+	state, err := stateStore.Load()
+	if err != nil {
+		return err
+	}
+
+	if !state.IsEmpty() {
+		return fmt.Errorf("state not empty, trying to initialize non empty state")
+	}
+
+	var statePB cmtProtoState.State
+	if err := proto.Unmarshal(meta.State, &statePB); err != nil {
+		return fmt.Errorf("failed to unmarshal consensus state: %w", err)
+	}
+	metaState, err := cmtState.FromProto(&statePB)
+	if err != nil {
+		return fmt.Errorf("failed to parse consensus state: %w", err)
+	}
+
+	var commitPB cmtProto.Commit
+	if err := proto.Unmarshal(meta.Commit, &commitPB); err != nil {
+		return fmt.Errorf("failed to unmarshal consensus commit: %w", err)
+	}
+	commit, err := cmttypes.CommitFromProto(&commitPB)
+	if err != nil {
+		return fmt.Errorf("failed to parse consensus commit: %w", err)
+	}
+
+	if err = stateStore.Bootstrap(*metaState); err != nil {
+		return err
+	}
+
+	err = blockStore.SaveSeenCommit(metaState.LastBlockHeight, commit)
+	if err != nil {
+		return err
+	}
+
+	// Once the stores are bootstrapped, we need to set the height at which the node has finished
+	// statesyncing. This will allow the blocksync reactor to fetch blocks at a proper height.
+	// In case this operation fails, it is equivalent to a failure in  online state sync where the operator
+	// needs to manually delete the state and blockstores and rerun the bootstrapping process.
+	err = stateStore.SetOfflineStateSyncHeight(metaState.LastBlockHeight)
+	if err != nil {
+		return fmt.Errorf("failed to set synced height: %w", err)
+	}
+
+	return err
+}
+
+// Commit is adapted and simplified and mimics StateProvider behaviour used in the upstream BootstrapState.
+func Commit(ctx context.Context, blockStore *store.BlockStore, height uint64) (*types.Commit, error) {
+	commit := blockStore.LoadBlockCommit(int64(height))
+	if commit == nil {
+		return nil, fmt.Errorf("commit not found at height %d", height)
+	}
+	return commit, nil
+}
+
+// State is adapted and mimics StateProvider behaviour used in the upstream BootstrapState.
+func State(ctx context.Context, height uint64, stateStore cmtState.Store, blockStore *store.BlockStore) (cmtState.State, error) {
+
+	// The snapshot height maps onto the state heights as follows:
+	//
+	// height: last block, i.e. the snapshotted height
+	// height+1: current block, i.e. the first block we'll process after the snapshot
+	// height+2: next block, i.e. the second block after the snapshot
+	//
+	// We need to fetch the NextValidators from height+2 because if the application changed
+	// the validator set at the snapshot height then this only takes effect at height+2.
+	h := int64(height)
+	lastMeta := blockStore.LoadBlockMeta(h)
+	if lastMeta == nil {
+		return cmtState.State{}, fmt.Errorf("block meta not found at height %d", h)
+	}
+	currentMeta := blockStore.LoadBlockMeta(h + 1)
+	if currentMeta == nil {
+		return cmtState.State{}, fmt.Errorf("block meta not found at height %d", h+1)
+	}
+	nextMeta := blockStore.LoadBlockMeta(h + 2)
+	if nextMeta == nil {
+		return cmtState.State{}, fmt.Errorf("block meta not found at height %d", h+2)
+	}
+
+	lastVals, err := stateStore.LoadValidators(h)
+	if err != nil {
+		return cmtState.State{}, err
+	}
+	currentVals, err := stateStore.LoadValidators(h + 1)
+	if err != nil {
+		return cmtState.State{}, err
+	}
+	nextVals, err := stateStore.LoadValidators(h + 2)
+	if err != nil {
+		return cmtState.State{}, err
+	}
+
+	consensusParams, err := stateStore.LoadConsensusParams(h + 1)
+	if err != nil {
+		return cmtState.State{}, err
+	}
+
+	storeState, err := stateStore.Load()
+	if err != nil {
+		return cmtState.State{}, err
+	}
+	if storeState.IsEmpty() {
+		return cmtState.State{}, fmt.Errorf("state store is empty")
+	}
+
+	state := cmtState.State{
+		ChainID: storeState.ChainID,
+		Version: cmtProtoState.Version{
+			Consensus: currentMeta.Header.Version,
+			Software:  version.TMCoreSemVer,
+		},
+		InitialHeight: storeState.InitialHeight,
+	}
+	if state.InitialHeight == 0 {
+		state.InitialHeight = 1
+	}
+
+	state.LastBlockHeight = lastMeta.Header.Height
+	state.LastBlockTime = lastMeta.Header.Time
+	state.LastBlockID = lastMeta.BlockID
+	state.AppHash = currentMeta.Header.AppHash
+	state.LastResultsHash = currentMeta.Header.LastResultsHash
+	state.LastValidators = lastVals
+	state.Validators = currentVals
+	state.NextValidators = nextVals
+	state.LastHeightValidatorsChanged = nextMeta.Header.Height
+	state.ConsensusParams = consensusParams
+	state.LastHeightConsensusParamsChanged = currentMeta.Header.Height
+
+	return state, nil
+}

go/oasis-node/cmd/storage/dbs.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
+	cometbftDB "github.com/cometbft/cometbft-db"
 	cmtCfg "github.com/cometbft/cometbft/config"
 	"github.com/cometbft/cometbft/state"
 	"github.com/cometbft/cometbft/store"
@@ -61,15 +62,19 @@ func openConsensusBlockstore(dataDir string) (*store.BlockStore, error) {
 	return blockstore, nil
 }
 
-func openConsensusStatestore(dataDir string) (state.Store, error) {
+func openConsensusStateDB(dataDir string) (cometbftDB.DB, error) {
 	cmtConfig := cmtCfg.DefaultConfig()
 	cmtConfig.SetRoot(filepath.Join(dataDir, cmtCommon.StateDir))
 
 	dbProvider, err := cmtDB.Provider()
 	if err != nil {
 		return nil, fmt.Errorf("failed to obtain db provider: %w", err)
 	}
-	stateDB, err := cmtDB.OpenStateDB(dbProvider, cmtConfig)
+	return cmtDB.OpenStateDB(dbProvider, cmtConfig)
+}
+
+func openConsensusStatestore(dataDir string) (state.Store, error) {
+	stateDB, err := openConsensusStateDB(dataDir)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open state db: %w", err)
 	}