From 3267392074dbb151e4d90c9d547182469de1c5b5 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 2 Aug 2022 20:51:29 -0700 Subject: [PATCH 01/56] Added updateStateCommitment and GetAppsUpdated --- persistence/application.go | 35 +++++++++++++++++++ persistence/pre_persistence/app.go | 5 +++ persistence/schema/base_actor.go | 4 +++ persistence/schema/protocol_actor.go | 2 ++ persistence/schema/shared_sql.go | 5 +++ persistence/shared_sql.go | 52 ++++++++++++++++++++++++++++ persistence/test/state_hash_test.go | 7 ++++ shared/modules/persistence_module.go | 1 + shared/types/genesis/validator.go | 3 ++ utility/block.go | 3 ++ utility/state.go | 44 +++++++++++++++++++++++ 11 files changed, 161 insertions(+) create mode 100644 persistence/test/state_hash_test.go create mode 100644 utility/state.go diff --git a/persistence/application.go b/persistence/application.go index 169524df4..e2cf8ee13 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -4,16 +4,51 @@ import ( "encoding/hex" "log" + "github.com/golang/protobuf/proto" "github.com/pokt-network/pocket/persistence/schema" "github.com/pokt-network/pocket/shared/types" + + typesGenesis "github.com/pokt-network/pocket/shared/types/genesis" ) func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool, err error) { return p.GetExists(schema.ApplicationActor, address, height) } +func (p PostgresContext) GetAppsUpdated(height int64) (apps [][]byte, err error) { + actors, err := p.GetActorsUpdated(schema.ApplicationActor, height) + if err != nil { + return nil, err + } + + for _, actor := range actors { + // This breaks the pattern of protos in persistence + app := typesGenesis.App{ + Address: []byte(actor.Address), + PublicKey: []byte(actor.PublicKey), + // Paused: actor.Paused, + // Status: actor.Status, + Chains: actor.Chains, + MaxRelays: actor.ActorSpecificParam, + StakedTokens: actor.StakedTokens, + PausedHeight: actor.PausedHeight, + UnstakingHeight: actor.UnstakingHeight, + Output: []byte(actor.OutputAddress), + } + appBytes, err := proto.Marshal(&app) + if err != nil { + return nil, err + } + apps = append(apps, appBytes) + } + return +} + func (p PostgresContext) GetApp(address []byte, height int64) (operator, publicKey, stakedTokens, maxRelays, outputAddress string, pauseHeight, unstakingHeight int64, chains []string, err error) { actor, err := p.GetActor(schema.ApplicationActor, address, height) + if err != nil { + return + } operator = actor.Address publicKey = actor.PublicKey stakedTokens = actor.StakedTokens diff --git a/persistence/pre_persistence/app.go b/persistence/pre_persistence/app.go index a1d5ac455..e0de24234 100644 --- a/persistence/pre_persistence/app.go +++ b/persistence/pre_persistence/app.go @@ -11,6 +11,11 @@ import ( "google.golang.org/protobuf/proto" ) +func (m *PrePersistenceContext) GetAppsUpdated(height int64) ([][]byte, error) { + // Not implemented + return nil, nil +} + func (m *PrePersistenceContext) GetAppExists(address []byte, height int64) (exists bool, err error) { db := m.Store() key := append(AppPrefixKey, address...) diff --git a/persistence/schema/base_actor.go b/persistence/schema/base_actor.go index 3b0c6c0a3..77c92842c 100644 --- a/persistence/schema/base_actor.go +++ b/persistence/schema/base_actor.go @@ -64,6 +64,10 @@ func (actor *BaseProtocolActorSchema) GetChainsTableSchema() string { return ProtocolActorChainsTableSchema(actor.chainsHeightConstraintName) } +func (actor *BaseProtocolActorSchema) GetUpdatedAtHeightQuery(height int64) string { + return SelectAtHeight(AllColsSelector, height, actor.tableName) +} + func (actor *BaseProtocolActorSchema) GetQuery(address string, height int64) string { return Select(AllColsSelector, address, height, actor.tableName) } diff --git a/persistence/schema/protocol_actor.go b/persistence/schema/protocol_actor.go index a24bf1afd..19cc5f461 100644 --- a/persistence/schema/protocol_actor.go +++ b/persistence/schema/protocol_actor.go @@ -16,6 +16,8 @@ type ProtocolActorSchema interface { /*** Read/Get Queries ***/ + // Returns a query to retrieve all of a Actors updated at that specific height. + GetUpdatedAtHeightQuery(height int64) string // Returns a query to retrieve all of a single Actor's attributes. GetQuery(address string, height int64) string // Returns a query for the existence of an Actor given its address. diff --git a/persistence/schema/shared_sql.go b/persistence/schema/shared_sql.go index 63f059051..a47ae7ab3 100644 --- a/persistence/schema/shared_sql.go +++ b/persistence/schema/shared_sql.go @@ -69,6 +69,11 @@ func ProtocolActorChainsTableSchema(constraintName string) string { )`, AddressCol, ChainIDCol, HeightCol, DefaultBigInt, constraintName, AddressCol, ChainIDCol, HeightCol) } +func SelectAtHeight(selector string, height int64, tableName string) string { + return fmt.Sprintf(`SELECT %s FROM %s WHERE height=%d`, + selector, tableName, height) +} + func Select(selector, address string, height int64, tableName string) string { return fmt.Sprintf(`SELECT %s FROM %s WHERE address='%s' AND height<=%d ORDER BY height DESC LIMIT 1`, selector, tableName, address, height) diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index 369cbe402..4319ced81 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -41,6 +41,58 @@ func (p *PostgresContext) GetExists(actorSchema schema.ProtocolActorSchema, addr return } +func (p *PostgresContext) GetActorsUpdated(actorSchema schema.ProtocolActorSchema, height int64) (actors []schema.BaseActor, err error) { + ctx, conn, err := p.GetCtxAndConnection() + if err != nil { + return + } + + rows, err := conn.Query(ctx, actorSchema.GetUpdatedAtHeightQuery(height)) + if err != nil { + return nil, err + } + defer rows.Close() + + var actor schema.BaseActor + for rows.Next() { + if err = rows.Scan( + &actor.Address, &actor.PublicKey, &actor.StakedTokens, &actor.ActorSpecificParam, + &actor.OutputAddress, &actor.PausedHeight, &actor.UnstakingHeight, + &actor.Chains, &height, + ); err != nil { + return + } + + if actorSchema.GetChainsTableName() == "" { + continue + } + + chainRows, chainsErr := conn.Query(ctx, actorSchema.GetChainsQuery(actor.Address, height)) + if err != nil { + return nil, chainsErr // Why couldn't I just `return` here and use `err`? + } + defer chainRows.Close() + + var chainAddr string + var chainID string + var chainEndHeight int64 // unused + for rows.Next() { + err = rows.Scan(&chainAddr, &chainID, &chainEndHeight) + if err != nil { + return + } + if chainAddr != actor.Address { + return nil, fmt.Errorf("weird") + } + actor.Chains = append(actor.Chains, chainID) + } + + actors = append(actors, actor) + } + + return +} + func (p *PostgresContext) GetActor(actorSchema schema.ProtocolActorSchema, address []byte, height int64) (actor schema.BaseActor, err error) { ctx, conn, err := p.GetCtxAndConnection() if err != nil { diff --git a/persistence/test/state_hash_test.go b/persistence/test/state_hash_test.go new file mode 100644 index 000000000..c57c2c699 --- /dev/null +++ b/persistence/test/state_hash_test.go @@ -0,0 +1,7 @@ +package test + +import "testing" + +func TestSomething(t *testing.T) { + +} diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 06854ca39..d9b6fd4e4 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -71,6 +71,7 @@ type PersistenceContext interface { SetAccountAmount(address []byte, amount string) error // TECHDEBT(team): Delete this function // App Operations + GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height GetAppExists(address []byte, height int64) (exists bool, err error) InsertApp(address []byte, publicKey []byte, output []byte, paused bool, status int, maxRelays string, stakedAmount string, chains []string, pausedHeight int64, unstakingHeight int64) error UpdateApp(address []byte, maxRelays string, stakedAmount string, chainsToUpdate []string) error diff --git a/shared/types/genesis/validator.go b/shared/types/genesis/validator.go index d40406af8..0030f4452 100644 --- a/shared/types/genesis/validator.go +++ b/shared/types/genesis/validator.go @@ -7,6 +7,9 @@ import ( "google.golang.org/protobuf/encoding/protojson" ) +// TODO_IN_THIS_COMMIT: See https://github.com/pokt-network/pocket/pull/139/files to remove this shit + + // HACK: Since the protocol actor protobufs (e.g. validator, fisherman, etc) use `bytes` for some // fields (e.g. `address`, `output`, `publicKey`), we need to use a helper struct to unmarshal the // the types when they are defined via json (e.g. genesis file, testing configurations, etc...). diff --git a/utility/block.go b/utility/block.go index 8d57dcbac..3a8d2e501 100644 --- a/utility/block.go +++ b/utility/block.go @@ -74,6 +74,9 @@ func (u *UtilityContext) EndBlock(proposer []byte) types.Error { if err := u.BeginUnstakingMaxPaused(); err != nil { return err } + if err := u.updateStateCommitment(); err != nil { + return err + } return nil } diff --git a/utility/state.go b/utility/state.go new file mode 100644 index 000000000..4f128ac2a --- /dev/null +++ b/utility/state.go @@ -0,0 +1,44 @@ +package utility + +import ( + "fmt" + "log" + + "github.com/pokt-network/pocket/shared/types" + typesUtil "github.com/pokt-network/pocket/utility/types" +) + +func (u *UtilityContext) updateStateCommitment() types.Error { + // Update the Merkle Tree associated with each actor + for _, actorType := range typesUtil.ActorTypes { + // Need to get all the actors updated at this height + switch actorType { + case typesUtil.ActorType_App: + apps, err := u.Context.GetAppsUpdated(u.LatestHeight) // shouldn't need to pass in a height here + if err != nil { + return types.NewError(types.Code(42), "Couldn't figure out apps updated") + } + fmt.Println("apps: ", apps) + case typesUtil.ActorType_Val: + fallthrough + case typesUtil.ActorType_Fish: + fallthrough + case typesUtil.ActorType_Node: + fallthrough + default: + log.Fatalf("Not supported yet") + } + } + + // TODO: Update Merkle Tree for Accounts + + // TODO: Update Merkle Tree for Pools + + // TODO:Update Merkle Tree for Blocks + + // TODO: Update Merkle Tree for Params + + // TODO: Update Merkle Tree for Flags + + return nil +} From b0bbbb4e0fed1ff1c461dfb205e45dac16a8d7e0 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 2 Aug 2022 20:55:00 -0700 Subject: [PATCH 02/56] Added celestiaorg/smt tree --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fca0bc766..11300d8ae 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect From b52d59a0e4eeceebcd8a351f2f24be25fc718082 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 2 Aug 2022 21:49:03 -0700 Subject: [PATCH 03/56] Added updating merkle tree using app protos --- go.mod | 1 + go.sum | 2 ++ persistence/application.go | 13 +++++++ persistence/db.go | 6 ++-- persistence/module.go | 51 ++++++++++++++++++++++++++++ persistence/pre_persistence/app.go | 5 +++ shared/modules/persistence_module.go | 4 ++- utility/state.go | 5 +-- 8 files changed, 82 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 11300d8ae..3b51bf12d 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( require ( filippo.io/edwards25519 v1.0.0-rc.1 // indirect + github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect diff --git a/go.sum b/go.sum index 786bc0d0c..873a26c37 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/ProtonMail/go-ecvrf v0.0.1/go.mod h1:fhZbiRYn62/JGnBG2NGwCx0oT+gr/+I5 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 h1:iRNKw2WmAbVgGMNYzDH5Y2yY3+jyxwEK9Hc5pwIjZAE= +github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884/go.mod h1:/sdYDakowo/XaxS2Fl7CBqtuf/O2uTqF2zmAUFAtAiw= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/persistence/application.go b/persistence/application.go index e2cf8ee13..ccc1a5ff9 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -15,6 +15,19 @@ func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool return p.GetExists(schema.ApplicationActor, address, height) } +func (p PostgresContext) UpdateAppTree(apps [][]byte) error { + for _, app := range apps { + appProto := typesGenesis.App{} + if err := proto.Unmarshal(app, &appProto); err != nil { + return err + } + if _, err := p.MerkleTrees[AppMerkleTree].Update(appProto.Address, app); err != nil { + return err + } + } + return nil +} + func (p PostgresContext) GetAppsUpdated(height int64) (apps [][]byte, err error) { actors, err := p.GetActorsUpdated(schema.ApplicationActor, height) if err != nil { diff --git a/persistence/db.go b/persistence/db.go index 56e6be62c..a144d1ce0 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -7,6 +7,7 @@ import ( "math/rand" "time" + "github.com/celestiaorg/smt" "github.com/jackc/pgx/v4" "github.com/pokt-network/pocket/persistence/kvstore" "github.com/pokt-network/pocket/persistence/schema" @@ -35,8 +36,9 @@ type PostgresContext struct { // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get // access to these directly via the postgres module. - PostgresDB *pgx.Conn - BlockStore kvstore.KVStore + PostgresDB *pgx.Conn + BlockStore kvstore.KVStore + MerkleTrees map[MerkleTree]*smt.SparseMerkleTree } func (pg *PostgresContext) GetCtxAndConnection() (context.Context, *pgx.Conn, error) { diff --git a/persistence/module.go b/persistence/module.go index 49f21d627..f5ed6d152 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -1,8 +1,10 @@ package persistence import ( + "crypto/sha256" "log" + "github.com/celestiaorg/smt" "github.com/jackc/pgx/v4" "github.com/pokt-network/pocket/persistence/kvstore" "github.com/pokt-network/pocket/shared/config" @@ -46,6 +48,20 @@ func (p PostgresContext) SetValidatorStakeAmount(address []byte, stakeAmount str panic("TODO: implement PostgresContext.SetValidatorStakeAmount") } +type MerkleTree float64 + +const ( + AppMerkleTree MerkleTree = iota + ValMerkleTree + FishMerkleTree + ServiceNodeMerkleTree + AccountMerkleTree + PoolMerkleTree + BlocksMerkleTree + ParamsMerkleTree + FlagsMerkleTree +) + type persistenceModule struct { bus modules.Bus @@ -56,6 +72,8 @@ type persistenceModule struct { blockStore kvstore.KVStore // A mapping of context IDs to persistence contexts contexts map[contextId]modules.PersistenceContext + // Merkle trees + trees map[MerkleTree]*smt.SparseMerkleTree } type contextId uint64 @@ -77,6 +95,7 @@ func Create(c *config.Config) (modules.PersistenceModule, error) { postgresConn: postgresDb, blockStore: blockStore, contexts: make(map[contextId]modules.PersistenceContext), + trees: make(map[MerkleTree]*smt.SparseMerkleTree), }, nil } @@ -98,6 +117,10 @@ func (p *persistenceModule) Start() error { log.Println("Loading state from previous state...") } + if err != p.initializeTrees() { + return err + } + return nil } @@ -159,3 +182,31 @@ func (m *persistenceModule) shouldHydrateGenesisDb() (bool, error) { return m.blockStore.Exists(heightToBytes(int64(maxHeight))) } + +func (m *persistenceModule) initializeTrees() error { + // Initialise two new key-value store to store the nodes and values of the tree + nodeStore := smt.NewSimpleMap() + valueStore := smt.NewSimpleMap() + + // Initialise the tree + tree := smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) + + m.trees[AppMerkleTree] = tree + + return nil +} + +// // Update the key "foo" with the value "bar" +// _, _ = tree.Update([]byte("foo"), []byte("bar")) + +// // Generate a Merkle proof for foo=bar +// proof, _ := tree.Prove([]byte("foo")) +// root := tree.Root() // We also need the current tree root for the proof + +// // Verify the Merkle proof for foo=bar +// if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), sha256.New()) { +// fmt.Println("Proof verification succeeded.") +// } else { +// fmt.Println("Proof verification failed.") +// } +// } diff --git a/persistence/pre_persistence/app.go b/persistence/pre_persistence/app.go index e0de24234..1d1b1a9ae 100644 --- a/persistence/pre_persistence/app.go +++ b/persistence/pre_persistence/app.go @@ -16,6 +16,11 @@ func (m *PrePersistenceContext) GetAppsUpdated(height int64) ([][]byte, error) { return nil, nil } +func (m *PrePersistenceContext) UpdateAppTree([][]byte) error { + // Not implemented + return nil +} + func (m *PrePersistenceContext) GetAppExists(address []byte, height int64) (exists bool, err error) { db := m.Store() key := append(AppPrefixKey, address...) diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index d9b6fd4e4..a03deb5d4 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -71,7 +71,6 @@ type PersistenceContext interface { SetAccountAmount(address []byte, amount string) error // TECHDEBT(team): Delete this function // App Operations - GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height GetAppExists(address []byte, height int64) (exists bool, err error) InsertApp(address []byte, publicKey []byte, output []byte, paused bool, status int, maxRelays string, stakedAmount string, chains []string, pausedHeight int64, unstakingHeight int64) error UpdateApp(address []byte, maxRelays string, stakedAmount string, chainsToUpdate []string) error @@ -85,6 +84,9 @@ type PersistenceContext interface { SetAppStatusAndUnstakingHeightIfPausedBefore(pausedBeforeHeight, unstakingHeight int64, status int) error SetAppPauseHeight(address []byte, height int64) error GetAppOutputAddress(operator []byte, height int64) (output []byte, err error) + // App Operations - For Tree Merkling + GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height + UpdateAppTree([][]byte) error // ServiceNode Operations GetServiceNodeExists(address []byte, height int64) (exists bool, err error) diff --git a/utility/state.go b/utility/state.go index 4f128ac2a..b75e02287 100644 --- a/utility/state.go +++ b/utility/state.go @@ -1,7 +1,6 @@ package utility import ( - "fmt" "log" "github.com/pokt-network/pocket/shared/types" @@ -18,7 +17,9 @@ func (u *UtilityContext) updateStateCommitment() types.Error { if err != nil { return types.NewError(types.Code(42), "Couldn't figure out apps updated") } - fmt.Println("apps: ", apps) + if err := u.Context.UpdateAppTree(apps); err != nil { + return nil + } case typesUtil.ActorType_Val: fallthrough case typesUtil.ActorType_Fish: From 24f358ada37154b8799e6d48cc15c4320ef2b3c0 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 5 Aug 2022 23:14:07 -0400 Subject: [PATCH 04/56] WIP - trees --- Makefile | 5 ++ persistence/application.go | 17 ++-- persistence/block.go | 8 ++ persistence/db.go | 4 +- persistence/kvstore/kvstore.go | 11 ++- persistence/module.go | 66 +++----------- persistence/pre_persistence/app.go | 8 +- persistence/state.go | 124 +++++++++++++++++++++++++++ persistence/state_test.go | 15 ++++ persistence/test/state_hash_test.go | 7 -- shared/modules/persistence_module.go | 2 +- utility/state.go | 45 ---------- 12 files changed, 191 insertions(+), 121 deletions(-) create mode 100644 persistence/state.go create mode 100644 persistence/state_test.go delete mode 100644 persistence/test/state_hash_test.go delete mode 100644 utility/state.go diff --git a/Makefile b/Makefile index 61c5621c6..9c59a435c 100644 --- a/Makefile +++ b/Makefile @@ -221,6 +221,11 @@ test_sortition: test_persistence: go test ${VERBOSE_TEST} -p=1 ./persistence/... +.PHONY: test_persistence_state_hash +## Run all go unit tests in the Persistence module +test_persistence_state_hash: + go test -run StateHash ${VERBOSE_TEST} -p=1 ./persistence/... + .PHONY: benchmark_sortition ## Benchmark the Sortition library benchmark_sortition: diff --git a/persistence/application.go b/persistence/application.go index ccc1a5ff9..98d62fd7f 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -28,7 +28,9 @@ func (p PostgresContext) UpdateAppTree(apps [][]byte) error { return nil } -func (p PostgresContext) GetAppsUpdated(height int64) (apps [][]byte, err error) { +// TODO_IN_THIS_COMMIT: Not exposed via interface yet +// func (p PostgresContext) getAppsUpdated(height int64) (apps [][]byte, err error) { +func (p PostgresContext) getAppsUpdated(height int64) (apps []*typesGenesis.App, err error) { actors, err := p.GetActorsUpdated(schema.ApplicationActor, height) if err != nil { return nil, err @@ -36,7 +38,7 @@ func (p PostgresContext) GetAppsUpdated(height int64) (apps [][]byte, err error) for _, actor := range actors { // This breaks the pattern of protos in persistence - app := typesGenesis.App{ + app := &typesGenesis.App{ Address: []byte(actor.Address), PublicKey: []byte(actor.PublicKey), // Paused: actor.Paused, @@ -48,11 +50,12 @@ func (p PostgresContext) GetAppsUpdated(height int64) (apps [][]byte, err error) UnstakingHeight: actor.UnstakingHeight, Output: []byte(actor.OutputAddress), } - appBytes, err := proto.Marshal(&app) - if err != nil { - return nil, err - } - apps = append(apps, appBytes) + // appBytes, err := proto.Marshal(&app) + // if err != nil { + // return nil, err + // } + // apps = append(apps, appBytes) + apps = append(apps, app) } return } diff --git a/persistence/block.go b/persistence/block.go index fc9c3eceb..8055ee566 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -4,10 +4,18 @@ import ( "encoding/binary" "encoding/hex" "log" + "os" "github.com/pokt-network/pocket/persistence/schema" ) +func (p *persistenceModule) shouldLoadBlockStore() bool { + if _, err := os.Stat(p.GetBus().GetConfig().Persistence.BlockStorePath); err == nil { + return true + } + return false +} + // OPTIMIZE(team): get from blockstore or keep in cache/memory func (p PostgresContext) GetLatestBlockHeight() (latestHeight int64, err error) { ctx, conn, err := p.GetCtxAndConnection() diff --git a/persistence/db.go b/persistence/db.go index a144d1ce0..2a2f411a6 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -37,8 +37,8 @@ type PostgresContext struct { // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get // access to these directly via the postgres module. PostgresDB *pgx.Conn - BlockStore kvstore.KVStore - MerkleTrees map[MerkleTree]*smt.SparseMerkleTree + BlockStore kvstore.KVStore // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the module (i.e. not context) + MerkleTrees map[MerkleTree]*smt.SparseMerkleTree // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the module (i.e. not context) } func (pg *PostgresContext) GetCtxAndConnection() (context.Context, *pgx.Conn, error) { diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index b598038f9..ec00c5427 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -1,6 +1,7 @@ package kvstore import ( + "errors" "log" badger "github.com/dgraph-io/badger/v3" @@ -19,11 +20,19 @@ type KVStore interface { var _ KVStore = &badgerKVStore{} +var ( + ErrKVStoreExists = errors.New("kvstore already exists") + ErrKVStoreNotExists = errors.New("kvstore does not exist") +) + type badgerKVStore struct { db *badger.DB } -func NewKVStore(path string) (KVStore, error) { +// REFACTOR: Loads or creates a badgerDb at `path`. This may potentially need to be refactored +// into `NewKVStore` and `LoadKVStore` depending on how state sync evolves by leveraging `os.Stat` +// on the file path. +func OpenKVStore(path string) (KVStore, error) { db, err := badger.Open(badger.DefaultOptions(path)) if err != nil { return nil, err diff --git a/persistence/module.go b/persistence/module.go index f5ed6d152..08346c423 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -1,7 +1,6 @@ package persistence import ( - "crypto/sha256" "log" "github.com/celestiaorg/smt" @@ -48,20 +47,6 @@ func (p PostgresContext) SetValidatorStakeAmount(address []byte, stakeAmount str panic("TODO: implement PostgresContext.SetValidatorStakeAmount") } -type MerkleTree float64 - -const ( - AppMerkleTree MerkleTree = iota - ValMerkleTree - FishMerkleTree - ServiceNodeMerkleTree - AccountMerkleTree - PoolMerkleTree - BlocksMerkleTree - ParamsMerkleTree - FlagsMerkleTree -) - type persistenceModule struct { bus modules.Bus @@ -84,7 +69,7 @@ func Create(c *config.Config) (modules.PersistenceModule, error) { return nil, err } - blockStore, err := kvstore.NewKVStore(c.Persistence.BlockStorePath) + blockStore, err := kvstore.OpenKVStore(c.Persistence.BlockStorePath) if err != nil { return nil, err } @@ -102,24 +87,25 @@ func Create(c *config.Config) (modules.PersistenceModule, error) { func (p *persistenceModule) Start() error { log.Println("Starting persistence module...") - shouldHydrateGenesis := false - shouldHydrateGenesis, err := p.shouldHydrateGenesisDb() - if err != nil { - return err - } - - if shouldHydrateGenesis { + if shouldHydrateGenesis, err := p.shouldHydrateGenesisDb(); err != nil { + return nil + } else if shouldHydrateGenesis { + log.Println("Hydrating genesis state...") if err := p.hydrateGenesisDbState(); err != nil { return err } - log.Println("Hydrating genesis state...") } else { - log.Println("Loading state from previous state...") + log.Println("TODO: Finish loading previous state...") + } - if err != p.initializeTrees() { + // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this is cleaner and easier to test + trees, err := initializeTrees() + if err != nil { return err } + // TODO_IN_THIS_COMMIT: load trees from state + p.trees = trees return nil } @@ -182,31 +168,3 @@ func (m *persistenceModule) shouldHydrateGenesisDb() (bool, error) { return m.blockStore.Exists(heightToBytes(int64(maxHeight))) } - -func (m *persistenceModule) initializeTrees() error { - // Initialise two new key-value store to store the nodes and values of the tree - nodeStore := smt.NewSimpleMap() - valueStore := smt.NewSimpleMap() - - // Initialise the tree - tree := smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) - - m.trees[AppMerkleTree] = tree - - return nil -} - -// // Update the key "foo" with the value "bar" -// _, _ = tree.Update([]byte("foo"), []byte("bar")) - -// // Generate a Merkle proof for foo=bar -// proof, _ := tree.Prove([]byte("foo")) -// root := tree.Root() // We also need the current tree root for the proof - -// // Verify the Merkle proof for foo=bar -// if smt.VerifyProof(proof, root, []byte("foo"), []byte("bar"), sha256.New()) { -// fmt.Println("Proof verification succeeded.") -// } else { -// fmt.Println("Proof verification failed.") -// } -// } diff --git a/persistence/pre_persistence/app.go b/persistence/pre_persistence/app.go index 1d1b1a9ae..cad4d4ce2 100644 --- a/persistence/pre_persistence/app.go +++ b/persistence/pre_persistence/app.go @@ -11,10 +11,10 @@ import ( "google.golang.org/protobuf/proto" ) -func (m *PrePersistenceContext) GetAppsUpdated(height int64) ([][]byte, error) { - // Not implemented - return nil, nil -} +// func (m *PrePersistenceContext) GetAppsUpdated(height int64) ([][]byte, error) { +// // Not implemented +// return nil, nil +// } func (m *PrePersistenceContext) UpdateAppTree([][]byte) error { // Not implemented diff --git a/persistence/state.go b/persistence/state.go new file mode 100644 index 000000000..ffab52aea --- /dev/null +++ b/persistence/state.go @@ -0,0 +1,124 @@ +package persistence + +import ( + "bytes" + "crypto/sha256" + "log" + "sort" + + "github.com/celestiaorg/smt" + "github.com/pokt-network/pocket/shared/types" + "google.golang.org/protobuf/proto" +) + +type MerkleTree float64 + +// A work-in-progress list of all the trees we need to update to maintain the overall state +const ( + AppMerkleTree MerkleTree = iota + ValMerkleTree + FishMerkleTree + ServiceNodeMerkleTree + AccountMerkleTree + PoolMerkleTree + BlocksMerkleTree + ParamsMerkleTree + FlagsMerkleTree + lastMerkleTree // Used for iteration purposes only - see https://stackoverflow.com/a/64178235/768439 +) + +func initializeTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { + // We need a separate Merkle tree for each type of actor or storage + trees := make(map[MerkleTree]*smt.SparseMerkleTree, int(lastMerkleTree)) + + for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { + // Initialize two new key-value store to store the nodes and values of the tree + nodeStore := smt.NewSimpleMap() + valueStore := smt.NewSimpleMap() + + trees[treeType] = smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) + } + return trees, nil +} + +func loadTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { + +} + +func (p *PostgresContext) updateStateCommitment() ([]byte, error) { + for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { + switch treeType { + case AppMerkleTree: + apps, err := p.getAppsUpdated(p.Height) + if err != nil { + return nil, types.NewError(types.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT + } + for _, app := range apps { + // OPTIMIZE: Do we want to store the serialized bytes or a hash of it in the KV store? + appBytes, err := proto.Marshal(app) + if err != nil { + return nil, err + } + if _, err := p.MerkleTrees[treeType].Update(app.Address, appBytes); err != nil { + return nil, err + } + } + default: + log.Fatalln("Not handeled uet in state commitment update") + } + } + + // Get the root of each Merkle Tree + roots := make([][]byte, 0) + for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { + roots = append(roots, p.MerkleTrees[treeType].Root()) + } + + // Sort the merkle roots lexicographically + sort.Slice(roots, func(r1, r2 int) bool { + return bytes.Compare(roots[r1], roots[r2]) < 0 + }) + + // Get the state hash + rootsConcat := bytes.Join(roots, []byte{}) + stateHash := sha256.Sum256(rootsConcat) + + return stateHash[:], nil +} + +// computeStateHash(root) +// context := p. +// Update the Merkle Tree associated with each actor +// for _, actorType := range typesUtil.ActorTypes { +// // Need to get all the actors updated at this height +// switch actorType { +// case typesUtil.ActorType_App: +// apps, err := u.Context.GetAppsUpdated(u.LatestHeight) // shouldn't need to pass in a height here +// if err != nil { +// return types.NewError(types.Code(42), "Couldn't figure out apps updated") +// } +// if err := u.Context.UpdateAppTree(apps); err != nil { +// return nil +// } +// case typesUtil.ActorType_Val: +// fallthrough +// case typesUtil.ActorType_Fish: +// fallthrough +// case typesUtil.ActorType_Node: +// fallthrough +// default: +// log.Fatalf("Actor type not supported: %s", actorType) +// } +// } + +// TODO: Update Merkle Tree for Accounts + +// TODO: Update Merkle Tree for Pools + +// TODO:Update Merkle Tree for Blocks + +// TODO: Update Merkle Tree for Params + +// TODO: Update Merkle Tree for Flags + +// return nil diff --git a/persistence/state_test.go b/persistence/state_test.go new file mode 100644 index 000000000..9570f4c00 --- /dev/null +++ b/persistence/state_test.go @@ -0,0 +1,15 @@ +package persistence + +import "testing" + +func TestStateHash_InitializeTrees(t *testing.T) { + +} + +func TestStateHash_LoadTrees(t *testing.T) { + +} + +func TestStateHash_ComputeStateHash(t *testing.T) { + +} diff --git a/persistence/test/state_hash_test.go b/persistence/test/state_hash_test.go deleted file mode 100644 index c57c2c699..000000000 --- a/persistence/test/state_hash_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package test - -import "testing" - -func TestSomething(t *testing.T) { - -} diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index a03deb5d4..a3a7242d4 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -85,7 +85,7 @@ type PersistenceContext interface { SetAppPauseHeight(address []byte, height int64) error GetAppOutputAddress(operator []byte, height int64) (output []byte, err error) // App Operations - For Tree Merkling - GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height + // GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height UpdateAppTree([][]byte) error // ServiceNode Operations diff --git a/utility/state.go b/utility/state.go deleted file mode 100644 index b75e02287..000000000 --- a/utility/state.go +++ /dev/null @@ -1,45 +0,0 @@ -package utility - -import ( - "log" - - "github.com/pokt-network/pocket/shared/types" - typesUtil "github.com/pokt-network/pocket/utility/types" -) - -func (u *UtilityContext) updateStateCommitment() types.Error { - // Update the Merkle Tree associated with each actor - for _, actorType := range typesUtil.ActorTypes { - // Need to get all the actors updated at this height - switch actorType { - case typesUtil.ActorType_App: - apps, err := u.Context.GetAppsUpdated(u.LatestHeight) // shouldn't need to pass in a height here - if err != nil { - return types.NewError(types.Code(42), "Couldn't figure out apps updated") - } - if err := u.Context.UpdateAppTree(apps); err != nil { - return nil - } - case typesUtil.ActorType_Val: - fallthrough - case typesUtil.ActorType_Fish: - fallthrough - case typesUtil.ActorType_Node: - fallthrough - default: - log.Fatalf("Not supported yet") - } - } - - // TODO: Update Merkle Tree for Accounts - - // TODO: Update Merkle Tree for Pools - - // TODO:Update Merkle Tree for Blocks - - // TODO: Update Merkle Tree for Params - - // TODO: Update Merkle Tree for Flags - - return nil -} From a493f1f1870b860df64e7545fb6e8711e623d5d1 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 6 Aug 2022 19:27:19 -0400 Subject: [PATCH 05/56] Update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3b51bf12d..65f7bd77d 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( ) require ( + github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 github.com/dgraph-io/badger/v3 v3.2103.2 github.com/iancoleman/strcase v0.2.0 ) @@ -58,7 +59,6 @@ require ( require ( filippo.io/edwards25519 v1.0.0-rc.1 // indirect - github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect From fce0b424611042392ebb43f6a39314f014e19472 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 6 Aug 2022 20:04:32 -0400 Subject: [PATCH 06/56] Got tests to pass after self review --- consensus/module.go | 2 +- persistence/application.go | 10 ++-- persistence/block.go | 3 +- persistence/context.go | 17 +++++- persistence/db.go | 5 +- persistence/kvstore/kvstore.go | 18 ++++-- persistence/module.go | 5 +- persistence/pre_persistence/persistence.go | 5 ++ persistence/shared_sql.go | 13 ++++- persistence/state.go | 64 +++++++--------------- shared/modules/persistence_module.go | 1 + utility/block.go | 4 +- 12 files changed, 79 insertions(+), 68 deletions(-) diff --git a/consensus/module.go b/consensus/module.go index 832ce5f0e..1008a715c 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -243,7 +243,7 @@ func (m *consensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) } func (m *consensusModule) AppHash() string { - return m.appHash + return m.appHash // TODO: This is a problem } func (m *consensusModule) CurrentHeight() uint64 { diff --git a/persistence/application.go b/persistence/application.go index 98d62fd7f..c3a13e0f2 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -28,8 +28,6 @@ func (p PostgresContext) UpdateAppTree(apps [][]byte) error { return nil } -// TODO_IN_THIS_COMMIT: Not exposed via interface yet -// func (p PostgresContext) getAppsUpdated(height int64) (apps [][]byte, err error) { func (p PostgresContext) getAppsUpdated(height int64) (apps []*typesGenesis.App, err error) { actors, err := p.GetActorsUpdated(schema.ApplicationActor, height) if err != nil { @@ -37,12 +35,14 @@ func (p PostgresContext) getAppsUpdated(height int64) (apps []*typesGenesis.App, } for _, actor := range actors { - // This breaks the pattern of protos in persistence + // DISCUSS_IN_THIS_COMMIT: This breaks the pattern of protos in persistence. + // - Is it okay? + // - Do we embed this logic in `UpdateAppTree` app := &typesGenesis.App{ Address: []byte(actor.Address), PublicKey: []byte(actor.PublicKey), - // Paused: actor.Paused, - // Status: actor.Status, + // Paused: actor.Paused, // DISCUSS_IN_THIS_COMMIT: Is this just a check for pause height = -1? + // Status: actor.Status, // TODO_IN_THIS_COMMIT: Use logic from `GetActorStatus` without an extra query Chains: actor.Chains, MaxRelays: actor.ActorSpecificParam, StakedTokens: actor.StakedTokens, diff --git a/persistence/block.go b/persistence/block.go index 8055ee566..2f01afeca 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -9,6 +9,7 @@ import ( "github.com/pokt-network/pocket/persistence/schema" ) +// TODO_IN_THIS_COMMIT: Out of scope so we might just need to do this as part of state sync func (p *persistenceModule) shouldLoadBlockStore() bool { if _, err := os.Stat(p.GetBus().GetConfig().Persistence.BlockStorePath); err == nil { return true @@ -61,7 +62,7 @@ func (p PostgresContext) StoreBlock(blockProtoBytes []byte) error { // INVESTIGATE: Note that we are writing this directly to the blockStore. Depending on how // the use of the PostgresContext evolves, we may need to write this to `ContextStore` and copy // over to `BlockStore` when the block is committed. - return p.BlockStore.Put(heightToBytes(p.Height), blockProtoBytes) + return p.BlockStore.Set(heightToBytes(p.Height), blockProtoBytes) } func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { diff --git a/persistence/context.go b/persistence/context.go index 1cbf22739..9809bbd2c 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -14,9 +14,17 @@ func (p PostgresContext) RollbackToSavePoint(bytes []byte) error { return nil } +func (p PostgresContext) UpdateAppHash() ([]byte, error) { + if _, err := p.updateStateHash(); err != nil { + return nil, err + } + return p.StateHash, nil +} + func (p PostgresContext) AppHash() ([]byte, error) { - log.Println("TODO: AppHash not implemented") - return []byte("A real app hash, I am not"), nil + // log.Println("TODO: AppHash not implemented") + // return []byte("A real app hash, I am not"), n + return p.StateHash, nil } func (p PostgresContext) Reset() error { @@ -24,8 +32,11 @@ func (p PostgresContext) Reset() error { } func (p PostgresContext) Commit() error { - // HACK: The data has already been written to the postgres DB, so what should we do here? The idea I have is: log.Println("TODO: Postgres context commit is currently a NOOP") + // HACK: The data has already been written to the postgres DB, so what should we do here? The idea I have is: + // if _, err := p.updateStateHash(); err != nil { + // return err + // } return nil } diff --git a/persistence/db.go b/persistence/db.go index 2a2f411a6..469f7abf7 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -33,12 +33,13 @@ var _ modules.PersistenceContext = &PostgresContext{} type PostgresContext struct { Height int64 ContextStore kvstore.KVStore + StateHash []byte // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get // access to these directly via the postgres module. PostgresDB *pgx.Conn - BlockStore kvstore.KVStore // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the module (i.e. not context) - MerkleTrees map[MerkleTree]*smt.SparseMerkleTree // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the module (i.e. not context) + BlockStore kvstore.KVStore // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the persistence module (i.e. not context) + MerkleTrees map[MerkleTree]*smt.SparseMerkleTree // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the persistence module (i.e. not context) } func (pg *PostgresContext) GetCtxAndConnection() (context.Context, *pgx.Conn, error) { diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index ec00c5427..7eb632089 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -4,21 +4,26 @@ import ( "errors" "log" + "github.com/celestiaorg/smt" badger "github.com/dgraph-io/badger/v3" ) +// TODO_IN_THIS_COMMIT: We might be able to remove the `KVStore` interface altogether if we end up using smt.MapStore type KVStore interface { // Lifecycle methods Stop() error - // Accessors - Put(key []byte, value []byte) error - Get(key []byte) ([]byte, error) Exists(key []byte) (bool, error) ClearAll() error + + // Same interface as in `smt.MapStore`` + Set(key []byte, value []byte) error + Get(key []byte) ([]byte, error) + Delete(key []byte) error } var _ KVStore = &badgerKVStore{} +var _ smt.MapStore = &badgerKVStore{} var ( ErrKVStoreExists = errors.New("kvstore already exists") @@ -48,7 +53,7 @@ func NewMemKVStore() KVStore { return badgerKVStore{db: db} } -func (store badgerKVStore) Put(key []byte, value []byte) error { +func (store badgerKVStore) Set(key []byte, value []byte) error { txn := store.db.NewTransaction(true) defer txn.Discard() @@ -85,6 +90,11 @@ func (store badgerKVStore) Get(key []byte) ([]byte, error) { return value, nil } +func (store badgerKVStore) Delete(key []byte) error { + log.Fatalf("badgerKVStore.Delete not implemented yet") + return nil +} + func (store badgerKVStore) Exists(key []byte) (bool, error) { val, err := store.Get(key) if err != nil { diff --git a/persistence/module.go b/persistence/module.go index 08346c423..3ca47ddcc 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -99,8 +99,9 @@ func (p *persistenceModule) Start() error { } - // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this is cleaner and easier to test - trees, err := initializeTrees() + // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this making `initializeTrees` + // be able to create and/or load trees outside the scope of the persistence module makes it easier to test. + trees, err := newMerkleTrees() if err != nil { return err } diff --git a/persistence/pre_persistence/persistence.go b/persistence/pre_persistence/persistence.go index 7bdb3b7b4..237ba93f1 100644 --- a/persistence/pre_persistence/persistence.go +++ b/persistence/pre_persistence/persistence.go @@ -184,6 +184,11 @@ func (m *PrePersistenceContext) RollbackToSavePoint(bytes []byte) error { return nil } +func (m *PrePersistenceContext) UpdateAppHash() ([]byte, error) { + // log.Fatalf("PrePersistenceContext not implementing UpdateAppHash") + return nil, nil +} + // AppHash creates a unique hash based on the global state object // NOTE: AppHash is an inefficient, arbitrary, mock implementation that enables the functionality // TODO written for replacement, taking any and all better implementation suggestions - even if a temporary measure diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index 4319ced81..aacfb36e5 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -41,6 +41,7 @@ func (p *PostgresContext) GetExists(actorSchema schema.ProtocolActorSchema, addr return } +// TODO_IN_THIS_COMMIT: Consolidate logic with `GetActor` to reduce code footprint func (p *PostgresContext) GetActorsUpdated(actorSchema schema.ProtocolActorSchema, height int64) (actors []schema.BaseActor, err error) { ctx, conn, err := p.GetCtxAndConnection() if err != nil { @@ -56,9 +57,15 @@ func (p *PostgresContext) GetActorsUpdated(actorSchema schema.ProtocolActorSchem var actor schema.BaseActor for rows.Next() { if err = rows.Scan( - &actor.Address, &actor.PublicKey, &actor.StakedTokens, &actor.ActorSpecificParam, - &actor.OutputAddress, &actor.PausedHeight, &actor.UnstakingHeight, - &actor.Chains, &height, + &actor.Address, + &actor.PublicKey, + &actor.StakedTokens, + &actor.ActorSpecificParam, + &actor.OutputAddress, + &actor.PausedHeight, + &actor.UnstakingHeight, + &actor.Chains, + &height, ); err != nil { return } diff --git a/persistence/state.go b/persistence/state.go index ffab52aea..b685208ba 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -15,24 +15,26 @@ type MerkleTree float64 // A work-in-progress list of all the trees we need to update to maintain the overall state const ( + // Actors AppMerkleTree MerkleTree = iota ValMerkleTree FishMerkleTree ServiceNodeMerkleTree AccountMerkleTree PoolMerkleTree + // Data / state BlocksMerkleTree ParamsMerkleTree FlagsMerkleTree lastMerkleTree // Used for iteration purposes only - see https://stackoverflow.com/a/64178235/768439 ) -func initializeTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { +func newMerkleTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { // We need a separate Merkle tree for each type of actor or storage trees := make(map[MerkleTree]*smt.SparseMerkleTree, int(lastMerkleTree)) for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { - // Initialize two new key-value store to store the nodes and values of the tree + // TODO_IN_THIS_COMMIT: Rather than using `NewSimpleMap`, use a disk based key-value store nodeStore := smt.NewSimpleMap() valueStore := smt.NewSimpleMap() @@ -41,11 +43,18 @@ func initializeTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { return trees, nil } -func loadTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { - +func loadMerkleTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { + log.Fatalf("loadMerkleTrees not implemented yet") } -func (p *PostgresContext) updateStateCommitment() ([]byte, error) { +// DISCUSS_IN_THIS_COMMIT(drewskey): Thoughts on this approach? +// 1. Retrieves all of the actors / data types updated at the current height +// 2. Updates the Merkle Tree associated with each actor / data type +// - This operation is idempotent so you can call `updateStateHash` as often as you want +// 3. Update the context's "cached" state hash +// 4. Returns the state hash +func (p *PostgresContext) updateStateHash() ([]byte, error) { + // Update all the merkle trees for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { switch treeType { case AppMerkleTree: @@ -54,17 +63,18 @@ func (p *PostgresContext) updateStateCommitment() ([]byte, error) { return nil, types.NewError(types.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT } for _, app := range apps { - // OPTIMIZE: Do we want to store the serialized bytes or a hash of it in the KV store? appBytes, err := proto.Marshal(app) if err != nil { return nil, err } + // An update results in a create/update that is idempotent if _, err := p.MerkleTrees[treeType].Update(app.Address, appBytes); err != nil { return nil, err } + // TODO_IN_THIS_COMMIT: Add support for `Delete` operations to remove it from the tree } default: - log.Fatalln("Not handeled uet in state commitment update") + log.Fatalln("Not handled yet in state commitment update", treeType) } } @@ -83,42 +93,6 @@ func (p *PostgresContext) updateStateCommitment() ([]byte, error) { rootsConcat := bytes.Join(roots, []byte{}) stateHash := sha256.Sum256(rootsConcat) - return stateHash[:], nil + p.StateHash = stateHash[:] + return p.StateHash, nil } - -// computeStateHash(root) -// context := p. -// Update the Merkle Tree associated with each actor -// for _, actorType := range typesUtil.ActorTypes { -// // Need to get all the actors updated at this height -// switch actorType { -// case typesUtil.ActorType_App: -// apps, err := u.Context.GetAppsUpdated(u.LatestHeight) // shouldn't need to pass in a height here -// if err != nil { -// return types.NewError(types.Code(42), "Couldn't figure out apps updated") -// } -// if err := u.Context.UpdateAppTree(apps); err != nil { -// return nil -// } -// case typesUtil.ActorType_Val: -// fallthrough -// case typesUtil.ActorType_Fish: -// fallthrough -// case typesUtil.ActorType_Node: -// fallthrough -// default: -// log.Fatalf("Actor type not supported: %s", actorType) -// } -// } - -// TODO: Update Merkle Tree for Accounts - -// TODO: Update Merkle Tree for Pools - -// TODO:Update Merkle Tree for Blocks - -// TODO: Update Merkle Tree for Params - -// TODO: Update Merkle Tree for Flags - -// return nil diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index a3a7242d4..555075312 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -37,6 +37,7 @@ type PersistenceContext interface { Commit() error Release() // IMPROVE: Return an error? + UpdateAppHash() ([]byte, error) AppHash() ([]byte, error) GetHeight() (int64, error) diff --git a/utility/block.go b/utility/block.go index 3a8d2e501..4ce17aaf3 100644 --- a/utility/block.go +++ b/utility/block.go @@ -74,8 +74,8 @@ func (u *UtilityContext) EndBlock(proposer []byte) types.Error { if err := u.BeginUnstakingMaxPaused(); err != nil { return err } - if err := u.updateStateCommitment(); err != nil { - return err + if _, err := u.Context.UpdateAppHash(); err != nil { + return types.ErrAppHash(err) } return nil } From 4551843bad85bcf5cd4704f3ec0c852fda078548 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 20 Sep 2022 16:04:50 -0700 Subject: [PATCH 07/56] Resolved some conflicts to make 'make develop_test' work --- go.mod | 2 +- persistence/application.go | 30 +++++----- persistence/block.go | 5 +- persistence/kvstore/kvstore.go | 2 - persistence/module.go | 28 ++++----- persistence/shared_sql.go | 10 ++-- persistence/state.go | 11 +++- shared/indexer/indexer.go | 11 ++-- shared/types/genesis/validator.go | 95 +++++++++++++++---------------- utility/block.go | 2 +- 10 files changed, 100 insertions(+), 96 deletions(-) diff --git a/go.mod b/go.mod index 72201023a..f1507cdd9 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v1.12.1 // indirect diff --git a/persistence/application.go b/persistence/application.go index a7f2c4325..cc5a8ed72 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -5,10 +5,9 @@ import ( "log" "github.com/golang/protobuf/proto" - "github.com/pokt-network/pocket/persistence/schema" - "github.com/pokt-network/pocket/shared/types" + "github.com/pokt-network/pocket/persistence/types" - typesGenesis "github.com/pokt-network/pocket/shared/types/genesis" + "github.com/pokt-network/pocket/shared/modules" ) func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool, err error) { @@ -17,19 +16,20 @@ func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool func (p PostgresContext) UpdateAppTree(apps [][]byte) error { for _, app := range apps { - appProto := typesGenesis.App{} + appProto := types.Actor{} if err := proto.Unmarshal(app, &appProto); err != nil { return err } - if _, err := p.MerkleTrees[AppMerkleTree].Update(appProto.Address, app); err != nil { + bzAddr, _ := hex.DecodeString(appProto.Address) + if _, err := p.MerkleTrees[AppMerkleTree].Update(bzAddr, app); err != nil { return err } } return nil } -func (p PostgresContext) getAppsUpdated(height int64) (apps []*typesGenesis.App, err error) { - actors, err := p.GetActorsUpdated(schema.ApplicationActor, height) +func (p PostgresContext) getAppsUpdated(height int64) (apps []*types.Actor, err error) { + actors, err := p.GetActorsUpdated(types.ApplicationActor, height) if err != nil { return nil, err } @@ -38,17 +38,17 @@ func (p PostgresContext) getAppsUpdated(height int64) (apps []*typesGenesis.App, // DISCUSS_IN_THIS_COMMIT: This breaks the pattern of protos in persistence. // - Is it okay? // - Do we embed this logic in `UpdateAppTree` - app := &typesGenesis.App{ - Address: []byte(actor.Address), - PublicKey: []byte(actor.PublicKey), + app := &types.Actor{ + Address: actor.Address, + PublicKey: actor.PublicKey, // Paused: actor.Paused, // DISCUSS_IN_THIS_COMMIT: Is this just a check for pause height = -1? // Status: actor.Status, // TODO_IN_THIS_COMMIT: Use logic from `GetActorStatus` without an extra query - Chains: actor.Chains, - MaxRelays: actor.ActorSpecificParam, - StakedTokens: actor.StakedTokens, + Chains: actor.Chains, + // MaxRelays: actor.ActorSpecificParam, + // StakedTokens: actor.StakedTokens, PausedHeight: actor.PausedHeight, UnstakingHeight: actor.UnstakingHeight, - Output: []byte(actor.OutputAddress), + Output: actor.OutputAddress, } // appBytes, err := proto.Marshal(&app) // if err != nil { @@ -61,7 +61,7 @@ func (p PostgresContext) getAppsUpdated(height int64) (apps []*typesGenesis.App, } func (p PostgresContext) GetApp(address []byte, height int64) (operator, publicKey, stakedTokens, maxRelays, outputAddress string, pauseHeight, unstakingHeight int64, chains []string, err error) { - actor, err := p.GetActor(schema.ApplicationActor, address, height) + actor, err := p.GetActor(types.ApplicationActor, address, height) if err != nil { return } diff --git a/persistence/block.go b/persistence/block.go index 170f81984..a6ba799a8 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -3,8 +3,9 @@ package persistence import ( "encoding/binary" "encoding/hex" - "github.com/pokt-network/pocket/persistence/types" "log" + + "github.com/pokt-network/pocket/persistence/types" ) // OPTIMIZE(team): get from blockstore or keep in memory @@ -52,7 +53,7 @@ func (p PostgresContext) StoreBlock(blockProtoBytes []byte) error { // INVESTIGATE: Note that we are writing this directly to the blockStore. Depending on how // the use of the PostgresContext evolves, we may need to write this to `ContextStore` and copy // over to `BlockStore` when the block is committed. - return p.DB.Blockstore.Put(heightToBytes(p.Height), blockProtoBytes) + return p.DB.Blockstore.Set(heightToBytes(p.Height), blockProtoBytes) } func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index 03ee1b83a..b2b11a5bd 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -15,8 +15,6 @@ type KVStore interface { // Accessors // TODO: Add a proper iterator interface - Put(key []byte, value []byte) error - Get(key []byte) ([]byte, error) // TODO: Add pagination for `GetAll` GetAll(prefixKey []byte, descending bool) ([][]byte, error) diff --git a/persistence/module.go b/persistence/module.go index b9007bee4..a8ece5bae 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -26,7 +26,6 @@ type PersistenceModule struct { postgresURL string nodeSchema string genesisPath string - blockStore kvstore.KVStore // INVESTIGATE: We may need to create a custom `BlockStore` package in the future // TECHDEBT: Need to implement context pooling (for writes), timeouts (for read & writes), etc... writeContext *PostgresContext // only one write context is allowed at a time @@ -37,7 +36,7 @@ type PersistenceModule struct { // INVESTIGATE: We may need to create a custom `BlockStore` package in the future. blockStore kvstore.KVStore // A mapping of context IDs to persistence contexts - contexts map[contextId]modules.PersistenceContext + // contexts map[contextId]modules.PersistenceRWContext // Merkle trees trees map[MerkleTree]*smt.SparseMerkleTree } @@ -81,10 +80,20 @@ func Create(configPath, genesisPath string) (modules.PersistenceModule, error) { genesisPath: genesisPath, blockStore: blockStore, writeContext: nil, - contexts: make(map[contextId]modules.PersistenceContext), - trees: make(map[MerkleTree]*smt.SparseMerkleTree), + // contexts: make(map[contextId]modules.PersistenceContext), + trees: make(map[MerkleTree]*smt.SparseMerkleTree), } + // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this making `initializeTrees` + // be able to create and/or load trees outside the scope of the persistence module makes it easier to test. + trees, err := newMerkleTrees() + if err != nil { + return nil, err + } + + // TODO_IN_THIS_COMMIT: load trees from state + persistenceMod.trees = trees + // Determine if we should hydrate the genesis db or use the current state of the DB attached if shouldHydrateGenesis, err := persistenceMod.shouldHydrateGenesisDb(); err != nil { return nil, err @@ -98,15 +107,6 @@ func Create(configPath, genesisPath string) (modules.PersistenceModule, error) { log.Println("Loading state from previous state...") } - // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this making `initializeTrees` - // be able to create and/or load trees outside the scope of the persistence module makes it easier to test. - trees, err := newMerkleTrees() - if err != nil { - return err - } - // TODO_IN_THIS_COMMIT: load trees from state - p.trees = trees - return persistenceMod, nil } @@ -241,7 +241,7 @@ func initializeBlockStore(blockStorePath string) (kvstore.KVStore, error) { if blockStorePath == "" { return kvstore.NewMemKVStore(), nil } - return kvstore.NewKVStore(blockStorePath) + return kvstore.OpenKVStore(blockStorePath) } // TODO(drewsky): Simplify and externalize the logic for whether genesis should be populated and diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index d0b1a323d..1721ce884 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -45,19 +45,19 @@ func (p *PostgresContext) GetExists(actorSchema types.ProtocolActorSchema, addre } // TODO_IN_THIS_COMMIT: Consolidate logic with `GetActor` to reduce code footprint -func (p *PostgresContext) GetActorsUpdated(actorSchema schema.ProtocolActorSchema, height int64) (actors []schema.BaseActor, err error) { - ctx, conn, err := p.GetCtxAndConnection() +func (p *PostgresContext) GetActorsUpdated(actorSchema types.ProtocolActorSchema, height int64) (actors []types.BaseActor, err error) { + ctx, tx, err := p.DB.GetCtxAndTxn() if err != nil { return } - rows, err := conn.Query(ctx, actorSchema.GetUpdatedAtHeightQuery(height)) + rows, err := tx.Query(ctx, actorSchema.GetUpdatedAtHeightQuery(height)) if err != nil { return nil, err } defer rows.Close() - var actor schema.BaseActor + var actor types.BaseActor for rows.Next() { if err = rows.Scan( &actor.Address, @@ -77,7 +77,7 @@ func (p *PostgresContext) GetActorsUpdated(actorSchema schema.ProtocolActorSchem continue } - chainRows, chainsErr := conn.Query(ctx, actorSchema.GetChainsQuery(actor.Address, height)) + chainRows, chainsErr := tx.Query(ctx, actorSchema.GetChainsQuery(actor.Address, height)) if err != nil { return nil, chainsErr // Why couldn't I just `return` here and use `err`? } diff --git a/persistence/state.go b/persistence/state.go index b685208ba..6e7abd9e7 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -3,11 +3,12 @@ package persistence import ( "bytes" "crypto/sha256" + "encoding/hex" "log" "sort" "github.com/celestiaorg/smt" - "github.com/pokt-network/pocket/shared/types" + typesUtil "github.com/pokt-network/pocket/utility/types" "google.golang.org/protobuf/proto" ) @@ -60,7 +61,7 @@ func (p *PostgresContext) updateStateHash() ([]byte, error) { case AppMerkleTree: apps, err := p.getAppsUpdated(p.Height) if err != nil { - return nil, types.NewError(types.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT + return nil, typesUtil.NewError(typesUtil.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT } for _, app := range apps { appBytes, err := proto.Marshal(app) @@ -68,7 +69,11 @@ func (p *PostgresContext) updateStateHash() ([]byte, error) { return nil, err } // An update results in a create/update that is idempotent - if _, err := p.MerkleTrees[treeType].Update(app.Address, appBytes); err != nil { + addrBz, err := hex.DecodeString(app.Address) + if err != nil { + return nil, err + } + if _, err := p.MerkleTrees[treeType].Update(addrBz, appBytes); err != nil { return nil, err } // TODO_IN_THIS_COMMIT: Add support for `Delete` operations to remove it from the tree diff --git a/shared/indexer/indexer.go b/shared/indexer/indexer.go index 559ca1f6e..3c5d7167a 100644 --- a/shared/indexer/indexer.go +++ b/shared/indexer/indexer.go @@ -5,6 +5,7 @@ package indexer import ( "encoding/hex" "fmt" + "github.com/pokt-network/pocket/shared/codec" "github.com/jordanorelli/lexnum" @@ -111,7 +112,7 @@ type txIndexer struct { } func NewTxIndexer(databasePath string) (TxIndexer, error) { - db, err := kvstore.NewKVStore(databasePath) + db, err := kvstore.OpenKVStore(databasePath) return &txIndexer{ db: db, }, err @@ -198,22 +199,22 @@ func (indexer *txIndexer) get(key []byte) (TxResult, error) { func (indexer *txIndexer) indexByHash(hash, bz []byte) (hashKey []byte, err error) { key := indexer.hashKey(hash) - return key, indexer.db.Put(key, bz) + return key, indexer.db.Set(key, bz) } func (indexer *txIndexer) indexByHeightAndIndex(height int64, index int32, bz []byte) error { - return indexer.db.Put(indexer.heightAndIndexKey(height, index), bz) + return indexer.db.Set(indexer.heightAndIndexKey(height, index), bz) } func (indexer *txIndexer) indexBySender(sender string, bz []byte) error { - return indexer.db.Put(indexer.senderKey(sender), bz) + return indexer.db.Set(indexer.senderKey(sender), bz) } func (indexer *txIndexer) indexByRecipient(recipient string, bz []byte) error { if recipient == "" { return nil } - return indexer.db.Put(indexer.recipientKey(recipient), bz) + return indexer.db.Set(indexer.recipientKey(recipient), bz) } // key helper functions diff --git a/shared/types/genesis/validator.go b/shared/types/genesis/validator.go index 0030f4452..17c2af5fd 100644 --- a/shared/types/genesis/validator.go +++ b/shared/types/genesis/validator.go @@ -1,50 +1,49 @@ package genesis -import ( - "encoding/hex" - "encoding/json" - - "google.golang.org/protobuf/encoding/protojson" -) - -// TODO_IN_THIS_COMMIT: See https://github.com/pokt-network/pocket/pull/139/files to remove this shit - - -// HACK: Since the protocol actor protobufs (e.g. validator, fisherman, etc) use `bytes` for some -// fields (e.g. `address`, `output`, `publicKey`), we need to use a helper struct to unmarshal the -// the types when they are defined via json (e.g. genesis file, testing configurations, etc...). -// Alternative solutions could include whole wrapper structs (i.e. duplication of schema definition), -// using strings instead of bytes (i.e. major change with downstream effects) or avoid defining these -// types in json altogether (i.e. limitation of usability). -type JsonBytesLoaderHelper struct { - Address HexData `json:"address,omitempty"` - PublicKey HexData `json:"public_key,omitempty"` - Output HexData `json:"output,omitempty"` -} - -type HexData []byte - -func (h *HexData) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - decoded, err := hex.DecodeString(s) - if err != nil { - return err - } - *h = HexData(decoded) - return nil -} - -func (v *Validator) UnmarshalJSON(data []byte) error { - var jh JsonBytesLoaderHelper - json.Unmarshal(data, &jh) - - protojson.Unmarshal(data, v) - v.Address = jh.Address - v.PublicKey = jh.PublicKey - v.Output = jh.Output - - return nil -} +// import ( +// "encoding/hex" +// "encoding/json" + +// "google.golang.org/protobuf/encoding/protojson" +// ) + +// // TODO_IN_THIS_COMMIT: See https://github.com/pokt-network/pocket/pull/139/files to remove this shit + +// // HACK: Since the protocol actor protobufs (e.g. validator, fisherman, etc) use `bytes` for some +// // fields (e.g. `address`, `output`, `publicKey`), we need to use a helper struct to unmarshal the +// // the types when they are defined via json (e.g. genesis file, testing configurations, etc...). +// // Alternative solutions could include whole wrapper structs (i.e. duplication of schema definition), +// // using strings instead of bytes (i.e. major change with downstream effects) or avoid defining these +// // types in json altogether (i.e. limitation of usability). +// type JsonBytesLoaderHelper struct { +// Address HexData `json:"address,omitempty"` +// PublicKey HexData `json:"public_key,omitempty"` +// Output HexData `json:"output,omitempty"` +// } + +// type HexData []byte + +// func (h *HexData) UnmarshalJSON(data []byte) error { +// var s string +// if err := json.Unmarshal(data, &s); err != nil { +// return err +// } +// decoded, err := hex.DecodeString(s) +// if err != nil { +// return err +// } +// *h = HexData(decoded) +// return nil +// } + +// func (v *Validator) UnmarshalJSON(data []byte) error { +// var jh JsonBytesLoaderHelper +// json.Unmarshal(data, &jh) + +// protojson.Unmarshal(data, v) +// v.Address = jh.Address +// v.PublicKey = jh.PublicKey +// v.Output = jh.Output + +// return nil +// } diff --git a/utility/block.go b/utility/block.go index bc387fabf..acf148ca4 100644 --- a/utility/block.go +++ b/utility/block.go @@ -90,7 +90,7 @@ func (u *UtilityContext) EndBlock(proposer []byte) typesUtil.Error { return err } if _, err := u.Context.UpdateAppHash(); err != nil { - return types.ErrAppHash(err) + return typesUtil.ErrAppHash(err) } return nil } From eb9de301cb34df7b703a134c94b9869b41bc091e Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 21 Sep 2022 18:28:10 -0700 Subject: [PATCH 08/56] WIP --- Makefile | 3 +- go.mod | 2 +- persistence/StateHash.md | 71 ++++++++++++++++++++++++++++ persistence/application.go | 52 ++++++++++---------- persistence/db.go | 1 + persistence/genesis.go | 2 +- persistence/shared_sql.go | 49 +++++-------------- persistence/state.go | 31 ++++++------ persistence/types/base_actor.go | 4 +- persistence/types/protocol_actor.go | 2 +- shared/modules/persistence_module.go | 53 +++++++++++++++++++-- 11 files changed, 182 insertions(+), 88 deletions(-) create mode 100644 persistence/StateHash.md diff --git a/Makefile b/Makefile index 12338ed37..c0aa0036f 100644 --- a/Makefile +++ b/Makefile @@ -347,9 +347,10 @@ benchmark_p2p_addrbook: # HACK - Like TECHDEBT, but much worse. This needs to be prioritized # REFACTOR - Similar to TECHDEBT, but will require a substantial rewrite and change across the codebase # CONSIDERATION - A comment that involves extra work but was thoughts / considered as part of some implementation +# CONSOLIDATE - We likely have similar implementations/types of the same thing, and we should consolidate them. # DISCUSS_IN_THIS_COMMIT - SHOULD NEVER BE COMMITTED TO MASTER. It is a way for the reviewer of a PR to start / reply to a discussion. # TODO_IN_THIS_COMMIT - SHOULD NEVER BE COMMITTED TO MASTER. It is a way to start the review process while non-critical changes are still in progress -TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "TODO_IN_THIS_COMMIT" -e "DISCUSS_IN_THIS_COMMIT" +TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "TODO_IN_THIS_COMMIT" -e "DISCUSS_IN_THIS_COMMIT" -e "CONSOLIDATE" # How do I use TODOs? # 1. : ; diff --git a/go.mod b/go.mod index f1507cdd9..6e76ac253 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect diff --git a/persistence/StateHash.md b/persistence/StateHash.md new file mode 100644 index 000000000..bf22494b3 --- /dev/null +++ b/persistence/StateHash.md @@ -0,0 +1,71 @@ +# This discussion is aimed at: + +1. Defining how we should compute the state hash +2. Identify potential changes needed in the current codebase +3. Propose next steps and actionable on implementation + +## Goals: + +- Define how the state hash will be computed +- Propose the necessary changes in separate tasks +- Implement each of the necessary pieces + +## Non-goals: + +- Choice/decision of Merkle Tree Design & Implementation +- Selection of a key-value store engine + +## Primitives / non-negotiables: + +- We will be using Merkle Trees (and Merkle Proofs) for this design (i.e. not vector commitments) +- We will be using a SQL engine for this (i.e. specifically PostgresSQL) +- We will be using Protobufs (not Flatbuffers, json, yaml or other) for the schema + +## Necessary technical context: + +### DB Engines + +Insert table from here: [Merkle Tree Design & Implementation](https://tikv.org/deep-dive/key-value-engine/b-tree-vs-lsm/#summary) + +- Most **Key-Value Store DB** Engines use **LSM-trees** -> good for writes +- Most **SQL DB** Engines use **B-Trees** -> good for reads + +_Basically all but there can be exceptions_ + +### Addressable Merkle Trees + +State is stored use an Account Based (non UTXO) based Modle + +Insert image from: https://www.horizen.io/blockchain-academy/technology/expert/utxo-vs-account-model/#:~:text=The%20UTXO%20model%20is%20a,constructions%2C%20as%20well%20a%20sharding. + +--- + +### Data Flow + +## Basics: + +1. Get each actor (flag, param, etc...) updated at a certain height (the context's height) +2. Compute the protobuf (the deterministic schema we use as source of truth) +3. Serialize the data struct +4. Update the corresponding merkle tree +5. Compute a state hash from the aggregated roots of all trees as per pokt-network/pocket-network-protocol@main/persistence#562-state-transition-sequence-diagram + +## Q&A + +Q: Can the SQL Engine be changed? +A: Yes + +Q: Can the SQL Engine be removed altogether? +A: Yes, but hard + +Q: Can the protobuf schema change? +A: Yes, but out-of-scope + +Q: Can protobufs be replaced? +A: Maybe, but out-of-scope + +--- + +Consolidations: +- Consolidate `UtilActorType` and `persistence.ActorType` +- `modules.Actors` interface vs `types.Actor` in persistenceGenesis \ No newline at end of file diff --git a/persistence/application.go b/persistence/application.go index cc5a8ed72..292bc83ed 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -4,62 +4,60 @@ import ( "encoding/hex" "log" - "github.com/golang/protobuf/proto" "github.com/pokt-network/pocket/persistence/types" + "google.golang.org/protobuf/proto" "github.com/pokt-network/pocket/shared/modules" ) -func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool, err error) { - return p.GetExists(types.ApplicationActor, address, height) -} - -func (p PostgresContext) UpdateAppTree(apps [][]byte) error { +func (p PostgresContext) UpdateApplicationsTree(apps []modules.Actor) error { for _, app := range apps { - appProto := types.Actor{} - if err := proto.Unmarshal(app, &appProto); err != nil { + bzAddr, err := hex.DecodeString(app.GetAddress()) + if err != nil { return err } - bzAddr, _ := hex.DecodeString(appProto.Address) - if _, err := p.MerkleTrees[AppMerkleTree].Update(bzAddr, app); err != nil { + + appBz, err := proto.Marshal(app.(*types.Actor)) + if err != nil { + return err + } + + if _, err := p.MerkleTrees[appMerkleTree].Update(bzAddr, appBz); err != nil { // NOTE: This is the only line unique to `Application` return err } } + return nil } -func (p PostgresContext) getAppsUpdated(height int64) (apps []*types.Actor, err error) { - actors, err := p.GetActorsUpdated(types.ApplicationActor, height) +func (p PostgresContext) getApplicationsUpdatedAtHeight(height int64) (apps []*types.Actor, err error) { + actors, err := p.GetActorsUpdated(types.ApplicationActor, height) // NOTE: This is the only line unique to `Application` if err != nil { return nil, err } + apps = make([]*types.Actor, len(actors)) for _, actor := range actors { - // DISCUSS_IN_THIS_COMMIT: This breaks the pattern of protos in persistence. - // - Is it okay? - // - Do we embed this logic in `UpdateAppTree` app := &types.Actor{ - Address: actor.Address, - PublicKey: actor.PublicKey, - // Paused: actor.Paused, // DISCUSS_IN_THIS_COMMIT: Is this just a check for pause height = -1? - // Status: actor.Status, // TODO_IN_THIS_COMMIT: Use logic from `GetActorStatus` without an extra query - Chains: actor.Chains, - // MaxRelays: actor.ActorSpecificParam, - // StakedTokens: actor.StakedTokens, + ActorType: types.ActorType_App, + Address: actor.Address, + PublicKey: actor.PublicKey, + Chains: actor.Chains, + GenericParam: actor.ActorSpecificParam, + StakedAmount: actor.StakedTokens, PausedHeight: actor.PausedHeight, UnstakingHeight: actor.UnstakingHeight, Output: actor.OutputAddress, } - // appBytes, err := proto.Marshal(&app) - // if err != nil { - // return nil, err - // } - // apps = append(apps, appBytes) apps = append(apps, app) } return } +func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool, err error) { + return p.GetExists(types.ApplicationActor, address, height) +} + func (p PostgresContext) GetApp(address []byte, height int64) (operator, publicKey, stakedTokens, maxRelays, outputAddress string, pauseHeight, unstakingHeight int64, chains []string, err error) { actor, err := p.GetActor(types.ApplicationActor, address, height) if err != nil { diff --git a/persistence/db.go b/persistence/db.go index 60526a3ed..ae3bab35a 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/pokt-network/pocket/persistence/types" "github.com/celestiaorg/smt" diff --git a/persistence/genesis.go b/persistence/genesis.go index e7c6abcaa..727fa4f57 100644 --- a/persistence/genesis.go +++ b/persistence/genesis.go @@ -316,7 +316,7 @@ func (p PostgresContext) GetAllFishermen(height int64) (f []modules.Actor, err e return } -// TODO (Team) deprecate with interface #163 as #163 is getting large +// CONSOLIDATE: Consolidate `types.BaseActor` with `types.Actor` func (p PostgresContext) BaseActorToActor(ba types.BaseActor, actorType types.ActorType) *types.Actor { actor := new(types.Actor) actor.ActorType = actorType diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index 1721ce884..da6498782 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -44,7 +44,6 @@ func (p *PostgresContext) GetExists(actorSchema types.ProtocolActorSchema, addre return } -// TODO_IN_THIS_COMMIT: Consolidate logic with `GetActor` to reduce code footprint func (p *PostgresContext) GetActorsUpdated(actorSchema types.ProtocolActorSchema, height int64) (actors []types.BaseActor, err error) { ctx, tx, err := p.DB.GetCtxAndTxn() if err != nil { @@ -57,44 +56,16 @@ func (p *PostgresContext) GetActorsUpdated(actorSchema types.ProtocolActorSchema } defer rows.Close() - var actor types.BaseActor + // OPTIMIZE: Consolidate logic with `GetActor` to reduce code footprint + var addr string for rows.Next() { - if err = rows.Scan( - &actor.Address, - &actor.PublicKey, - &actor.StakedTokens, - &actor.ActorSpecificParam, - &actor.OutputAddress, - &actor.PausedHeight, - &actor.UnstakingHeight, - &actor.Chains, - &height, - ); err != nil { + if err = rows.Scan(&addr); err != nil { return } - if actorSchema.GetChainsTableName() == "" { - continue - } - - chainRows, chainsErr := tx.Query(ctx, actorSchema.GetChainsQuery(actor.Address, height)) + actor, err := p.GetActor(actorSchema, []byte(addr), height) if err != nil { - return nil, chainsErr // Why couldn't I just `return` here and use `err`? - } - defer chainRows.Close() - - var chainAddr string - var chainID string - var chainEndHeight int64 // unused - for rows.Next() { - err = rows.Scan(&chainAddr, &chainID, &chainEndHeight) - if err != nil { - return - } - if chainAddr != actor.Address { - return nil, fmt.Errorf("weird") - } - actor.Chains = append(actor.Chains, chainID) + return nil, err } actors = append(actors, actor) @@ -116,10 +87,16 @@ func (p *PostgresContext) GetActor(actorSchema types.ProtocolActorSchema, addres return p.GetChainsForActor(ctx, txn, actorSchema, actor, height) } +// IMPORTANT: Need to consolidate `persistence/types.BaseActor` with `persistence/genesisTypes.Actor` func (p *PostgresContext) GetActorFromRow(row pgx.Row) (actor types.BaseActor, height int64, err error) { err = row.Scan( - &actor.Address, &actor.PublicKey, &actor.StakedTokens, &actor.ActorSpecificParam, - &actor.OutputAddress, &actor.PausedHeight, &actor.UnstakingHeight, + &actor.Address, + &actor.PublicKey, + &actor.StakedTokens, + &actor.ActorSpecificParam, + &actor.OutputAddress, + &actor.PausedHeight, + &actor.UnstakingHeight, &height) return } diff --git a/persistence/state.go b/persistence/state.go index 6e7abd9e7..221ed3f90 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -16,18 +16,21 @@ type MerkleTree float64 // A work-in-progress list of all the trees we need to update to maintain the overall state const ( - // Actors - AppMerkleTree MerkleTree = iota - ValMerkleTree - FishMerkleTree - ServiceNodeMerkleTree - AccountMerkleTree - PoolMerkleTree - // Data / state - BlocksMerkleTree - ParamsMerkleTree - FlagsMerkleTree - lastMerkleTree // Used for iteration purposes only - see https://stackoverflow.com/a/64178235/768439 + // Actor Merkle Trees + appMerkleTree MerkleTree = iota + valMerkleTree + fishMerkleTree + serviceNodeMerkleTree + accountMerkleTree + poolMerkleTree + + // Data / State Merkle Trees + blocksMerkleTree + paramsMerkleTree + flagsMerkleTree + + // Used for iteration purposes only - see https://stackoverflow.com/a/64178235/768439 + lastMerkleTree ) func newMerkleTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { @@ -58,8 +61,8 @@ func (p *PostgresContext) updateStateHash() ([]byte, error) { // Update all the merkle trees for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { switch treeType { - case AppMerkleTree: - apps, err := p.getAppsUpdated(p.Height) + case appMerkleTree: + apps, err := p.getApplicationsUpdatedAtHeight(p.Height) if err != nil { return nil, typesUtil.NewError(typesUtil.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT } diff --git a/persistence/types/base_actor.go b/persistence/types/base_actor.go index 9cead8390..054d10393 100644 --- a/persistence/types/base_actor.go +++ b/persistence/types/base_actor.go @@ -61,7 +61,7 @@ func (actor *BaseProtocolActorSchema) GetChainsTableSchema() string { } func (actor *BaseProtocolActorSchema) GetUpdatedAtHeightQuery(height int64) string { - return SelectAtHeight(AllColsSelector, height, actor.tableName) + return SelectAtHeight(AddressCol, height, actor.tableName) } func (actor *BaseProtocolActorSchema) GetQuery(address string, height int64) string { @@ -154,5 +154,5 @@ func (x *Actor) GetActorTyp() modules.ActorType { if x != nil { return x.GetActorType() } - return ActorType_App + return ActorType_App // HACK: Is this a hack? } diff --git a/persistence/types/protocol_actor.go b/persistence/types/protocol_actor.go index 9dd894fca..cb1e23fbf 100644 --- a/persistence/types/protocol_actor.go +++ b/persistence/types/protocol_actor.go @@ -16,7 +16,7 @@ type ProtocolActorSchema interface { /*** Read/Get Queries ***/ - // Returns a query to retrieve all of a Actors updated at that specific height. + // Returns a query to retrieve the addresses of all the Actors updated at that specific height GetUpdatedAtHeightQuery(height int64) string // Returns a query to retrieve all of a single Actor's attributes. GetQuery(address string, height int64) string diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index fdb0c0875..2030f0861 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -1,7 +1,7 @@ package modules import ( - "github.com/pokt-network/pocket/persistence/kvstore" + "github.com/pokt-network/pocket/persistence/kvstore" // Should be moved to shared "github.com/pokt-network/pocket/shared/debug" ) @@ -86,10 +86,6 @@ type PersistenceWriteContext interface { SetAppUnstakingHeightAndStatus(address []byte, unstakingHeight int64, status int) error SetAppStatusAndUnstakingHeightIfPausedBefore(pausedBeforeHeight, unstakingHeight int64, status int) error SetAppPauseHeight(address []byte, height int64) error - GetAppOutputAddress(operator []byte, height int64) (output []byte, err error) - // App Operations - For Tree Merkling - // GetAppsUpdated(height int64) ([][]byte, error) // Returns the apps updates at the given height - UpdateAppTree([][]byte) error // ServiceNode Operations InsertServiceNode(address []byte, publicKey []byte, output []byte, paused bool, status int, serviceURL string, stakedTokens string, chains []string, pausedHeight int64, unstakingHeight int64) error @@ -129,6 +125,29 @@ type PersistenceWriteContext interface { // Flag Operations InitFlags() error SetFlag(paramName string, value interface{}, enabled bool) error + + // Tree Operations + + // # Option 1: + + UpdateApplicationsTree([]Actor) error + // UpdateValidatorsTree([]Actor) error + // UpdateServiceNodesTree([]Actor) error + // UpdateFishermanTree([]Actor) error + // UpdateTree([]Actor) error + // UpdateTree([]Other) error + + // # Option 2: + // UpdateActorTree(types.ProtocolActorSchema, []Actor) error + // UpdateTree([]Other) error + + // # Option 3: + // UpdateApplicationsTree([]Application) error + // UpdateValidatorsTree([]Validator) error + // UpdateServiceNodesTree([]ServiceNode) error + // UpdateFishermanTree([]Fisherman) error + // UpdateTree([]FutureActor) error + // UpdateTree([]Other) error } type PersistenceReadContext interface { @@ -207,4 +226,28 @@ type PersistenceReadContext interface { GetIntFlag(paramName string, height int64) (int, bool, error) GetStringFlag(paramName string, height int64) (string, bool, error) GetBytesFlag(paramName string, height int64) ([]byte, bool, error) + + // Tree Operations + + // # Option 1: + + // GetApplicationsUpdatedAtHeight(height int64) ([]Actor, error) + // GetValidatorsUpdatedAtHeight(height int64) ([]Actor, error) + // GetServiceNodesUpdatedAtHeight(height int64) ([]Actor, error) + // GetFishermanUpdatedAtHeight(height int64) ([]Actor, error) + // GetUpdatedAtHeight(height int64) ([]Actor, error) + // GetUpdatedAtHeight(height int64) ([]Actor, error) + // UpdateTree(height int64) ([]Actor, error) + + // # Option 2: + // GetUpdatedAtHeight(types.ProtocolActorSchema, height int64) ([]Actor, error) + // GetUpdatedAtHeight(height int64) ([]Other, error) + + // # Option 3: + // GetApplicationsUpdatedAtHeight(height int64) ([]Application, error) + // GetValidatorsUpdatedAtHeight(height int64) ([]Validator, error) + // GetServiceNodesUpdatedAtHeight(height int64) ([]ServiceNode, error) + // GetFishermanUpdatedAtHeight(height int64) ([]Fisherman, error) + // GetUpdatedAtHeight(height int64) ([]FutureActor, error) + // GetUpdatedAtHeight(height int64) ([]Other, error) } From e27ff8e8895625d1926bd1f12f9afd8e27fd62ee Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 21 Sep 2022 19:28:18 -0700 Subject: [PATCH 09/56] Interim commit on block related stuff --- consensus/block.go | 15 +++------------ consensus/consensus_tests/utils_test.go | 3 +-- persistence/StateHash.md | 5 +++-- persistence/application.go | 6 ++++-- persistence/block.go | 11 ++++------- persistence/context.go | 3 --- persistence/db.go | 3 +-- persistence/kvstore/kvstore.go | 11 ++++++++--- persistence/state.go | 7 +------ shared/indexer/indexer.go | 8 ++++---- shared/modules/persistence_module.go | 3 ++- shared/modules/utility_module.go | 1 - utility/block.go | 13 +------------ 13 files changed, 32 insertions(+), 57 deletions(-) diff --git a/consensus/block.go b/consensus/block.go index ae531bcb8..80b495d95 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -2,9 +2,10 @@ package consensus import ( "encoding/hex" - "github.com/pokt-network/pocket/shared/codec" "unsafe" + "github.com/pokt-network/pocket/shared/codec" + typesCons "github.com/pokt-network/pocket/consensus/types" ) @@ -111,18 +112,8 @@ func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { return err } - // IMPROVE(olshansky): temporary solution. `ApplyBlock` above applies the - // transactions to the postgres database, and this stores it in the KV store upon commitment. - // Instead of calling this directly, an alternative solution is to store the block metadata in - // the persistence context and have `CommitPersistenceContext` do this under the hood. However, - // additional `Block` metadata will need to be passed through and may change when we merkle the - // state hash. - if err := m.utilityContext.StoreBlock(blockProtoBytes); err != nil { - return err - } - // Commit and release the context - if err := m.utilityContext.CommitPersistenceContext(); err != nil { + if err := m.utilityContext.CommitPersistenceContext(blockProtoBytes); err != nil { return err } diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 2ab4ff850..b14d01ab3 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -354,7 +354,7 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti MaxTimes(4) utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() - utilityContextMock.EXPECT().CommitPersistenceContext().Return(nil).AnyTimes() + utilityContextMock.EXPECT().CommitPersistenceContext(gomock.Any()).Return(nil).AnyTimes() utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() utilityContextMock.EXPECT(). GetProposalTransactions(gomock.Any(), maxTxBytes, gomock.AssignableToTypeOf(emptyByzValidators)). @@ -364,7 +364,6 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti ApplyBlock(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(appHash, nil). AnyTimes() - utilityContextMock.EXPECT().StoreBlock(gomock.Any()).AnyTimes().Return(nil) persistenceContextMock.EXPECT().Commit().Return(nil).AnyTimes() diff --git a/persistence/StateHash.md b/persistence/StateHash.md index bf22494b3..a930df4fc 100644 --- a/persistence/StateHash.md +++ b/persistence/StateHash.md @@ -66,6 +66,7 @@ A: Maybe, but out-of-scope --- -Consolidations: +Learnings / Ideas: + - Consolidate `UtilActorType` and `persistence.ActorType` -- `modules.Actors` interface vs `types.Actor` in persistenceGenesis \ No newline at end of file +- `modules.Actors` interface vs `types.Actor` in persistenceGenesis diff --git a/persistence/application.go b/persistence/application.go index 292bc83ed..0bd6750bd 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -22,7 +22,8 @@ func (p PostgresContext) UpdateApplicationsTree(apps []modules.Actor) error { return err } - if _, err := p.MerkleTrees[appMerkleTree].Update(bzAddr, appBz); err != nil { // NOTE: This is the only line unique to `Application` + // OPTIMIZE: This is the only line unique to `Application` + if _, err := p.MerkleTrees[appMerkleTree].Update(bzAddr, appBz); err != nil { return err } } @@ -31,7 +32,8 @@ func (p PostgresContext) UpdateApplicationsTree(apps []modules.Actor) error { } func (p PostgresContext) getApplicationsUpdatedAtHeight(height int64) (apps []*types.Actor, err error) { - actors, err := p.GetActorsUpdated(types.ApplicationActor, height) // NOTE: This is the only line unique to `Application` + // OPTIMIZE: This is the only line unique to `Application` + actors, err := p.GetActorsUpdated(types.ApplicationActor, height) if err != nil { return nil, err } diff --git a/persistence/block.go b/persistence/block.go index a6ba799a8..49de236fc 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -49,13 +49,6 @@ func (p PostgresContext) StoreTransaction(transactionProtoBytes []byte) error { return nil } -func (p PostgresContext) StoreBlock(blockProtoBytes []byte) error { - // INVESTIGATE: Note that we are writing this directly to the blockStore. Depending on how - // the use of the PostgresContext evolves, we may need to write this to `ContextStore` and copy - // over to `BlockStore` when the block is committed. - return p.DB.Blockstore.Set(heightToBytes(p.Height), blockProtoBytes) -} - func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { ctx, tx, err := p.DB.GetCtxAndTxn() if err != nil { @@ -72,3 +65,7 @@ func heightToBytes(height int64) []byte { binary.LittleEndian.PutUint64(heightBytes, uint64(height)) return heightBytes } + +func (p PostgresContext) storeBlock(blockProtoBytes []byte) error { + return p.DB.Blockstore.Put(heightToBytes(p.Height), blockProtoBytes) +} diff --git a/persistence/context.go b/persistence/context.go index da6ae9117..57e15e0ad 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -23,8 +23,6 @@ func (p PostgresContext) UpdateAppHash() ([]byte, error) { } func (p PostgresContext) AppHash() ([]byte, error) { - // log.Println("TODO: AppHash not implemented") - // return []byte("A real app hash, I am not"), n return p.StateHash, nil } @@ -46,7 +44,6 @@ func (p PostgresContext) Commit() error { } if err := p.DB.conn.Close(ctx); err != nil { log.Println("[TODO][ERROR] Implement connection pooling. Error when closing DB connecting...", err) - } return nil } diff --git a/persistence/db.go b/persistence/db.go index ae3bab35a..060cec041 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -45,8 +45,7 @@ type PostgresContext struct { Height int64 DB PostgresDB - ContextStore kvstore.KVStore - StateHash []byte + StateHash []byte // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get // access to these directly via the postgres module. diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index b2b11a5bd..2b5e94292 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -8,7 +8,6 @@ import ( badger "github.com/dgraph-io/badger/v3" ) -// TODO_IN_THIS_COMMIT: We might be able to remove the `KVStore` interface altogether if we end up using smt.MapStore type KVStore interface { // Lifecycle methods Stop() error @@ -22,7 +21,7 @@ type KVStore interface { ClearAll() error // Same interface as in `smt.MapStore`` - Set(key []byte, value []byte) error + Put(key, value []byte) error Get(key []byte) ([]byte, error) Delete(key []byte) error } @@ -58,7 +57,7 @@ func NewMemKVStore() KVStore { return badgerKVStore{db: db} } -func (store badgerKVStore) Set(key []byte, value []byte) error { +func (store badgerKVStore) Put(key, value []byte) error { txn := store.db.NewTransaction(true) defer txn.Discard() @@ -70,6 +69,12 @@ func (store badgerKVStore) Set(key []byte, value []byte) error { return txn.Commit() } +// CONSOLIDATE: We might be able to remove the `KVStore` interface altogether if we end up using `smt.MapStore` +// A wrapper around `Put` to conform to the `smt.MapStore` interface +func (store *badgerKVStore) Set(key, value []byte) error { + return store.Put(key, value) +} + func (store badgerKVStore) Get(key []byte) ([]byte, error) { txn := store.db.NewTransaction(false) defer txn.Discard() diff --git a/persistence/state.go b/persistence/state.go index 221ed3f90..f35f3622b 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -51,12 +51,7 @@ func loadMerkleTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { log.Fatalf("loadMerkleTrees not implemented yet") } -// DISCUSS_IN_THIS_COMMIT(drewskey): Thoughts on this approach? -// 1. Retrieves all of the actors / data types updated at the current height -// 2. Updates the Merkle Tree associated with each actor / data type -// - This operation is idempotent so you can call `updateStateHash` as often as you want -// 3. Update the context's "cached" state hash -// 4. Returns the state hash +// Question: Is this the right approach? func (p *PostgresContext) updateStateHash() ([]byte, error) { // Update all the merkle trees for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { diff --git a/shared/indexer/indexer.go b/shared/indexer/indexer.go index 3c5d7167a..59355787f 100644 --- a/shared/indexer/indexer.go +++ b/shared/indexer/indexer.go @@ -199,22 +199,22 @@ func (indexer *txIndexer) get(key []byte) (TxResult, error) { func (indexer *txIndexer) indexByHash(hash, bz []byte) (hashKey []byte, err error) { key := indexer.hashKey(hash) - return key, indexer.db.Set(key, bz) + return key, indexer.db.Put(key, bz) } func (indexer *txIndexer) indexByHeightAndIndex(height int64, index int32, bz []byte) error { - return indexer.db.Set(indexer.heightAndIndexKey(height, index), bz) + return indexer.db.Put(indexer.heightAndIndexKey(height, index), bz) } func (indexer *txIndexer) indexBySender(sender string, bz []byte) error { - return indexer.db.Set(indexer.senderKey(sender), bz) + return indexer.db.Put(indexer.senderKey(sender), bz) } func (indexer *txIndexer) indexByRecipient(recipient string, bz []byte) error { if recipient == "" { return nil } - return indexer.db.Set(indexer.recipientKey(recipient), bz) + return indexer.db.Put(indexer.recipientKey(recipient), bz) } // key helper functions diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 2030f0861..14d102c1b 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -51,6 +51,8 @@ type PersistenceWriteContext interface { Commit() error Release() error + // Question: + UpdateAppHash() ([]byte, error) AppHash() ([]byte, error) @@ -63,7 +65,6 @@ type PersistenceWriteContext interface { // TODO_TEMPORARY: Including two functions for the SQL and KV Store as an interim solution // until we include the schema as part of the SQL Store because persistence // currently has no access to the protobuf schema which is the source of truth. - StoreBlock(blockProtoBytes []byte) error // Store the block in the KV Store InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error // Writes the block in the SQL database // Pool Operations diff --git a/shared/modules/utility_module.go b/shared/modules/utility_module.go index 6a58c9440..538579618 100644 --- a/shared/modules/utility_module.go +++ b/shared/modules/utility_module.go @@ -14,7 +14,6 @@ type UtilityContext interface { // Block operations GetProposalTransactions(proposer []byte, maxTransactionBytes int, lastBlockByzantineValidators [][]byte) (transactions [][]byte, err error) ApplyBlock(height int64, proposer []byte, transactions [][]byte, lastBlockByzantineValidators [][]byte) (appHash []byte, err error) - StoreBlock(blockProtoBytes []byte) error // Context operations ReleaseContext() diff --git a/utility/block.go b/utility/block.go index acf148ca4..00618152c 100644 --- a/utility/block.go +++ b/utility/block.go @@ -296,18 +296,7 @@ func (u *UtilityContext) StoreBlock(blockProtoBytes []byte) error { return err } - // Store in SQL Store - // OPTIMIZE: Ideally we'd pass in the block proto struct to utility so we don't - // have to unmarshal it here, but that's a major design decision for the interfaces. - codec := u.Codec() - block := &typesCons.Block{} - if err := codec.Unmarshal(blockProtoBytes, block); err != nil { - return typesUtil.ErrProtoUnmarshal(err) - } - header := block.BlockHeader - if err := store.InsertBlock(uint64(header.Height), header.Hash, header.ProposerAddress, header.QuorumCertificate); err != nil { - return err - } + return nil } From 6f5e40a842637fc4aeebe9c66022f066d2535221 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 23 Sep 2022 15:23:48 -0700 Subject: [PATCH 10/56] Interim commit while cleaning up consensus module --- Makefile | 3 +- consensus/block.go | 84 ++------------- consensus/consensus_tests/config.json | 9 ++ consensus/consensus_tests/genesis.json | 56 ++++++++++ consensus/consensus_tests/utils_test.go | 2 +- consensus/helpers.go | 11 +- consensus/hotstuff_handler.go | 57 ++++++++++ consensus/hotstuff_leader.go | 92 ++++++++++++---- consensus/hotstuff_replica.go | 80 ++++++++++---- consensus/module.go | 138 ++++++++---------------- consensus/types/block.go | 14 +++ consensus/types/errors.go | 2 - consensus/types/types.go | 4 +- utility/block.go | 16 +-- 14 files changed, 337 insertions(+), 231 deletions(-) create mode 100755 consensus/consensus_tests/config.json create mode 100755 consensus/consensus_tests/genesis.json create mode 100644 consensus/hotstuff_handler.go create mode 100644 consensus/types/block.go diff --git a/Makefile b/Makefile index ac936ae55..311821563 100644 --- a/Makefile +++ b/Makefile @@ -343,9 +343,10 @@ benchmark_p2p_addrbook: # REFACTOR - Similar to TECHDEBT, but will require a substantial rewrite and change across the codebase # CONSIDERATION - A comment that involves extra work but was thoughts / considered as part of some implementation # CONSOLIDATE - We likely have similar implementations/types of the same thing, and we should consolidate them. +# DEPRECATE - Code that should be removed in the future # DISCUSS_IN_THIS_COMMIT - SHOULD NEVER BE COMMITTED TO MASTER. It is a way for the reviewer of a PR to start / reply to a discussion. # TODO_IN_THIS_COMMIT - SHOULD NEVER BE COMMITTED TO MASTER. It is a way to start the review process while non-critical changes are still in progress -TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "TODO_IN_THIS_COMMIT" -e "DISCUSS_IN_THIS_COMMIT" -e "CONSOLIDATE" +TODO_KEYWORDS = -e "TODO" -e "TECHDEBT" -e "IMPROVE" -e "DISCUSS" -e "INCOMPLETE" -e "INVESTIGATE" -e "CLEANUP" -e "HACK" -e "REFACTOR" -e "CONSIDERATION" -e "TODO_IN_THIS_COMMIT" -e "DISCUSS_IN_THIS_COMMIT" -e "CONSOLIDATE" -e "DEPRECATE" # How do I use TODOs? # 1. : ; diff --git a/consensus/block.go b/consensus/block.go index 80b495d95..4ff0a9307 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -1,11 +1,6 @@ package consensus import ( - "encoding/hex" - "unsafe" - - "github.com/pokt-network/pocket/shared/codec" - typesCons "github.com/pokt-network/pocket/consensus/types" ) @@ -17,71 +12,6 @@ func (m *ConsensusModule) validateBlock(block *typesCons.Block) error { return nil } -// This is a helper function intended to be called by a leader/validator during a view change -func (m *ConsensusModule) prepareBlockAsLeader() (*typesCons.Block, error) { - if m.isReplica() { - return nil, typesCons.ErrReplicaPrepareBlock - } - - if err := m.refreshUtilityContext(); err != nil { - return nil, err - } - - txs, err := m.utilityContext.GetProposalTransactions(m.privateKey.Address(), maxTxBytes, lastByzValidators) - if err != nil { - return nil, err - } - - appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) - if err != nil { - return nil, err - } - - blockHeader := &typesCons.BlockHeader{ - Height: int64(m.Height), - Hash: hex.EncodeToString(appHash), - NumTxs: uint32(len(txs)), - LastBlockHash: m.appHash, - ProposerAddress: m.privateKey.Address().Bytes(), - QuorumCertificate: []byte("HACK: Temporary placeholder"), - } - - block := &typesCons.Block{ - BlockHeader: blockHeader, - Transactions: txs, - } - - return block, nil -} - -// This is a helper function intended to be called by a replica/voter during a view change -func (m *ConsensusModule) applyBlockAsReplica(block *typesCons.Block) error { - if m.isLeader() { - return typesCons.ErrLeaderApplyBLock - } - - // TODO(olshansky): Add unit tests to verify this. - if unsafe.Sizeof(*block) > uintptr(m.MaxBlockBytes) { - return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) - } - - if err := m.refreshUtilityContext(); err != nil { - return err - } - - appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), block.BlockHeader.ProposerAddress, block.Transactions, lastByzValidators) - if err != nil { - return err - } - - // DISCUSS(drewsky): Is `ApplyBlock` going to return blockHash or appHash? - if block.BlockHeader.Hash != hex.EncodeToString(appHash) { - return typesCons.ErrInvalidAppHash(block.BlockHeader.Hash, hex.EncodeToString(appHash)) - } - - return nil -} - // Creates a new Utility context and clears/nullifies any previous contexts if they exist func (m *ConsensusModule) refreshUtilityContext() error { // This is a catch-all to release the previous utility context if it wasn't cleaned up @@ -106,21 +36,21 @@ func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { m.nodeLog(typesCons.CommittingBlock(m.Height, len(block.Transactions))) // Store the block in the KV store - codec := codec.GetCodec() - blockProtoBytes, err := codec.Marshal(block) - if err != nil { - return err - } + // codec := codec.GetCodec() + // blockProtoBytes, err := codec.Marshal(block) + // if err != nil { + // return err + // } // Commit and release the context - if err := m.utilityContext.CommitPersistenceContext(blockProtoBytes); err != nil { + if err := m.utilityContext.CommitPersistenceContext(); err != nil { return err } m.utilityContext.ReleaseContext() m.utilityContext = nil - m.appHash = block.BlockHeader.Hash + m.lastAppHash = block.BlockHeader.Hash return nil } diff --git a/consensus/consensus_tests/config.json b/consensus/consensus_tests/config.json new file mode 100755 index 000000000..f3cf88b4a --- /dev/null +++ b/consensus/consensus_tests/config.json @@ -0,0 +1,9 @@ +{ + "consensus": { + "private_key": "70c42b40d1194fcd29a114b0c09fb7a9da32d1bf66039025cf75252cb3e04931b828c891955f35604500a71b9111bddb7bf9481a15250e9508547ea95057f2e9", + "max_mempool_bytes": 500000000, + "pacemaker_config": { + "timeout_msec": 5000 + } + } +} \ No newline at end of file diff --git a/consensus/consensus_tests/genesis.json b/consensus/consensus_tests/genesis.json new file mode 100755 index 000000000..8650b3b52 --- /dev/null +++ b/consensus/consensus_tests/genesis.json @@ -0,0 +1,56 @@ +{ + "consensus_genesis_state": { + "genesis_time": { + "seconds": 1663965677, + "nanos": 641809000 + }, + "chain_id": "testnet", + "max_block_bytes": 4000000, + "validators": [ + { + "address": "c85045886996fe295a4eaa24a7f07ea74032a0bc", + "public_key": "c9356e26cb2987665dc76c2836a1b614646753ab92a517dacecb1a0d472a44fb", + "chains": null, + "generic_param": "node1.consensus:8080", + "staked_amount": "1000000000000", + "paused_height": -1, + "unstaking_height": -1, + "output": "c85045886996fe295a4eaa24a7f07ea74032a0bc", + "actor_type": 3 + }, + { + "address": "e0db8408cea5ed6520e1adc5dc4d1d17239df172", + "public_key": "b828c891955f35604500a71b9111bddb7bf9481a15250e9508547ea95057f2e9", + "chains": null, + "generic_param": "node2.consensus:8080", + "staked_amount": "1000000000000", + "paused_height": -1, + "unstaking_height": -1, + "output": "e0db8408cea5ed6520e1adc5dc4d1d17239df172", + "actor_type": 3 + }, + { + "address": "4a1eb7a5787f4c6e556c3b9a3cbfd31a75a4eef4", + "public_key": "98762a7038ac4df608200a790ccb5b679d4f720d51a7acf115c5be4a6d0222c0", + "chains": null, + "generic_param": "node3.consensus:8080", + "staked_amount": "1000000000000", + "paused_height": -1, + "unstaking_height": -1, + "output": "4a1eb7a5787f4c6e556c3b9a3cbfd31a75a4eef4", + "actor_type": 3 + }, + { + "address": "3b7500e1509f05533c93c8f6d80af06419901b22", + "public_key": "052447ca3b1c8eabb8f5d66d0c34f9ddf466bd03718eb1504ab5a24421a1e5d8", + "chains": null, + "generic_param": "node4.consensus:8080", + "staked_amount": "1000000000000", + "paused_height": -1, + "unstaking_height": -1, + "output": "3b7500e1509f05533c93c8f6d80af06419901b22", + "actor_type": 3 + } + ] + } +} \ No newline at end of file diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index b14d01ab3..dd687953b 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -354,7 +354,7 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti MaxTimes(4) utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() - utilityContextMock.EXPECT().CommitPersistenceContext(gomock.Any()).Return(nil).AnyTimes() + utilityContextMock.EXPECT().CommitPersistenceContext().Return(nil).AnyTimes() utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() utilityContextMock.EXPECT(). GetProposalTransactions(gomock.Any(), maxTxBytes, gomock.AssignableToTypeOf(emptyByzValidators)). diff --git a/consensus/helpers.go b/consensus/helpers.go index 24a84612c..233912ef1 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -2,9 +2,10 @@ package consensus import ( "encoding/base64" - "github.com/pokt-network/pocket/shared/debug" "log" + "github.com/pokt-network/pocket/shared/debug" + "google.golang.org/protobuf/proto" typesCons "github.com/pokt-network/pocket/consensus/types" @@ -31,13 +32,11 @@ const ( var ( HotstuffSteps = [...]typesCons.HotstuffStep{NewRound, Prepare, PreCommit, Commit, Decide} - - maxTxBytes = 90000 // TODO(olshansky): Move this to config.json. - lastByzValidators = make([][]byte, 0) // TODO(olshansky): Retrieve this from persistence ) // ** Hotstuff Helpers ** // +// TODO: Make this method functional (i.e. not have the ConsensusModule receiver) func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.HotstuffStep, round uint64) (*typesCons.QuorumCertificate, error) { var pss []*typesCons.PartialSignature for _, msg := range m.MessagePool[step] { @@ -70,9 +69,9 @@ func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.Hot } return &typesCons.QuorumCertificate{ - Height: m.Height, + Height: height, Step: step, - Round: m.Round, + Round: round, Block: m.Block, ThresholdSignature: thresholdSig, }, nil diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go new file mode 100644 index 000000000..a2127cc61 --- /dev/null +++ b/consensus/hotstuff_handler.go @@ -0,0 +1,57 @@ +package consensus + +import typesCons "github.com/pokt-network/pocket/consensus/types" + +// TODO(discuss): Low priority design: think of a way to make `hotstuff_*` files be a sub-package under consensus. +// This is currently not possible because functions tied to the `ConsensusModule` +// struct (implementing the ConsensusModule module), which spans multiple files. +/* +TODO(discuss): The reason we do not assign both the leader and the replica handlers +to the leader (which should also act as a replica when it is a leader) is because it +can create a weird inconsistent state (e.g. both the replica and leader try to restart +the Pacemaker timeout). This requires additional "replica-like" logic in the leader +handler which has both pros and cons: + Pros: + * The leader can short-circuit and optimize replica related logic + * Avoids additional code flowing through the P2P pipeline + * Allows for micro-optimizations + Cons: + * The leader's "replica related logic" requires an additional code path + * Code is less "generalizable" and therefore potentially more error prone +*/ + +type HotstuffMessageHandler interface { + HandleNewRoundMessage(*ConsensusModule, *typesCons.HotstuffMessage) + HandlePrepareMessage(*ConsensusModule, *typesCons.HotstuffMessage) + HandlePrecommitMessage(*ConsensusModule, *typesCons.HotstuffMessage) + HandleCommitMessage(*ConsensusModule, *typesCons.HotstuffMessage) + HandleDecideMessage(*ConsensusModule, *typesCons.HotstuffMessage) +} + +func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) { + m.nodeLog(typesCons.DebugHandlingHotstuffMessage(msg)) + + // Liveness & safety checks + if err := m.paceMaker.ValidateMessage(msg); err != nil { + // If a replica is not a leader for this round, but has already determined a leader, + // and continues to receive NewRound messages, we avoid logging the "message discard" + // because it creates unnecessary spam. + if !(m.LeaderId != nil && !m.isLeader() && msg.Step == NewRound) { + m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) + } + return + } + + // Need to execute leader election if there is no leader and we are in a new round. + if m.Step == NewRound && m.LeaderId == nil { + m.electNextLeader(msg) + } + + if m.isReplica() { + replicaHandlers[msg.Step](m, msg) + return + } + + // Note that the leader also acts as a replica, but this logic is implemented in the underlying code. + leaderHandlers[msg.Step](m, msg) +} diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index d691ebc0c..a22ece366 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -1,12 +1,15 @@ package consensus import ( + "encoding/hex" "unsafe" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" ) +type HotstuffLeaderMessageHandler struct{} + var ( LeaderMessageHandler HotstuffMessageHandler = &HotstuffLeaderMessageHandler{} leaderHandlers = map[typesCons.HotstuffStep]func(*ConsensusModule, *typesCons.HotstuffMessage){ @@ -18,32 +21,37 @@ var ( } ) -type HotstuffLeaderMessageHandler struct{} - /*** Prepare Step ***/ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation + + // DISCUSS: Do we need to pause for `MinBlockFreqMSec` here to let more transactions or should we stick with optimistic responsiveness? + if err := m.didReceiveEnoughMessageForStep(NewRound); err != nil { m.nodeLog(typesCons.OptimisticVoteCountWaiting(NewRound, err.Error())) return } - - // TODO(olshansky): Do we need to pause for `MinBlockFreqMSec` here to let more transactions come in? m.nodeLog(typesCons.OptimisticVoteCountPassed(NewRound)) + // Clear the previous utility context, if it exists, and create a new one + if err := m.refreshUtilityContext(); err != nil { + return + } + // Likely to be `nil` if blockchain is progressing well. - highPrepareQC := m.findHighQC(NewRound) + highPrepareQC := m.findHighQC(NewRound) // TECHDEBT: How do we validate `highPrepareQC` here? - // TODO(olshansky): Add more unit tests for these checks... + // TODO: Add more unit tests for these checks... if highPrepareQC == nil || highPrepareQC.Height < m.Height || highPrepareQC.Round < m.Round { - block, err := m.prepareBlockAsLeader() + // Leader prepares a new block if `highPrepareQC` is not applicable + block, err := m.prepareAndApplyBlock() if err != nil { m.nodeLogError(typesCons.ErrPrepareBlock.Error(), err) m.paceMaker.InterruptRound() @@ -51,13 +59,17 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM } m.Block = block } else { - // TODO(discuss): Do we need to validate highPrepareQC here? + // Leader acts like a replica if `highPrepareQC` is not `nil` + if err := m.applyBlock(highPrepareQC.Block); err != nil { + m.nodeLogError(typesCons.ErrApplyBlock.Error(), err) + m.paceMaker.InterruptRound() + return + } m.Block = highPrepareQC.Block } m.Step = Prepare m.MessagePool[NewRound] = nil - m.paceMaker.RestartTimer() prepareProposeMessage, err := CreateProposeMessage(m, Prepare, highPrepareQC) if err != nil { @@ -71,7 +83,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM prepareVoteMessage, err := CreateVoteMessage(m, Prepare, m.Block) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Prepare).Error(), err) - return // TODO(olshansky): Should we interrupt the round here? + return } m.sendToNode(prepareVoteMessage) } @@ -80,18 +92,20 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation + if err := m.didReceiveEnoughMessageForStep(Prepare); err != nil { m.nodeLog(typesCons.OptimisticVoteCountWaiting(Prepare, err.Error())) return } m.nodeLog(typesCons.OptimisticVoteCountPassed(Prepare)) + // DISCUSS: What prevents leader from swapping out the block here? prepareQC, err := m.getQuorumCertificate(m.Height, Prepare, m.Round) if err != nil { m.nodeLogError(typesCons.ErrQCInvalid(Prepare).Error(), err) @@ -101,7 +115,6 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo m.Step = PreCommit m.HighPrepareQC = prepareQC m.MessagePool[Prepare] = nil - m.paceMaker.RestartTimer() precommitProposeMessages, err := CreateProposeMessage(m, PreCommit, prepareQC) if err != nil { @@ -115,7 +128,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo precommitVoteMessage, err := CreateVoteMessage(m, PreCommit, m.Block) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(PreCommit).Error(), err) - return // TODO(olshansky): Should we interrupt the round here? + return } m.sendToNode(precommitVoteMessage) } @@ -124,12 +137,13 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation + if err := m.didReceiveEnoughMessageForStep(PreCommit); err != nil { m.nodeLog(typesCons.OptimisticVoteCountWaiting(PreCommit, err.Error())) return @@ -145,7 +159,6 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus m.Step = Commit m.LockedQC = preCommitQC m.MessagePool[PreCommit] = nil - m.paceMaker.RestartTimer() commitProposeMessage, err := CreateProposeMessage(m, Commit, preCommitQC) if err != nil { @@ -168,12 +181,13 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation + if err := m.didReceiveEnoughMessageForStep(Commit); err != nil { m.nodeLog(typesCons.OptimisticVoteCountWaiting(Commit, err.Error())) return @@ -188,7 +202,6 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusMod m.Step = Decide m.MessagePool[Commit] = nil - m.paceMaker.RestartTimer() decideProposeMessage, err := CreateProposeMessage(m, Decide, commitQC) if err != nil { @@ -217,6 +230,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusMod func (handler *HotstuffLeaderMessageHandler) HandleDecideMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -300,3 +314,45 @@ func (m *ConsensusModule) aggregateMessage(msg *typesCons.HotstuffMessage) { // Only the leader needs to aggregate consensus related messages. m.MessagePool[msg.Step] = append(m.MessagePool[msg.Step], msg) } + +// This is a helper function intended to be called by a leader/validator during a view change +// to prepare a new block that is applied to the new underlying context. +func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { + if m.isReplica() { + return nil, typesCons.ErrReplicaPrepareBlock + } + + // TECHDEBT: Retrieve this from consensus consensus config + maxTxBytes := 90000 + + // TECHDEBT: Retrieve this from persistence + lastByzValidators := make([][]byte, 0) + + // Reap the mempool for transactions to be applied in this block + txs, err := m.utilityContext.GetProposalTransactions(m.privateKey.Address(), maxTxBytes, lastByzValidators) + if err != nil { + return nil, err + } + + // Apply all the transactions in the block + appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) + if err != nil { + return nil, err + } + + // Construct the block + blockHeader := &typesCons.BlockHeader{ + Height: int64(m.Height), + Hash: hex.EncodeToString(appHash), + NumTxs: uint32(len(txs)), + LastBlockHash: m.lastAppHash, + ProposerAddress: m.privateKey.Address().Bytes(), + QuorumCertificate: []byte("HACK: Temporary placeholder"), + } + block := &typesCons.Block{ + BlockHeader: blockHeader, + Transactions: txs, + } + + return block, nil +} diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 25041c766..1e814ed41 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -1,8 +1,10 @@ package consensus import ( + "encoding/hex" "fmt" "log" + "unsafe" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" @@ -25,13 +27,18 @@ var ( func (handler *HotstuffReplicaMessageHandler) HandleNewRoundMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation - m.paceMaker.RestartTimer() + + // Clear the previous utility context, if it exists, and create a new one + if err := m.refreshUtilityContext(); err != nil { + return + } + m.Step = Prepare } @@ -39,31 +46,33 @@ func (handler *HotstuffReplicaMessageHandler) HandleNewRoundMessage(m *Consensus func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation + if err := m.validateProposal(msg); err != nil { m.nodeLogError(fmt.Sprintf("Invalid proposal in %s message", Prepare), err) m.paceMaker.InterruptRound() return } - if err := m.applyBlockAsReplica(msg.Block); err != nil { + block := msg.GetBlock() + if err := m.applyBlock(block); err != nil { m.nodeLogError(typesCons.ErrApplyBlock.Error(), err) m.paceMaker.InterruptRound() return } + m.Block = block m.Step = PreCommit - m.paceMaker.RestartTimer() - prepareVoteMessage, err := CreateVoteMessage(m, Prepare, msg.Block) + prepareVoteMessage, err := CreateVoteMessage(m, Prepare, block) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Prepare).Error(), err) - return // TODO(olshansky): Should we interrupt the round here? + return } m.sendToNode(prepareVoteMessage) } @@ -72,26 +81,27 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *ConsensusM func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation - if err := m.validateQuorumCertificate(msg.GetQuorumCertificate()); err != nil { + + quorumCert := msg.GetQuorumCertificate() + if err := m.validateQuorumCertificate(quorumCert); err != nil { m.nodeLogError(typesCons.ErrQCInvalid(PreCommit).Error(), err) m.paceMaker.InterruptRound() return } m.Step = Commit - m.HighPrepareQC = msg.GetQuorumCertificate() // TODO(discuss): Why are we never using this for validation? - m.paceMaker.RestartTimer() + m.HighPrepareQC = quorumCert // INVESTIGATE: Why are we never using this for validation? preCommitVoteMessage, err := CreateVoteMessage(m, PreCommit, msg.Block) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(PreCommit).Error(), err) - return // TODO(olshansky): Should we interrupt the round here? + return } m.sendToNode(preCommitVoteMessage) } @@ -100,26 +110,27 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *Consensu func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation - if err := m.validateQuorumCertificate(msg.GetQuorumCertificate()); err != nil { + + quorumCert := msg.GetQuorumCertificate() + if err := m.validateQuorumCertificate(quorumCert); err != nil { m.nodeLogError(typesCons.ErrQCInvalid(Commit).Error(), err) m.paceMaker.InterruptRound() return } m.Step = Decide - m.LockedQC = msg.GetQuorumCertificate() // TODO(discuss): How do the replica recover if it's locked? Replica `formally` agrees on the QC while the rest of the network `verbally` agrees on the QC. - m.paceMaker.RestartTimer() + m.LockedQC = quorumCert // DISCUSS: How does the replica recover if it's locked? Replica `formally` agrees on the QC while the rest of the network `verbally` agrees on the QC. commitVoteMessage, err := CreateVoteMessage(m, Commit, msg.Block) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Commit).Error(), err) - return // TODO(olshansky): Should we interrupt the round here? + return } m.sendToNode(commitVoteMessage) } @@ -128,13 +139,15 @@ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusMo func (handler *HotstuffReplicaMessageHandler) HandleDecideMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { handler.emitTelemetryEvent(m, msg) + defer m.paceMaker.RestartTimer() if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) return } - // TODO(olshansky): add step specific validation - if err := m.validateQuorumCertificate(msg.GetQuorumCertificate()); err != nil { + + quorumCert := msg.GetQuorumCertificate() + if err := m.validateQuorumCertificate(quorumCert); err != nil { m.nodeLogError(typesCons.ErrQCInvalid(Decide).Error(), err) m.paceMaker.InterruptRound() return @@ -169,7 +182,7 @@ func (handler *HotstuffReplicaMessageHandler) emitTelemetryEvent(m *ConsensusMod } func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error { - if !(msg.Type == Propose && msg.Step == Prepare) { + if !(msg.GetType() == Propose && msg.GetStep() == Prepare) { return typesCons.ErrProposalNotValidInPrepare } @@ -177,8 +190,9 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error return err } - // TODO(discuss): A nil QC implies a successfull CommitQC or TimeoutQC, which have been omitted intentionally since + // TODO(discuss): A nil QC implies a successful CommitQC or TimeoutQC, which have been omitted intentionally since // they are not needed for consensus validity. However, if a QC is specified, it must be valid. + quorumCert := if msg.GetQuorumCertificate() != nil { if err := m.validateQuorumCertificate(msg.GetQuorumCertificate()); err != nil { return err @@ -246,6 +260,30 @@ func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertific return nil } +// This is a helper function intended to be called by a replica/voter during a view change +func (m *ConsensusModule) applyBlock(block *typesCons.Block) error { + // TODO: Add unit tests to verify this. + if unsafe.Sizeof(*block) > uintptr(m.MaxBlockBytes) { + return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) + } + + // TECHDEBT: Retrieve this from persistence + lastByzValidators := make([][]byte, 0) + + // Apply all the transactions in the block and get the appHash + appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), block.BlockHeader.ProposerAddress, block.Transactions, lastByzValidators) + if err != nil { + return err + } + + // CONSOLIDATE: Terminology of `blockHash`, `appHash` and `stateHash` + if block.BlockHeader.Hash != hex.EncodeToString(appHash) { + return typesCons.ErrInvalidAppHash(block.BlockHeader.Hash, hex.EncodeToString(appHash)) + } + + return nil +} + func qcToHotstuffMessage(qc *typesCons.QuorumCertificate) *typesCons.HotstuffMessage { return &typesCons.HotstuffMessage{ Height: qc.Height, diff --git a/consensus/module.go b/consensus/module.go index d2e3e1481..7095c7a4f 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -18,7 +18,7 @@ import ( ) const ( - DefaultLogPrefix = "NODE" // Just a default that'll be replaced during consensus operations. + DefaultLogPrefix = "NODE" // TODO(#164): Make implicit when logging is standardized ConsensusModuleName = "consensus" ) @@ -27,8 +27,7 @@ var _ modules.PacemakerConfig = &typesCons.PacemakerConfig{} var _ modules.ConsensusConfig = &typesCons.ConsensusConfig{} var _ modules.ConsensusModule = &ConsensusModule{} -// TODO(olshansky): Any reason to make all of these attributes local only (i.e. not exposed outside the struct)? -// TODO(olshansky): Look for a way to not externalize the `ConsensusModule` struct +// TODO: Do not export the `ConsensusModule` struct or the fields inside of it. type ConsensusModule struct { bus modules.Bus privateKey cryptoPocket.Ed25519PrivateKey @@ -38,7 +37,7 @@ type ConsensusModule struct { Height uint64 Round uint64 Step typesCons.HotstuffStep - Block *typesCons.Block // The current block being voted on prior to committing to finality + Block *typesCons.Block // The current block being proposed / voted on; it has not been committed to finality HighPrepareQC *typesCons.QuorumCertificate // Highest QC for which replica voted PRECOMMIT LockedQC *typesCons.QuorumCertificate // Highest QC for which replica voted COMMIT @@ -46,11 +45,11 @@ type ConsensusModule struct { // Leader Election LeaderId *typesCons.NodeId NodeId typesCons.NodeId - ValAddrToIdMap typesCons.ValAddrToIdMap // TODO(design): This needs to be updated every time the ValMap is modified - IdToValAddrMap typesCons.IdToValAddrMap // TODO(design): This needs to be updated every time the ValMap is modified + ValAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified + IdToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified // Consensus State - appHash string + lastAppHash string // TODO: Need to make sure this is populated and updated correctly validatorMap typesCons.ValidatorMap // Module Dependencies @@ -58,10 +57,13 @@ type ConsensusModule struct { paceMaker Pacemaker leaderElectionMod leader_election.LeaderElectionModule - logPrefix string // TODO(design): Remove later when we build a shared/proper/injected logger - MessagePool map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage // TODO(design): Move this over to the persistence module or elsewhere? - // TODO(andrew): Explain (or remove) why have an explicit `MaxBlockBytes` if we are already storing a reference to `consCfg` above? - // TODO(andrew): This needs to be updated every time the utility module changes this value. It can be accessed via the "application specific bus" (mimicking the intermodule interface in ABCI) + // DEPRECATE: Remove later when we build a shared/proper/injected logger + logPrefix string + + // TECHDEBT: Move this over to use the txIndexer + MessagePool map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage + + // CLEANUP: Access this value from the configs MaxBlockBytes uint64 } @@ -120,7 +122,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus ValAddrToIdMap: valIdMap, IdToValAddrMap: idValMap, - appHash: "", + lastAppHash: "", validatorMap: valMap, utilityContext: nil, @@ -216,65 +218,11 @@ func (m *ConsensusModule) SetBus(pocketBus modules.Bus) { m.leaderElectionMod.SetBus(pocketBus) } -func (m *ConsensusModule) loadPersistedState() error { - persistenceContext, err := m.GetBus().GetPersistenceModule().NewReadContext(-1) // Unknown height - if err != nil { - return nil - } - defer persistenceContext.Close() - - latestHeight, err := persistenceContext.GetLatestBlockHeight() - if err != nil || latestHeight == 0 { - m.nodeLog("TODO: State sync not implement") - return nil - } - - appHash, err := persistenceContext.GetBlockHash(int64(latestHeight)) - if err != nil { - return fmt.Errorf("error getting block hash for height %d even though it's in the database: %s", latestHeight, err) - } - - // TODO: Populate the rest of the state from the persistence module: validator set, quorum cert, last block hash, etc... - m.Height = uint64(latestHeight) + 1 // +1 because the height of the consensus module is where it is actively participating in consensus - m.appHash = string(appHash) - - m.nodeLog(fmt.Sprintf("Starting node at height %d", latestHeight)) - return nil -} - -// TODO(discuss): Low priority design: think of a way to make `hotstuff_*` files be a sub-package under consensus. -// This is currently not possible because functions tied to the `ConsensusModule` -// struct (implementing the ConsensusModule module), which spans multiple files. -/* -TODO(discuss): The reason we do not assign both the leader and the replica handlers -to the leader (which should also act as a replica when it is a leader) is because it -can create a weird inconsistent state (e.g. both the replica and leader try to restart -the Pacemaker timeout). This requires additional "replica-like" logic in the leader -handler which has both pros and cons: - Pros: - * The leader can short-circuit and optimize replica related logic - * Avoids additional code flowing through the P2P pipeline - * Allows for micro-optimizations - Cons: - * The leader's "replica related logic" requires an additional code path - * Code is less "generalizable" and therefore potentially more error prone -*/ - -// TODO(olshansky): Should we just make these singletons or embed them directly in the ConsensusModule? -type HotstuffMessageHandler interface { - HandleNewRoundMessage(*ConsensusModule, *typesCons.HotstuffMessage) - HandlePrepareMessage(*ConsensusModule, *typesCons.HotstuffMessage) - HandlePrecommitMessage(*ConsensusModule, *typesCons.HotstuffMessage) - HandleCommitMessage(*ConsensusModule, *typesCons.HotstuffMessage) - HandleDecideMessage(*ConsensusModule, *typesCons.HotstuffMessage) -} - func (m *ConsensusModule) HandleMessage(message *anypb.Any) error { switch message.MessageName() { case HotstuffMessage: var hotstuffMessage typesCons.HotstuffMessage - err := anypb.UnmarshalTo(message, &hotstuffMessage, proto.UnmarshalOptions{}) - if err != nil { + if err := anypb.UnmarshalTo(message, &hotstuffMessage, proto.UnmarshalOptions{}); err != nil { return err } m.handleHotstuffMessage(&hotstuffMessage) @@ -287,36 +235,8 @@ func (m *ConsensusModule) HandleMessage(message *anypb.Any) error { return nil } -func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) { - m.nodeLog(typesCons.DebugHandlingHotstuffMessage(msg)) - - // Liveness & safety checks - if err := m.paceMaker.ValidateMessage(msg); err != nil { - // If a replica is not a leader for this round, but has already determined a leader, - // and continues to receive NewRound messages, we avoid logging the "message discard" - // because it creates unnecessary spam. - if !(m.LeaderId != nil && !m.isLeader() && msg.Step == NewRound) { - m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) - } - return - } - - // Need to execute leader election if there is no leader and we are in a new round. - if m.Step == NewRound && m.LeaderId == nil { - m.electNextLeader(msg) - } - - if m.isReplica() { - replicaHandlers[msg.Step](m, msg) - return - } - - // Note that the leader also acts as a replica, but this logic is implemented in the underlying code. - leaderHandlers[msg.Step](m, msg) -} - func (m *ConsensusModule) AppHash() string { - return m.appHash + return m.lastAppHash } func (m *ConsensusModule) CurrentHeight() uint64 { @@ -326,3 +246,29 @@ func (m *ConsensusModule) CurrentHeight() uint64 { func (m *ConsensusModule) ValidatorMap() modules.ValidatorMap { return typesCons.ValidatorMapToModulesValidatorMap(m.validatorMap) } + +// TODO: Populate the entire state from the persistence module: validator set, quorum cert, last block hash, etc... +func (m *ConsensusModule) loadPersistedState() error { + persistenceContext, err := m.GetBus().GetPersistenceModule().NewReadContext(-1) // Unknown height + if err != nil { + return nil + } + defer persistenceContext.Close() + + latestHeight, err := persistenceContext.GetLatestBlockHeight() + if err != nil || latestHeight == 0 { + m.nodeLog("TODO: State sync not implement") + return nil + } + + appHash, err := persistenceContext.GetBlockHash(int64(latestHeight)) + if err != nil { + return fmt.Errorf("error getting block hash for height %d even though it's in the database: %s", latestHeight, err) + } + + m.Height = uint64(latestHeight) + 1 // +1 because the height of the consensus module is where it is actively participating in consensus + m.lastAppHash = string(appHash) + + m.nodeLog(fmt.Sprintf("Starting node at height %d", latestHeight)) + return nil +} diff --git a/consensus/types/block.go b/consensus/types/block.go new file mode 100644 index 000000000..f6091b6b3 --- /dev/null +++ b/consensus/types/block.go @@ -0,0 +1,14 @@ +package types + +import ( + "github.com/pokt-network/pocket/shared/codec" +) + +func (b *Block) Bytes() ([]byte, error) { + codec := codec.GetCodec() + blockProtoBz, err := codec.Marshal(b) + if err != nil { + return nil, err + } + return blockProtoBz, nil +} diff --git a/consensus/types/errors.go b/consensus/types/errors.go index e3f4d3987..1556295c4 100644 --- a/consensus/types/errors.go +++ b/consensus/types/errors.go @@ -142,7 +142,6 @@ const ( prepareBlockError = "could not prepare block" commitBlockError = "could not commit block" replicaPrepareBlockError = "node should not call `prepareBlock` if it is not a leader" - leaderErrApplyBlock = "node should not call `applyBlock` if it is leader" blockSizeTooLargeError = "block size is too large" sendMessageError = "error sending message" broadcastMessageError = "error broadcasting message" @@ -177,7 +176,6 @@ var ( ErrPrepareBlock = errors.New(prepareBlockError) ErrCommitBlock = errors.New(commitBlockError) ErrReplicaPrepareBlock = errors.New(replicaPrepareBlockError) - ErrLeaderApplyBLock = errors.New(leaderErrApplyBlock) ErrSendMessage = errors.New(sendMessageError) ErrBroadcastMessage = errors.New(broadcastMessageError) ErrCreateConsensusMessage = errors.New(createConsensusMessageError) diff --git a/consensus/types/types.go b/consensus/types/types.go index 36896cfaf..86b9a0624 100644 --- a/consensus/types/types.go +++ b/consensus/types/types.go @@ -1,8 +1,10 @@ package types +// TODO: Split this file into multiple types files. import ( - "github.com/pokt-network/pocket/shared/modules" "sort" + + "github.com/pokt-network/pocket/shared/modules" ) type NodeId uint64 diff --git a/utility/block.go b/utility/block.go index 00618152c..c762b35a7 100644 --- a/utility/block.go +++ b/utility/block.go @@ -3,7 +3,7 @@ package utility import ( "math/big" - typesCons "github.com/pokt-network/pocket/consensus/types" // TODO (andrew) importing consensus and persistence in this file? + // TODO (andrew) importing consensus and persistence in this file? typesGenesis "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" @@ -61,11 +61,13 @@ func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, // return nil, err // } } + // end block lifecycle phase if err := u.EndBlock(proposerAddress); err != nil { return nil, err } - // return the app hash (consensus module will get the validator set directly + + // return the app hash; consensus module will get the validator set directly return u.GetAppHash() } @@ -289,14 +291,12 @@ func (u *UtilityContext) SetValidatorMissedBlocks(address []byte, missedBlocks i } func (u *UtilityContext) StoreBlock(blockProtoBytes []byte) error { - store := u.Store() + // store := u.Store() // Store in KV Store - if err := store.StoreBlock(blockProtoBytes); err != nil { - return err - } - - + // if err := store.StoreBlock(blockProtoBytes); err != nil { + // return err + // } return nil } From ed735e105ddad84c8acec7198a6002f330629886 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 23 Sep 2022 15:49:02 -0700 Subject: [PATCH 11/56] Tests running but not passing --- consensus/consensus_tests/config.json | 2 +- consensus/consensus_tests/genesis.json | 28 ++++++------- consensus/helpers.go | 6 ++- consensus/hotstuff_leader.go | 25 +++++------ consensus/hotstuff_replica.go | 20 ++++----- consensus/messages.go | 57 ++++++++++++++------------ consensus/module.go | 2 +- 7 files changed, 73 insertions(+), 67 deletions(-) diff --git a/consensus/consensus_tests/config.json b/consensus/consensus_tests/config.json index f3cf88b4a..688a66b27 100755 --- a/consensus/consensus_tests/config.json +++ b/consensus/consensus_tests/config.json @@ -1,6 +1,6 @@ { "consensus": { - "private_key": "70c42b40d1194fcd29a114b0c09fb7a9da32d1bf66039025cf75252cb3e04931b828c891955f35604500a71b9111bddb7bf9481a15250e9508547ea95057f2e9", + "private_key": "e8561cc9a8cb6396ceeae0747e8365029fb7772f40ee7e82df906fbd6b329a5a8ae7e70016d48453e3fe173d1457a7426c0479623c6ca5ba042d5303ebb0814b", "max_mempool_bytes": 500000000, "pacemaker_config": { "timeout_msec": 5000 diff --git a/consensus/consensus_tests/genesis.json b/consensus/consensus_tests/genesis.json index 8650b3b52..88156506b 100755 --- a/consensus/consensus_tests/genesis.json +++ b/consensus/consensus_tests/genesis.json @@ -1,54 +1,54 @@ { "consensus_genesis_state": { "genesis_time": { - "seconds": 1663965677, - "nanos": 641809000 + "seconds": 1663973128, + "nanos": 273119000 }, "chain_id": "testnet", "max_block_bytes": 4000000, "validators": [ { - "address": "c85045886996fe295a4eaa24a7f07ea74032a0bc", - "public_key": "c9356e26cb2987665dc76c2836a1b614646753ab92a517dacecb1a0d472a44fb", + "address": "27ce0b388a1a3b2c663f755d78206e96eea4362a", + "public_key": "bb6329890486544b0101e23f26fd5a74f0355843d6b63e48ca95954c6fea6ee2", "chains": null, "generic_param": "node1.consensus:8080", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, - "output": "c85045886996fe295a4eaa24a7f07ea74032a0bc", + "output": "27ce0b388a1a3b2c663f755d78206e96eea4362a", "actor_type": 3 }, { - "address": "e0db8408cea5ed6520e1adc5dc4d1d17239df172", - "public_key": "b828c891955f35604500a71b9111bddb7bf9481a15250e9508547ea95057f2e9", + "address": "210658c6938dd7b02fad44774c06b8d33067c283", + "public_key": "26b785bddc4f48911dcb7c99ad7657f4cce39f3af659ef17ccec26f44aeb4dee", "chains": null, "generic_param": "node2.consensus:8080", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, - "output": "e0db8408cea5ed6520e1adc5dc4d1d17239df172", + "output": "210658c6938dd7b02fad44774c06b8d33067c283", "actor_type": 3 }, { - "address": "4a1eb7a5787f4c6e556c3b9a3cbfd31a75a4eef4", - "public_key": "98762a7038ac4df608200a790ccb5b679d4f720d51a7acf115c5be4a6d0222c0", + "address": "9c17a6256e9fddf888608d7573f403f35b57a5ce", + "public_key": "92e1f441b5002f110e3321ed122ac3e2dba1a800a7477ffcae0dfb5ccfcde95c", "chains": null, "generic_param": "node3.consensus:8080", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, - "output": "4a1eb7a5787f4c6e556c3b9a3cbfd31a75a4eef4", + "output": "9c17a6256e9fddf888608d7573f403f35b57a5ce", "actor_type": 3 }, { - "address": "3b7500e1509f05533c93c8f6d80af06419901b22", - "public_key": "052447ca3b1c8eabb8f5d66d0c34f9ddf466bd03718eb1504ab5a24421a1e5d8", + "address": "e4aa8d7cebb13f1832fe91695818ce09fb1c444b", + "public_key": "8ae7e70016d48453e3fe173d1457a7426c0479623c6ca5ba042d5303ebb0814b", "chains": null, "generic_param": "node4.consensus:8080", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, - "output": "3b7500e1509f05533c93c8f6d80af06419901b22", + "output": "e4aa8d7cebb13f1832fe91695818ce09fb1c444b", "actor_type": 3 } ] diff --git a/consensus/helpers.go b/consensus/helpers.go index 233912ef1..dd90c9b66 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -97,13 +97,13 @@ func getThresholdSignature( return thresholdSig, nil } -func isSignatureValid(m *typesCons.HotstuffMessage, pubKeyString string, signature []byte) bool { +func isSignatureValid(msg *typesCons.HotstuffMessage, pubKeyString string, signature []byte) bool { pubKey, err := cryptoPocket.NewPublicKey(pubKeyString) if err != nil { log.Println("[WARN] Error getting PublicKey from bytes:", err) return false } - bytesToVerify, err := getSignableBytes(m) + bytesToVerify, err := getSignableBytes(msg) if err != nil { log.Println("[WARN] Error getting bytes to verify:", err) return false @@ -211,10 +211,12 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) { /*** General Infrastructure Helpers ***/ +// TODO(#164): Remove this once we have a proper logging system. func (m *ConsensusModule) nodeLog(s string) { log.Printf("[%s][%d] %s\n", m.logPrefix, m.NodeId, s) } +// TODO(#164): Remove this once we have a proper logging system. func (m *ConsensusModule) nodeLogError(s string, err error) { log.Printf("[ERROR][%s][%d] %s: %v\n", m.logPrefix, m.NodeId, s, err) } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index a22ece366..8b157e21b 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -71,7 +71,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM m.Step = Prepare m.MessagePool[NewRound] = nil - prepareProposeMessage, err := CreateProposeMessage(m, Prepare, highPrepareQC) + prepareProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Prepare, m.Block, highPrepareQC) if err != nil { m.nodeLogError(typesCons.ErrCreateProposeMessage(Prepare).Error(), err) m.paceMaker.InterruptRound() @@ -80,7 +80,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM m.broadcastToNodes(prepareProposeMessage) // Leader also acts like a replica - prepareVoteMessage, err := CreateVoteMessage(m, Prepare, m.Block) + prepareVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Prepare, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Prepare).Error(), err) return @@ -116,7 +116,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo m.HighPrepareQC = prepareQC m.MessagePool[Prepare] = nil - precommitProposeMessages, err := CreateProposeMessage(m, PreCommit, prepareQC) + precommitProposeMessages, err := CreateProposeMessage(m.Height, m.Round, PreCommit, m.Block, prepareQC) if err != nil { m.nodeLogError(typesCons.ErrCreateProposeMessage(PreCommit).Error(), err) m.paceMaker.InterruptRound() @@ -125,7 +125,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo m.broadcastToNodes(precommitProposeMessages) // Leader also acts like a replica - precommitVoteMessage, err := CreateVoteMessage(m, PreCommit, m.Block) + precommitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, PreCommit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(PreCommit).Error(), err) return @@ -160,7 +160,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus m.LockedQC = preCommitQC m.MessagePool[PreCommit] = nil - commitProposeMessage, err := CreateProposeMessage(m, Commit, preCommitQC) + commitProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Commit, m.Block, preCommitQC) if err != nil { m.nodeLogError(typesCons.ErrCreateProposeMessage(Commit).Error(), err) m.paceMaker.InterruptRound() @@ -169,7 +169,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus m.broadcastToNodes(commitProposeMessage) // Leader also acts like a replica - commitVoteMessage, err := CreateVoteMessage(m, Commit, m.Block) + commitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Commit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Commit).Error(), err) return // TODO(olshansky): Should we interrupt the round here? @@ -203,7 +203,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusMod m.Step = Decide m.MessagePool[Commit] = nil - decideProposeMessage, err := CreateProposeMessage(m, Decide, commitQC) + decideProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Decide, m.Block, commitQC) if err != nil { m.nodeLogError(typesCons.ErrCreateProposeMessage(Decide).Error(), err) m.paceMaker.InterruptRound() @@ -270,12 +270,12 @@ func (handler *HotstuffLeaderMessageHandler) validateBasic(m *ConsensusModule, m } func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessage) error { - if msg.Step == NewRound { + if msg.GetStep() == NewRound { m.nodeLog(typesCons.ErrUnnecessaryPartialSigForNewRound.Error()) return nil } - if msg.Type == Propose { + if msg.GetType() == Propose { m.nodeLog(typesCons.ErrUnnecessaryPartialSigForLeaderProposal.Error()) return nil } @@ -283,18 +283,19 @@ func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessag if msg.GetPartialSignature() == nil { return typesCons.ErrNilPartialSig } + partialSig := msg.GetPartialSignature() - if msg.GetPartialSignature().Signature == nil || len(msg.GetPartialSignature().Address) == 0 { + if partialSig.Signature == nil || len(partialSig.GetAddress()) == 0 { return typesCons.ErrNilPartialSigOrSourceNotSpecified } - address := msg.GetPartialSignature().Address + address := partialSig.GetAddress() validator, ok := m.validatorMap[address] if !ok { return typesCons.ErrMissingValidator(address, m.ValAddrToIdMap[address]) } pubKey := validator.GetPublicKey() - if isSignatureValid(msg, pubKey, msg.GetPartialSignature().Signature) { + if isSignatureValid(msg, pubKey, partialSig.GetSignature()) { return nil } diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 1e814ed41..cb333af70 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -69,7 +69,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *ConsensusM m.Step = PreCommit - prepareVoteMessage, err := CreateVoteMessage(m, Prepare, block) + prepareVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Prepare, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Prepare).Error(), err) return @@ -98,7 +98,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *Consensu m.Step = Commit m.HighPrepareQC = quorumCert // INVESTIGATE: Why are we never using this for validation? - preCommitVoteMessage, err := CreateVoteMessage(m, PreCommit, msg.Block) + preCommitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, PreCommit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(PreCommit).Error(), err) return @@ -127,7 +127,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusMo m.Step = Decide m.LockedQC = quorumCert // DISCUSS: How does the replica recover if it's locked? Replica `formally` agrees on the QC while the rest of the network `verbally` agrees on the QC. - commitVoteMessage, err := CreateVoteMessage(m, Commit, msg.Block) + commitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Commit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Commit).Error(), err) return @@ -153,7 +153,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleDecideMessage(m *ConsensusMo return } - if err := m.commitBlock(msg.Block); err != nil { + if err := m.commitBlock(m.Block); err != nil { m.nodeLogError("Could not commit block", err) m.paceMaker.InterruptRound() return @@ -186,21 +186,21 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error return typesCons.ErrProposalNotValidInPrepare } - if err := m.validateBlock(msg.Block); err != nil { + if err := m.validateBlock(msg.GetBlock()); err != nil { return err } // TODO(discuss): A nil QC implies a successful CommitQC or TimeoutQC, which have been omitted intentionally since // they are not needed for consensus validity. However, if a QC is specified, it must be valid. - quorumCert := - if msg.GetQuorumCertificate() != nil { - if err := m.validateQuorumCertificate(msg.GetQuorumCertificate()); err != nil { + quorumCert := msg.GetQuorumCertificate() + if quorumCert != nil { + if err := m.validateQuorumCertificate(quorumCert); err != nil { return err } } lockedQC := m.LockedQC - justifyQC := msg.GetQuorumCertificate() + justifyQC := quorumCert // Safety: not locked if lockedQC == nil { @@ -210,7 +210,7 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error // Safety: check the hash of the locked QC // TODO(olshansky): Extend implementation to adopt `ExtendsFrom` as described in the Hotstuff whitepaper. - if protoHash(lockedQC.Block) == protoHash(justifyQC.Block) { // && lockedQC.Block.ExtendsFrom(justifyQC.Block) + if protoHash(lockedQC.GetBlock()) == protoHash(justifyQC.Block) { // && lockedQC.Block.ExtendsFrom(justifyQC.Block) m.nodeLog(typesCons.ProposalBlockExtends) return nil } diff --git a/consensus/messages.go b/consensus/messages.go index 4fd09cea5..9a577e996 100644 --- a/consensus/messages.go +++ b/consensus/messages.go @@ -9,30 +9,29 @@ import ( ) func CreateProposeMessage( - m *ConsensusModule, - step typesCons.HotstuffStep, // step can be taken from `m` but is specified explicitly via interface to avoid ambiguity + height uint64, + round uint64, + step typesCons.HotstuffStep, + block *typesCons.Block, qc *typesCons.QuorumCertificate, ) (*typesCons.HotstuffMessage, error) { - if m.Block == nil { - return nil, typesCons.ErrNilBlockProposal - } - msg := &typesCons.HotstuffMessage{ Type: Propose, - Height: m.Height, + Height: height, Step: step, - Round: m.Round, - Block: m.Block, + Round: round, + Block: block, Justification: nil, // QC is set below if it is non-nil } - // TODO(olshansky): Add unit tests for this + // TODO: Add unit tests for this if qc == nil && step != Prepare { return nil, typesCons.ErrNilQCProposal } - // TODO(olshansky): Add unit tests for this - if qc != nil { // QC may optionally be nil for NEWROUND steps when everything is progressing smoothly + // TODO: Add unit tests for this + // QC may be nil during NEWROUND if following happy hotstuff path + if qc != nil { msg.Justification = &typesCons.HotstuffMessage_QuorumCertificate{ QuorumCertificate: qc, } @@ -42,9 +41,11 @@ func CreateProposeMessage( } func CreateVoteMessage( - m *ConsensusModule, - step typesCons.HotstuffStep, // step can be taken from `m` but is specified explicitly via interface to avoid ambiguity + height uint64, + round uint64, + step typesCons.HotstuffStep, block *typesCons.Block, + privKey crypto.PrivateKey, // used to sign the vote ) (*typesCons.HotstuffMessage, error) { if block == nil { return nil, typesCons.ErrNilBlockVote @@ -52,44 +53,46 @@ func CreateVoteMessage( msg := &typesCons.HotstuffMessage{ Type: Vote, - Height: m.Height, + Height: height, Step: step, - Round: m.Round, + Round: round, Block: block, Justification: nil, // signature is computed below } msg.Justification = &typesCons.HotstuffMessage_PartialSignature{ PartialSignature: &typesCons.PartialSignature{ - Signature: getMessageSignature(msg, m.privateKey), - Address: m.privateKey.PublicKey().Address().String(), + Signature: getMessageSignature(msg, privKey), + Address: privKey.PublicKey().Address().String(), }, } return msg, nil } -// Returns a "partial" signature of the hotstuff message from one of the validators -func getMessageSignature(m *typesCons.HotstuffMessage, privKey crypto.PrivateKey) []byte { - bytesToSign, err := getSignableBytes(m) +// Returns "partial" signature of the hotstuff message from one of the validators +func getMessageSignature(msg *typesCons.HotstuffMessage, privKey crypto.PrivateKey) []byte { + bytesToSign, err := getSignableBytes(msg) if err != nil { return nil } + signature, err := privKey.Sign(bytesToSign) if err != nil { log.Fatalf("Error signing message: %v", err) return nil } + return signature } -// Signature should only be over a subset of the fields in a HotstuffMessage -func getSignableBytes(m *typesCons.HotstuffMessage) ([]byte, error) { +// Signature only over subset of fields in HotstuffMessage +func getSignableBytes(msg *typesCons.HotstuffMessage) ([]byte, error) { msgToSign := &typesCons.HotstuffMessage{ - Height: m.Height, - Step: m.Step, - Round: m.Round, - Block: m.Block, + Height: msg.Height, + Step: msg.Step, + Round: msg.Round, + Block: msg.Block, } return proto.Marshal(msgToSign) } diff --git a/consensus/module.go b/consensus/module.go index 7095c7a4f..42c7354ae 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -257,7 +257,7 @@ func (m *ConsensusModule) loadPersistedState() error { latestHeight, err := persistenceContext.GetLatestBlockHeight() if err != nil || latestHeight == 0 { - m.nodeLog("TODO: State sync not implement") + m.nodeLog("TODO: State sync not implemented yet") return nil } From e1016e6b01d0e3ea41ae742de2269d04b8be8b6b Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 23 Sep 2022 15:52:15 -0700 Subject: [PATCH 12/56] Using proper getters when accessing hotstuff --- consensus/helpers.go | 2 +- consensus/hotstuff_handler.go | 7 ++++--- consensus/hotstuff_leader.go | 3 ++- consensus/messages.go | 8 ++++---- consensus/types/errors.go | 14 +++++++------- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/consensus/helpers.go b/consensus/helpers.go index dd90c9b66..34c36e054 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -46,7 +46,7 @@ func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.Hot continue } // TODO(olshansky): Add tests for this - if msg.Height != height || msg.Step != step || msg.Round != round { + if msg.GetHeight() != height || msg.GetStep() != step || msg.GetRound() != round { m.nodeLog(typesCons.WarnUnexpectedMessageInPool(msg, height, step, round)) continue } diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index a2127cc61..9b5ae3287 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -31,12 +31,13 @@ type HotstuffMessageHandler interface { func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) { m.nodeLog(typesCons.DebugHandlingHotstuffMessage(msg)) + step := msg.GetStep() // Liveness & safety checks if err := m.paceMaker.ValidateMessage(msg); err != nil { // If a replica is not a leader for this round, but has already determined a leader, // and continues to receive NewRound messages, we avoid logging the "message discard" // because it creates unnecessary spam. - if !(m.LeaderId != nil && !m.isLeader() && msg.Step == NewRound) { + if !(m.LeaderId != nil && !m.isLeader() && step == NewRound) { m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) } return @@ -48,10 +49,10 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) } if m.isReplica() { - replicaHandlers[msg.Step](m, msg) + replicaHandlers[step](m, msg) return } // Note that the leader also acts as a replica, but this logic is implemented in the underlying code. - leaderHandlers[msg.Step](m, msg) + leaderHandlers[step](m, msg) } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 8b157e21b..144da8e07 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -313,7 +313,8 @@ func (m *ConsensusModule) aggregateMessage(msg *typesCons.HotstuffMessage) { } // Only the leader needs to aggregate consensus related messages. - m.MessagePool[msg.Step] = append(m.MessagePool[msg.Step], msg) + step := msg.GetStep() + m.MessagePool[step] = append(m.MessagePool[step], msg) } // This is a helper function intended to be called by a leader/validator during a view change diff --git a/consensus/messages.go b/consensus/messages.go index 9a577e996..8020ec068 100644 --- a/consensus/messages.go +++ b/consensus/messages.go @@ -89,10 +89,10 @@ func getMessageSignature(msg *typesCons.HotstuffMessage, privKey crypto.PrivateK // Signature only over subset of fields in HotstuffMessage func getSignableBytes(msg *typesCons.HotstuffMessage) ([]byte, error) { msgToSign := &typesCons.HotstuffMessage{ - Height: msg.Height, - Step: msg.Step, - Round: msg.Round, - Block: msg.Block, + Height: msg.GetHeight(), + Step: msg.GetStep(), + Round: msg.GetRound(), + Block: msg.GetBlock(), } return proto.Marshal(msgToSign) } diff --git a/consensus/types/errors.go b/consensus/types/errors.go index 1556295c4..d90d85151 100644 --- a/consensus/types/errors.go +++ b/consensus/types/errors.go @@ -71,11 +71,11 @@ func ElectedSelfAsNewLeader(address string, nodeId NodeId, height, round uint64) } func SendingMessage(msg *HotstuffMessage, nodeId NodeId) string { - return fmt.Sprintf("Sending %s message to %d", StepToString[msg.Step], nodeId) + return fmt.Sprintf("Sending %s message to %d", StepToString[msg.GetStep()], nodeId) } func BroadcastingMessage(msg *HotstuffMessage) string { - return fmt.Sprintf("Broadcasting message for %s step", StepToString[msg.Step]) + return fmt.Sprintf("Broadcasting message for %s step", StepToString[msg.GetStep()]) } func WarnInvalidPartialSigInQC(address string, nodeId NodeId) string { @@ -83,7 +83,7 @@ func WarnInvalidPartialSigInQC(address string, nodeId NodeId) string { } func WarnMissingPartialSig(msg *HotstuffMessage) string { - return fmt.Sprintf("[WARN] No partial signature found for step %s which should not happen...", StepToString[msg.Step]) + return fmt.Sprintf("[WARN] No partial signature found for step %s which should not happen...", StepToString[msg.GetStep()]) } func WarnDiscardHotstuffMessage(_ *HotstuffMessage, reason string) string { @@ -95,7 +95,7 @@ func WarnUnexpectedMessageInPool(_ *HotstuffMessage, height uint64, step Hotstuf } func WarnIncompletePartialSig(ps *PartialSignature, msg *HotstuffMessage) string { - return fmt.Sprintf("[WARN] Partial signature is incomplete for step %s which should not happen...", StepToString[msg.Step]) + return fmt.Sprintf("[WARN] Partial signature is incomplete for step %s which should not happen...", StepToString[msg.GetStep()]) } func DebugTogglePacemakerManualMode(mode string) string { @@ -108,7 +108,7 @@ func DebugNodeState(state ConsensusNodeState) string { func DebugHandlingHotstuffMessage(msg *HotstuffMessage) string { // TODO(olshansky): Add source and destination NodeId of message here - return fmt.Sprintf("[DEBUG] Handling message w/ Height: %d; Type: %s; Round: %d.", msg.Height, StepToString[msg.Step], msg.Round) + return fmt.Sprintf("[DEBUG] Handling message w/ Height: %d; Type: %s; Round: %d.", msg.Height, StepToString[msg.GetStep()], msg.Round) } // Errors @@ -201,7 +201,7 @@ func ErrMissingValidator(address string, nodeId NodeId) error { func ErrValidatingPartialSig(senderAddr string, senderNodeId NodeId, msg *HotstuffMessage, pubKey string) error { return fmt.Errorf("%s: Sender: %s (%d); Height: %d; Step: %s; Round: %d; SigHash: %s; BlockHash: %s; PubKey: %s", - invalidPartialSignatureError, senderAddr, senderNodeId, msg.Height, StepToString[msg.Step], msg.Round, string(msg.GetPartialSignature().Signature), protoHash(msg.Block), pubKey) + invalidPartialSignatureError, senderAddr, senderNodeId, msg.Height, StepToString[msg.GetStep()], msg.Round, string(msg.GetPartialSignature().Signature), protoHash(msg.Block), pubKey) } func ErrPacemakerUnexpectedMessageHeight(err error, heightCurrent, heightMessage uint64) error { @@ -209,7 +209,7 @@ func ErrPacemakerUnexpectedMessageHeight(err error, heightCurrent, heightMessage } func ErrPacemakerUnexpectedMessageStepRound(err error, step HotstuffStep, round uint64, msg *HotstuffMessage) error { - return fmt.Errorf("%s: Current (step, round): (%s, %d); Message (step, round): (%s, %d)", err, StepToString[step], round, StepToString[msg.Step], msg.Round) + return fmt.Errorf("%s: Current (step, round): (%s, %d); Message (step, round): (%s, %d)", err, StepToString[step], round, StepToString[msg.GetStep()], msg.Round) } func ErrUnknownConsensusMessageType(msg interface{}) error { From a58926d09b9629c275b93353fc2d0c387ebaa0f1 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 23 Sep 2022 17:22:22 -0700 Subject: [PATCH 13/56] make develop_test passed --- consensus/block.go | 23 ++++--- consensus/consensus_tests/config.json | 9 --- consensus/consensus_tests/genesis.json | 56 ----------------- consensus/consensus_tests/pacemaker_test.go | 7 +++ consensus/debugging.go | 3 +- consensus/helpers.go | 6 +- consensus/hotstuff_handler.go | 35 ++++------- consensus/hotstuff_leader.go | 16 ++--- consensus/hotstuff_replica.go | 70 +++++++++++---------- consensus/messages.go | 6 +- consensus/module.go | 8 ++- 11 files changed, 93 insertions(+), 146 deletions(-) delete mode 100755 consensus/consensus_tests/config.json delete mode 100755 consensus/consensus_tests/genesis.json diff --git a/consensus/block.go b/consensus/block.go index 4ff0a9307..ce123c1d7 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -1,14 +1,21 @@ package consensus import ( + "unsafe" + typesCons "github.com/pokt-network/pocket/consensus/types" ) -// TODO(olshansky): Sync with Andrew on the type of validation we need here. +// TODO: Add additional basic block metadata validation w/ unit tests func (m *ConsensusModule) validateBlock(block *typesCons.Block) error { if block == nil { return typesCons.ErrNilBlock } + + if unsafe.Sizeof(*block) > uintptr(m.MaxBlockBytes) { + return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) + } + return nil } @@ -17,10 +24,10 @@ func (m *ConsensusModule) refreshUtilityContext() error { // This is a catch-all to release the previous utility context if it wasn't cleaned up // in the proper lifecycle (e.g. catch up, error, network partition, etc...). Ideally, this // should not be called. - if m.utilityContext != nil { + if m.UtilityContext != nil { m.nodeLog(typesCons.NilUtilityContextWarning) - m.utilityContext.ReleaseContext() - m.utilityContext = nil + m.UtilityContext.ReleaseContext() + m.UtilityContext = nil } utilityContext, err := m.GetBus().GetUtilityModule().NewContext(int64(m.Height)) @@ -28,7 +35,7 @@ func (m *ConsensusModule) refreshUtilityContext() error { return err } - m.utilityContext = utilityContext + m.UtilityContext = utilityContext return nil } @@ -43,12 +50,12 @@ func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { // } // Commit and release the context - if err := m.utilityContext.CommitPersistenceContext(); err != nil { + if err := m.UtilityContext.CommitPersistenceContext(); err != nil { return err } - m.utilityContext.ReleaseContext() - m.utilityContext = nil + m.UtilityContext.ReleaseContext() + m.UtilityContext = nil m.lastAppHash = block.BlockHeader.Hash diff --git a/consensus/consensus_tests/config.json b/consensus/consensus_tests/config.json deleted file mode 100755 index 688a66b27..000000000 --- a/consensus/consensus_tests/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "consensus": { - "private_key": "e8561cc9a8cb6396ceeae0747e8365029fb7772f40ee7e82df906fbd6b329a5a8ae7e70016d48453e3fe173d1457a7426c0479623c6ca5ba042d5303ebb0814b", - "max_mempool_bytes": 500000000, - "pacemaker_config": { - "timeout_msec": 5000 - } - } -} \ No newline at end of file diff --git a/consensus/consensus_tests/genesis.json b/consensus/consensus_tests/genesis.json deleted file mode 100755 index 88156506b..000000000 --- a/consensus/consensus_tests/genesis.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "consensus_genesis_state": { - "genesis_time": { - "seconds": 1663973128, - "nanos": 273119000 - }, - "chain_id": "testnet", - "max_block_bytes": 4000000, - "validators": [ - { - "address": "27ce0b388a1a3b2c663f755d78206e96eea4362a", - "public_key": "bb6329890486544b0101e23f26fd5a74f0355843d6b63e48ca95954c6fea6ee2", - "chains": null, - "generic_param": "node1.consensus:8080", - "staked_amount": "1000000000000", - "paused_height": -1, - "unstaking_height": -1, - "output": "27ce0b388a1a3b2c663f755d78206e96eea4362a", - "actor_type": 3 - }, - { - "address": "210658c6938dd7b02fad44774c06b8d33067c283", - "public_key": "26b785bddc4f48911dcb7c99ad7657f4cce39f3af659ef17ccec26f44aeb4dee", - "chains": null, - "generic_param": "node2.consensus:8080", - "staked_amount": "1000000000000", - "paused_height": -1, - "unstaking_height": -1, - "output": "210658c6938dd7b02fad44774c06b8d33067c283", - "actor_type": 3 - }, - { - "address": "9c17a6256e9fddf888608d7573f403f35b57a5ce", - "public_key": "92e1f441b5002f110e3321ed122ac3e2dba1a800a7477ffcae0dfb5ccfcde95c", - "chains": null, - "generic_param": "node3.consensus:8080", - "staked_amount": "1000000000000", - "paused_height": -1, - "unstaking_height": -1, - "output": "9c17a6256e9fddf888608d7573f403f35b57a5ce", - "actor_type": 3 - }, - { - "address": "e4aa8d7cebb13f1832fe91695818ce09fb1c444b", - "public_key": "8ae7e70016d48453e3fe173d1457a7426c0479623c6ca5ba042d5303ebb0814b", - "chains": null, - "generic_param": "node4.consensus:8080", - "staked_amount": "1000000000000", - "paused_height": -1, - "unstaking_height": -1, - "output": "e4aa8d7cebb13f1832fe91695818ce09fb1c444b", - "actor_type": 3 - } - ] - } -} \ No newline at end of file diff --git a/consensus/consensus_tests/pacemaker_test.go b/consensus/consensus_tests/pacemaker_test.go index 25553aa01..fb2d151ac 100644 --- a/consensus/consensus_tests/pacemaker_test.go +++ b/consensus/consensus_tests/pacemaker_test.go @@ -2,6 +2,7 @@ package consensus_tests import ( "encoding/hex" + "fmt" "log" "reflect" "testing" @@ -146,10 +147,16 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { // Set all nodes to the same STEP and HEIGHT BUT different ROUNDS for _, pocketNode := range pocketNodes { + // utilityContext is only set on new rounds, which is skipped in this test + utilityContext, err := pocketNode.GetBus().GetUtilityModule().NewContext(int64(testHeight)) + require.NoError(t, err) + fmt.Println("OLSH", utilityContext) + consensusModImpl := GetConsensusModImplementation(pocketNode) consensusModImpl.FieldByName("Height").SetUint(testHeight) consensusModImpl.FieldByName("Step").SetInt(testStep) consensusModImpl.FieldByName("LeaderId").Set(reflect.Zero(reflect.TypeOf(&leaderId))) // This is re-elected during paceMaker catchup + consensusModImpl.FieldByName("UtilityContext").Set(reflect.ValueOf(utilityContext)) } // Set the leader to be in the highest round. diff --git a/consensus/debugging.go b/consensus/debugging.go index 5e111095e..0419575ba 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -1,10 +1,11 @@ package consensus import ( - "github.com/pokt-network/pocket/shared/debug" "log" "time" + "github.com/pokt-network/pocket/shared/debug" + typesCons "github.com/pokt-network/pocket/consensus/types" ) diff --git a/consensus/helpers.go b/consensus/helpers.go index 34c36e054..63270826f 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -190,12 +190,12 @@ func (m *ConsensusModule) clearLeader() { m.LeaderId = nil } -func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) { +func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) error { leaderId, err := m.leaderElectionMod.ElectNextLeader(message) if err != nil || leaderId == 0 { m.nodeLogError(typesCons.ErrLeaderElection(message).Error(), err) m.clearLeader() - return + return err } m.LeaderId = &leaderId @@ -207,6 +207,8 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) { m.logPrefix = "REPLICA" m.nodeLog(typesCons.ElectedNewLeader(m.IdToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) } + + return nil } /*** General Infrastructure Helpers ***/ diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index 9b5ae3287..aa72b3a26 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -1,25 +1,10 @@ package consensus -import typesCons "github.com/pokt-network/pocket/consensus/types" - -// TODO(discuss): Low priority design: think of a way to make `hotstuff_*` files be a sub-package under consensus. -// This is currently not possible because functions tied to the `ConsensusModule` -// struct (implementing the ConsensusModule module), which spans multiple files. -/* -TODO(discuss): The reason we do not assign both the leader and the replica handlers -to the leader (which should also act as a replica when it is a leader) is because it -can create a weird inconsistent state (e.g. both the replica and leader try to restart -the Pacemaker timeout). This requires additional "replica-like" logic in the leader -handler which has both pros and cons: - Pros: - * The leader can short-circuit and optimize replica related logic - * Avoids additional code flowing through the P2P pipeline - * Allows for micro-optimizations - Cons: - * The leader's "replica related logic" requires an additional code path - * Code is less "generalizable" and therefore potentially more error prone -*/ +import ( + typesCons "github.com/pokt-network/pocket/consensus/types" +) +// DISCUSS: Should these functions return an error? type HotstuffMessageHandler interface { HandleNewRoundMessage(*ConsensusModule, *typesCons.HotstuffMessage) HandlePrepareMessage(*ConsensusModule, *typesCons.HotstuffMessage) @@ -28,7 +13,7 @@ type HotstuffMessageHandler interface { HandleDecideMessage(*ConsensusModule, *typesCons.HotstuffMessage) } -func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) { +func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) error { m.nodeLog(typesCons.DebugHandlingHotstuffMessage(msg)) step := msg.GetStep() @@ -40,19 +25,21 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) if !(m.LeaderId != nil && !m.isLeader() && step == NewRound) { m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) } - return + return err } // Need to execute leader election if there is no leader and we are in a new round. if m.Step == NewRound && m.LeaderId == nil { - m.electNextLeader(msg) + if err := m.electNextLeader(msg); err != nil { + return err + } } if m.isReplica() { replicaHandlers[step](m, msg) - return } - // Note that the leader also acts as a replica, but this logic is implemented in the underlying code. leaderHandlers[step](m, msg) + + return nil } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 144da8e07..423ac9a87 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -59,6 +59,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM } m.Block = block } else { + // DISCUSS: Do we need to call `validateProposal` here? // Leader acts like a replica if `highPrepareQC` is not `nil` if err := m.applyBlock(highPrepareQC.Block); err != nil { m.nodeLogError(typesCons.ErrApplyBlock.Error(), err) @@ -105,7 +106,6 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo } m.nodeLog(typesCons.OptimisticVoteCountPassed(Prepare)) - // DISCUSS: What prevents leader from swapping out the block here? prepareQC, err := m.getQuorumCertificate(m.Height, Prepare, m.Round) if err != nil { m.nodeLogError(typesCons.ErrQCInvalid(Prepare).Error(), err) @@ -116,13 +116,13 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo m.HighPrepareQC = prepareQC m.MessagePool[Prepare] = nil - precommitProposeMessages, err := CreateProposeMessage(m.Height, m.Round, PreCommit, m.Block, prepareQC) + preCommitProposeMessage, err := CreateProposeMessage(m.Height, m.Round, PreCommit, m.Block, prepareQC) if err != nil { m.nodeLogError(typesCons.ErrCreateProposeMessage(PreCommit).Error(), err) m.paceMaker.InterruptRound() return } - m.broadcastToNodes(precommitProposeMessages) + m.broadcastToNodes(preCommitProposeMessage) // Leader also acts like a replica precommitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, PreCommit, m.Block, m.privateKey) @@ -172,7 +172,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus commitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Commit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Commit).Error(), err) - return // TODO(olshansky): Should we interrupt the round here? + return } m.sendToNode(commitVoteMessage) } @@ -217,7 +217,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusMod return } - // There is no "replica behavior" to imitate here + // There is no "replica behavior" to imitate here because the leader already committed the block proposal. m.paceMaker.NewHeight() m.GetBus(). @@ -262,6 +262,8 @@ func (handler *HotstuffLeaderMessageHandler) emitTelemetryEvent(m *ConsensusModu // ValidateBasic general validation checks that apply to every HotstuffLeaderMessage func (handler *HotstuffLeaderMessageHandler) validateBasic(m *ConsensusModule, msg *typesCons.HotstuffMessage) error { + // DISCUSS: What prevents leader from swapping out the block here? + // Discard messages with invalid partial signatures before storing it in the leader's consensus mempool if err := m.validatePartialSignature(msg); err != nil { return err @@ -331,13 +333,13 @@ func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { lastByzValidators := make([][]byte, 0) // Reap the mempool for transactions to be applied in this block - txs, err := m.utilityContext.GetProposalTransactions(m.privateKey.Address(), maxTxBytes, lastByzValidators) + txs, err := m.UtilityContext.GetProposalTransactions(m.privateKey.Address(), maxTxBytes, lastByzValidators) if err != nil { return nil, err } // Apply all the transactions in the block - appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) + appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) if err != nil { return nil, err } diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index cb333af70..06792bfb7 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "fmt" "log" - "unsafe" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" @@ -72,7 +71,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *ConsensusM prepareVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Prepare, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Prepare).Error(), err) - return + return // Not interrupting the round because liveness could continue with one failed vote } m.sendToNode(prepareVoteMessage) } @@ -101,7 +100,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *Consensu preCommitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, PreCommit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(PreCommit).Error(), err) - return + return // Not interrupting the round because liveness could continue with one failed vote } m.sendToNode(preCommitVoteMessage) } @@ -130,7 +129,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusMo commitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Commit, m.Block, m.privateKey) if err != nil { m.nodeLogError(typesCons.ErrCreateVoteMessage(Commit).Error(), err) - return + return // Not interrupting the round because liveness could continue with one failed vote } m.sendToNode(commitVoteMessage) } @@ -182,17 +181,19 @@ func (handler *HotstuffReplicaMessageHandler) emitTelemetryEvent(m *ConsensusMod } func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error { + // Check if node should be accepting proposals if !(msg.GetType() == Propose && msg.GetStep() == Prepare) { return typesCons.ErrProposalNotValidInPrepare } + // Basic block metadata validation if err := m.validateBlock(msg.GetBlock()); err != nil { return err } - // TODO(discuss): A nil QC implies a successful CommitQC or TimeoutQC, which have been omitted intentionally since - // they are not needed for consensus validity. However, if a QC is specified, it must be valid. quorumCert := msg.GetQuorumCertificate() + // A nil QC implies a successful CommitQC or TimeoutQC, which have been omitted intentionally + // since they are not needed for consensus validity. However, if a QC is specified, it must be valid. if quorumCert != nil { if err := m.validateQuorumCertificate(quorumCert); err != nil { return err @@ -209,13 +210,14 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error } // Safety: check the hash of the locked QC - // TODO(olshansky): Extend implementation to adopt `ExtendsFrom` as described in the Hotstuff whitepaper. - if protoHash(lockedQC.GetBlock()) == protoHash(justifyQC.Block) { // && lockedQC.Block.ExtendsFrom(justifyQC.Block) + // The equivalent of `lockedQC.Block.ExtendsFrom(justifyQC.Block)` in the hotstuff whitepaper is done in `applyBlock` below. + if protoHash(lockedQC.GetBlock()) == protoHash(justifyQC.Block) { m.nodeLog(typesCons.ProposalBlockExtends) return nil } - // Liveness: node is locked on a QC from the past. [TODO]: Do we want to set `m.LockedQC = nil` here or something else? + // Liveness: node is locked on a QC from the past. + // DISCUSS: Where should additional logic be added to unlock the node? if justifyQC.Height > lockedQC.Height || (justifyQC.Height == lockedQC.Height && justifyQC.Round > lockedQC.Round) { return typesCons.ErrNodeIsLockedOnPastQC } @@ -223,6 +225,29 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error return typesCons.ErrUnhandledProposalCase } +// This helper applies the block metadata to the utility & persistence layers +func (m *ConsensusModule) applyBlock(block *typesCons.Block) error { + // TECHDEBT: Retrieve this from persistence + lastByzValidators := make([][]byte, 0) + + // Apply all the transactions in the block and get the appHash + fmt.Println("OLSH", m.UtilityContext, block, block.BlockHeader.ProposerAddress == nil, block.Transactions == nil) + tx := make([][]byte, 0) + byteArr := []byte("A") + tx = append(tx, byteArr) + appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), []byte("A"), tx, lastByzValidators) + if err != nil { + return err + } + + // CONSOLIDATE: Terminology of `blockHash`, `appHash` and `stateHash` + if block.BlockHeader.Hash != hex.EncodeToString(appHash) { + return typesCons.ErrInvalidAppHash(block.BlockHeader.Hash, hex.EncodeToString(appHash)) + } + + return nil +} + func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertificate) error { if qc == nil { return typesCons.ErrNilQC @@ -238,6 +263,8 @@ func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertific msgToJustify := qcToHotstuffMessage(qc) numValid := 0 + + // TODO(#109): Aggregate signatures once BLS or DKG is implemented for _, partialSig := range qc.ThresholdSignature.Signatures { validator, ok := m.validatorMap[partialSig.Address] if !ok { @@ -252,7 +279,6 @@ func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertific } numValid++ } - if err := m.isOptimisticThresholdMet(numValid); err != nil { return err } @@ -260,30 +286,6 @@ func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertific return nil } -// This is a helper function intended to be called by a replica/voter during a view change -func (m *ConsensusModule) applyBlock(block *typesCons.Block) error { - // TODO: Add unit tests to verify this. - if unsafe.Sizeof(*block) > uintptr(m.MaxBlockBytes) { - return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) - } - - // TECHDEBT: Retrieve this from persistence - lastByzValidators := make([][]byte, 0) - - // Apply all the transactions in the block and get the appHash - appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), block.BlockHeader.ProposerAddress, block.Transactions, lastByzValidators) - if err != nil { - return err - } - - // CONSOLIDATE: Terminology of `blockHash`, `appHash` and `stateHash` - if block.BlockHeader.Hash != hex.EncodeToString(appHash) { - return typesCons.ErrInvalidAppHash(block.BlockHeader.Hash, hex.EncodeToString(appHash)) - } - - return nil -} - func qcToHotstuffMessage(qc *typesCons.QuorumCertificate) *typesCons.HotstuffMessage { return &typesCons.HotstuffMessage{ Height: qc.Height, diff --git a/consensus/messages.go b/consensus/messages.go index 8020ec068..eb2801275 100644 --- a/consensus/messages.go +++ b/consensus/messages.go @@ -70,16 +70,18 @@ func CreateVoteMessage( return msg, nil } -// Returns "partial" signature of the hotstuff message from one of the validators +// Returns "partial" signature of the hotstuff message from one of the validators. +// If there is an error signing the bytes, nil is returned instead. func getMessageSignature(msg *typesCons.HotstuffMessage, privKey crypto.PrivateKey) []byte { bytesToSign, err := getSignableBytes(msg) if err != nil { + log.Printf("[WARN] error getting bytes to sign: %v\n", err) return nil } signature, err := privKey.Sign(bytesToSign) if err != nil { - log.Fatalf("Error signing message: %v", err) + log.Printf("[WARN] error signing message: %v\n", err) return nil } diff --git a/consensus/module.go b/consensus/module.go index 42c7354ae..68e3874cc 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -53,7 +53,7 @@ type ConsensusModule struct { validatorMap typesCons.ValidatorMap // Module Dependencies - utilityContext modules.UtilityContext + UtilityContext modules.UtilityContext paceMaker Pacemaker leaderElectionMod leader_election.LeaderElectionModule @@ -125,7 +125,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus lastAppHash: "", validatorMap: valMap, - utilityContext: nil, + UtilityContext: nil, paceMaker: paceMaker, leaderElectionMod: leaderElectionMod, @@ -225,7 +225,9 @@ func (m *ConsensusModule) HandleMessage(message *anypb.Any) error { if err := anypb.UnmarshalTo(message, &hotstuffMessage, proto.UnmarshalOptions{}); err != nil { return err } - m.handleHotstuffMessage(&hotstuffMessage) + if err := m.handleHotstuffMessage(&hotstuffMessage); err != nil { + return err + } case UtilityMessage: panic("[WARN] UtilityMessage handling is not implemented by consensus yet...") default: From fc904a0986b840b70aaee7469f0f26d955c1bc2e Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 23 Sep 2022 17:24:30 -0700 Subject: [PATCH 14/56] Remove OLSH logs --- consensus/consensus_tests/pacemaker_test.go | 2 -- consensus/consensus_tests/utils_test.go | 32 ++++++++++++++------- consensus/hotstuff_replica.go | 6 +--- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/consensus/consensus_tests/pacemaker_test.go b/consensus/consensus_tests/pacemaker_test.go index fb2d151ac..23466e7a3 100644 --- a/consensus/consensus_tests/pacemaker_test.go +++ b/consensus/consensus_tests/pacemaker_test.go @@ -2,7 +2,6 @@ package consensus_tests import ( "encoding/hex" - "fmt" "log" "reflect" "testing" @@ -150,7 +149,6 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { // utilityContext is only set on new rounds, which is skipped in this test utilityContext, err := pocketNode.GetBus().GetUtilityModule().NewContext(int64(testHeight)) require.NoError(t, err) - fmt.Println("OLSH", utilityContext) consensusModImpl := GetConsensusModImplementation(pocketNode) consensusModImpl.FieldByName("Height").SetUint(testHeight) diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index dd687953b..2d4bb01ae 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -343,8 +343,7 @@ func baseP2PMock(t *testing.T, testChannel modules.EventsChannel) *modulesMock.M func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUtilityModule { ctrl := gomock.NewController(t) utilityMock := modulesMock.NewMockUtilityModule(ctrl) - utilityContextMock := modulesMock.NewMockUtilityContext(ctrl) - persistenceContextMock := modulesMock.NewMockPersistenceRWContext(ctrl) + utilityContextMock := baseUtilityContextMock(t) utilityMock.EXPECT().Start().Return(nil).AnyTimes() utilityMock.EXPECT().SetBus(gomock.Any()).Do(func(modules.Bus) {}).AnyTimes() @@ -353,6 +352,14 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti Return(utilityContextMock, nil). MaxTimes(4) + return utilityMock +} + +func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { + ctrl := gomock.NewController(t) + utilityContextMock := modulesMock.NewMockUtilityContext(ctrl) + persistenceContextMock := basePersistenceContextMock(t) + utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() utilityContextMock.EXPECT().CommitPersistenceContext().Return(nil).AnyTimes() utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() @@ -365,9 +372,14 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti Return(appHash, nil). AnyTimes() - persistenceContextMock.EXPECT().Commit().Return(nil).AnyTimes() + return utilityContextMock +} - return utilityMock +func basePersistenceContextMock(t *testing.T) *modulesMock.MockPersistenceRWContext { + ctrl := gomock.NewController(t) + persistenceContextMock := modulesMock.NewMockPersistenceRWContext(ctrl) + persistenceContextMock.EXPECT().Commit().Return(nil).AnyTimes() + return persistenceContextMock } func baseTelemetryMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockTelemetryModule { @@ -378,25 +390,23 @@ func baseTelemetryMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockT telemetryMock.EXPECT().Start().Do(func() {}).AnyTimes() telemetryMock.EXPECT().SetBus(gomock.Any()).Do(func(modules.Bus) {}).AnyTimes() - telemetryMock.EXPECT().GetTimeSeriesAgent().Return(timeSeriesAgentMock).AnyTimes() - timeSeriesAgentMock.EXPECT().CounterRegister(gomock.Any(), gomock.Any()).MaxTimes(1) - timeSeriesAgentMock.EXPECT().CounterIncrement(gomock.Any()).AnyTimes() - telemetryMock.EXPECT().GetEventMetricsAgent().Return(eventMetricsAgentMock).AnyTimes() - eventMetricsAgentMock.EXPECT().EmitEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() return telemetryMock } func baseTelemetryTimeSeriesAgentMock(t *testing.T) *modulesMock.MockTimeSeriesAgent { ctrl := gomock.NewController(t) - timeseriesAgentMock := modulesMock.NewMockTimeSeriesAgent(ctrl) - return timeseriesAgentMock + timeSeriesAgentMock := modulesMock.NewMockTimeSeriesAgent(ctrl) + timeSeriesAgentMock.EXPECT().CounterRegister(gomock.Any(), gomock.Any()).MaxTimes(1) + timeSeriesAgentMock.EXPECT().CounterIncrement(gomock.Any()).AnyTimes() + return timeSeriesAgentMock } func baseTelemetryEventMetricsAgentMock(t *testing.T) *modulesMock.MockEventMetricsAgent { ctrl := gomock.NewController(t) eventMetricsAgentMock := modulesMock.NewMockEventMetricsAgent(ctrl) + eventMetricsAgentMock.EXPECT().EmitEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() return eventMetricsAgentMock } diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 06792bfb7..c7765de2f 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -231,11 +231,7 @@ func (m *ConsensusModule) applyBlock(block *typesCons.Block) error { lastByzValidators := make([][]byte, 0) // Apply all the transactions in the block and get the appHash - fmt.Println("OLSH", m.UtilityContext, block, block.BlockHeader.ProposerAddress == nil, block.Transactions == nil) - tx := make([][]byte, 0) - byteArr := []byte("A") - tx = append(tx, byteArr) - appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), []byte("A"), tx, lastByzValidators) + appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), block.BlockHeader.ProposerAddress, block.Transactions, lastByzValidators) if err != nil { return err } From 213892c896970a731a5b2827698e42a5ab6641f3 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 24 Sep 2022 10:23:33 -0700 Subject: [PATCH 15/56] /s/aggregateMessage/m.aggregateMessage --- consensus/hotstuff_leader.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 423ac9a87..54674186f 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -243,7 +243,7 @@ func (handler *HotstuffLeaderMessageHandler) anteHandle(m *ConsensusModule, msg if err := handler.validateBasic(m, msg); err != nil { return err } - m.aggregateMessage(msg) + m.tempIndexHotstuffMessage(msg) return nil } @@ -262,8 +262,6 @@ func (handler *HotstuffLeaderMessageHandler) emitTelemetryEvent(m *ConsensusModu // ValidateBasic general validation checks that apply to every HotstuffLeaderMessage func (handler *HotstuffLeaderMessageHandler) validateBasic(m *ConsensusModule, msg *typesCons.HotstuffMessage) error { - // DISCUSS: What prevents leader from swapping out the block here? - // Discard messages with invalid partial signatures before storing it in the leader's consensus mempool if err := m.validatePartialSignature(msg); err != nil { return err @@ -305,10 +303,11 @@ func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessag address, m.ValAddrToIdMap[address], msg, pubKey) } -func (m *ConsensusModule) aggregateMessage(msg *typesCons.HotstuffMessage) { - // TODO(olshansky): Add proper tests for this when we figure out where the mempool should live. - // NOTE: This is just a placeholder at the moment. It doesn't actually work because SizeOf returns - // the size of the map pointer, and does not recursively determine the size of all the underlying elements. +// TODO: This is just a placeholder at the moment. It doesn't actually work because SizeOf returns +// the size of the map pointer, and does not recursively determine the size of all the +// underlying elements +// Add proper tests and implementation once the mempool is implemented. +func (m *ConsensusModule) tempIndexHotstuffMessage(msg *typesCons.HotstuffMessage) { if m.consCfg.GetMaxMempoolBytes() < uint64(unsafe.Sizeof(m.MessagePool)) { m.nodeLogError(typesCons.DisregardHotstuffMessage, typesCons.ErrConsensusMempoolFull) return From 686c2ca5c7528e0c3f100e9b9f584bc194205ecc Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 24 Sep 2022 10:38:57 -0700 Subject: [PATCH 16/56] Updated validateBlockBasic --- consensus/block.go | 13 ++++++++++++- consensus/hotstuff_leader.go | 25 ++++++++++++------------- consensus/hotstuff_replica.go | 12 +++++------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/consensus/block.go b/consensus/block.go index ce123c1d7..a35e4f100 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -1,13 +1,14 @@ package consensus import ( + "log" "unsafe" typesCons "github.com/pokt-network/pocket/consensus/types" ) // TODO: Add additional basic block metadata validation w/ unit tests -func (m *ConsensusModule) validateBlock(block *typesCons.Block) error { +func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { if block == nil { return typesCons.ErrNilBlock } @@ -16,6 +17,16 @@ func (m *ConsensusModule) validateBlock(block *typesCons.Block) error { return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) } + // If the current block being processed (i.e. voted on) by consensus is non nil, we need to make + // sure that the data (height, round, step, txs, etc) is the same before we start validating the signatures + if m.Block != nil { + // DISCUSS: The only difference between blocks from one step to another is the QC, so we need + // to determine where/how to validate this + if protoHash(m.Block) != protoHash(block) { + log.Println("[TECHDEBT][ERROR] The block being processed is not the same as that received by the consensus module ") + } + } + return nil } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 54674186f..9c75148cc 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -240,9 +240,17 @@ func (handler *HotstuffLeaderMessageHandler) HandleDecideMessage(m *ConsensusMod // anteHandle is the general handler called for every before every specific HotstuffLeaderMessageHandler handler func (handler *HotstuffLeaderMessageHandler) anteHandle(m *ConsensusModule, msg *typesCons.HotstuffMessage) error { - if err := handler.validateBasic(m, msg); err != nil { + // Basic block metadata validation + if err := m.validateBlockBasic(msg.GetBlock()); err != nil { return err } + + // Discard messages with invalid partial signatures before storing it in the leader's consensus mempool + if err := m.validatePartialSignature(msg); err != nil { + return err + } + + // TECHDEBT: Until we integrate with the real mempool, this is a makeshift solution m.tempIndexHotstuffMessage(msg) return nil } @@ -260,15 +268,6 @@ func (handler *HotstuffLeaderMessageHandler) emitTelemetryEvent(m *ConsensusModu ) } -// ValidateBasic general validation checks that apply to every HotstuffLeaderMessage -func (handler *HotstuffLeaderMessageHandler) validateBasic(m *ConsensusModule, msg *typesCons.HotstuffMessage) error { - // Discard messages with invalid partial signatures before storing it in the leader's consensus mempool - if err := m.validatePartialSignature(msg); err != nil { - return err - } - return nil -} - func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessage) error { if msg.GetStep() == NewRound { m.nodeLog(typesCons.ErrUnnecessaryPartialSigForNewRound.Error()) @@ -303,9 +302,9 @@ func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessag address, m.ValAddrToIdMap[address], msg, pubKey) } -// TODO: This is just a placeholder at the moment. It doesn't actually work because SizeOf returns -// the size of the map pointer, and does not recursively determine the size of all the -// underlying elements +// TODO: This is just a placeholder at the moment for indexing hotstuff messages ONLY. +// It doesn't actually work because SizeOf returns the size of the map pointer, +// and does not recursively determine the size of all the underlying elements // Add proper tests and implementation once the mempool is implemented. func (m *ConsensusModule) tempIndexHotstuffMessage(msg *typesCons.HotstuffMessage) { if m.consCfg.GetMaxMempoolBytes() < uint64(unsafe.Sizeof(m.MessagePool)) { diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index c7765de2f..a4eda9951 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -3,7 +3,6 @@ package consensus import ( "encoding/hex" "fmt" - "log" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" typesCons "github.com/pokt-network/pocket/consensus/types" @@ -163,7 +162,11 @@ func (handler *HotstuffReplicaMessageHandler) HandleDecideMessage(m *ConsensusMo // anteHandle is the handler called on every replica message before specific handler func (handler *HotstuffReplicaMessageHandler) anteHandle(m *ConsensusModule, msg *typesCons.HotstuffMessage) error { - log.Println("TODO: Hotstuff replica ante handle not implemented yet") + // Basic block metadata validation + if err := m.validateBlockBasic(msg.GetBlock()); err != nil { + return err + } + return nil } @@ -186,11 +189,6 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error return typesCons.ErrProposalNotValidInPrepare } - // Basic block metadata validation - if err := m.validateBlock(msg.GetBlock()); err != nil { - return err - } - quorumCert := msg.GetQuorumCertificate() // A nil QC implies a successful CommitQC or TimeoutQC, which have been omitted intentionally // since they are not needed for consensus validity. However, if a QC is specified, it must be valid. From 78ed1d8adfffe4aaae682771bf37286bffd00fae Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 24 Sep 2022 12:04:22 -0700 Subject: [PATCH 17/56] Removed PersistenceContext exposure from utility context interface --- consensus/consensus_tests/utils_test.go | 6 +++--- consensus/hotstuff_leader.go | 2 +- consensus/types/proto/block.proto | 2 +- shared/modules/persistence_module.go | 12 ++++-------- shared/modules/utility_module.go | 6 ++++-- utility/block.go | 2 +- utility/context.go | 11 +++++------ 7 files changed, 19 insertions(+), 22 deletions(-) diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 2d4bb01ae..f7711d03f 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -358,10 +358,10 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { ctrl := gomock.NewController(t) utilityContextMock := modulesMock.NewMockUtilityContext(ctrl) - persistenceContextMock := basePersistenceContextMock(t) + // persistenceContextMock := basePersistenceContextMock(t) - utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() - utilityContextMock.EXPECT().CommitPersistenceContext().Return(nil).AnyTimes() + // utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() + utilityContextMock.EXPECT().CommitContext().Return(nil).AnyTimes() utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() utilityContextMock.EXPECT(). GetProposalTransactions(gomock.Any(), maxTxBytes, gomock.AssignableToTypeOf(emptyByzValidators)). diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 9c75148cc..89541c999 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -336,7 +336,7 @@ func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { return nil, err } - // Apply all the transactions in the block + // Apply all the transactions in the block - appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) if err != nil { return nil, err diff --git a/consensus/types/proto/block.proto b/consensus/types/proto/block.proto index 6024bcea4..b18d8dfa7 100644 --- a/consensus/types/proto/block.proto +++ b/consensus/types/proto/block.proto @@ -5,7 +5,7 @@ option go_package = "github.com/pokt-network/pocket/consensus/types"; import "google/protobuf/timestamp.proto"; -// TODO (Team) Discuss all tendermint legacy +// TECHDEBT: Re-evaluate some tendermint legacy fields message BlockHeader { int64 height = 1; string hash = 2; diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 3d74d137d..9f93c5d35 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -49,8 +49,9 @@ type PersistenceWriteContext interface { NewSavePoint([]byte) error RollbackToSavePoint([]byte) error - Reset() error Commit() error + // DISCUSS: Can we consolidate `Reset` and `Release` + Reset() error Release() error // Question: @@ -61,13 +62,8 @@ type PersistenceWriteContext interface { // Block Operations // Indexer Operations - StoreTransaction(transactionProtoBytes []byte) error - - // Block Operations - // TODO_TEMPORARY: Including two functions for the SQL and KV Store as an interim solution - // until we include the schema as part of the SQL Store because persistence - // currently has no access to the protobuf schema which is the source of truth. - InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error // Writes the block in the SQL database + StoreTransaction(transactionProtoBytes []byte) error // Stores a transaction + CommitTransactions(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error // Writes the block in the SQL database // Pool Operations AddPoolAmount(name string, amount string) error diff --git a/shared/modules/utility_module.go b/shared/modules/utility_module.go index edfbc94ee..8472e4582 100644 --- a/shared/modules/utility_module.go +++ b/shared/modules/utility_module.go @@ -14,13 +14,15 @@ type UtilityModule interface { // operations. type UtilityContext interface { // Block operations + + // Reaps the mempool for transactions that are ready to be proposed in a new block GetProposalTransactions(proposer []byte, maxTransactionBytes int, lastBlockByzantineValidators [][]byte) (transactions [][]byte, err error) + // Applies the transactions to an ephemeral state in the utility & underlying persistence context;similar to `SafeNode` in the Hotstuff whitepaper. ApplyBlock(height int64, proposer []byte, transactions [][]byte, lastBlockByzantineValidators [][]byte) (appHash []byte, err error) // Context operations ReleaseContext() - GetPersistenceContext() PersistenceRWContext - CommitPersistenceContext() error + CommitContext() error // Validation operations CheckTransaction(tx []byte) error diff --git a/utility/block.go b/utility/block.go index c762b35a7..ef2ab2894 100644 --- a/utility/block.go +++ b/utility/block.go @@ -51,7 +51,7 @@ func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, if err := u.ApplyTransaction(tx); err != nil { return nil, err } - if err := u.GetPersistenceContext().StoreTransaction(transactionProtoBytes); err != nil { + if err := u.Context.PersistenceRWContext.StoreTransaction(transactionProtoBytes); err != nil { return nil, err } diff --git a/utility/context.go b/utility/context.go index 460b7cfa1..37f26d637 100644 --- a/utility/context.go +++ b/utility/context.go @@ -2,6 +2,7 @@ package utility import ( "encoding/hex" + "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/modules" typesUtil "github.com/pokt-network/pocket/utility/types" @@ -39,12 +40,10 @@ func (u *UtilityContext) Store() *Context { return u.Context } -func (u *UtilityContext) GetPersistenceContext() modules.PersistenceRWContext { - return u.Context.PersistenceRWContext -} - -func (u *UtilityContext) CommitPersistenceContext() error { - return u.Context.PersistenceRWContext.Commit() +func (u *UtilityContext) CommitContext() error { + err := u.Context.PersistenceRWContext.Commit() + u.Context = nil + return err } func (u *UtilityContext) ReleaseContext() { From 113343830ab95cbd563bad80a1e1e07c4f1eedb9 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 24 Sep 2022 13:02:07 -0700 Subject: [PATCH 18/56] Tests passing - commit before creating a new block type --- consensus/block.go | 21 +++++++-------- consensus/consensus_tests/utils_test.go | 17 ++++++------ consensus/hotstuff_leader.go | 3 ++- consensus/types/errors.go | 2 ++ persistence/block.go | 6 ++++- persistence/context.go | 15 +++-------- persistence/db.go | 2 +- persistence/debug.go | 2 +- persistence/genesis.go | 2 +- persistence/state.go | 21 ++++++++------- persistence/test/module_test.go | 3 ++- shared/modules/persistence_module.go | 23 +++++++--------- shared/modules/utility_module.go | 4 +-- utility/block.go | 35 +++++++------------------ utility/context.go | 18 ++++++++----- utility/test/block_test.go | 18 +++---------- 16 files changed, 82 insertions(+), 110 deletions(-) diff --git a/consensus/block.go b/consensus/block.go index a35e4f100..48c452b2e 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -9,10 +9,14 @@ import ( // TODO: Add additional basic block metadata validation w/ unit tests func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { - if block == nil { + if block == nil && m.Step != NewRound { return typesCons.ErrNilBlock } + if block != nil && m.Step == NewRound { + return typesCons.ErrBlockExists + } + if unsafe.Sizeof(*block) > uintptr(m.MaxBlockBytes) { return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) } @@ -37,7 +41,10 @@ func (m *ConsensusModule) refreshUtilityContext() error { // should not be called. if m.UtilityContext != nil { m.nodeLog(typesCons.NilUtilityContextWarning) - m.UtilityContext.ReleaseContext() + if err := m.UtilityContext.ReleaseContext(); err != nil { + // Logging an error but allowing the consensus lifecycle to continue + m.nodeLogError("cannot release existing utility context", err) + } m.UtilityContext = nil } @@ -53,18 +60,10 @@ func (m *ConsensusModule) refreshUtilityContext() error { func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { m.nodeLog(typesCons.CommittingBlock(m.Height, len(block.Transactions))) - // Store the block in the KV store - // codec := codec.GetCodec() - // blockProtoBytes, err := codec.Marshal(block) - // if err != nil { - // return err - // } - // Commit and release the context - if err := m.UtilityContext.CommitPersistenceContext(); err != nil { + if err := m.UtilityContext.CommitContext(block.BlockHeader.QuorumCertificate); err != nil { return err } - m.UtilityContext.ReleaseContext() m.UtilityContext = nil diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index f7711d03f..7f12277a2 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -360,9 +360,6 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { utilityContextMock := modulesMock.NewMockUtilityContext(ctrl) // persistenceContextMock := basePersistenceContextMock(t) - // utilityContextMock.EXPECT().GetPersistenceContext().Return(persistenceContextMock).AnyTimes() - utilityContextMock.EXPECT().CommitContext().Return(nil).AnyTimes() - utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() utilityContextMock.EXPECT(). GetProposalTransactions(gomock.Any(), maxTxBytes, gomock.AssignableToTypeOf(emptyByzValidators)). Return(make([][]byte, 0), nil). @@ -371,16 +368,18 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { ApplyBlock(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(appHash, nil). AnyTimes() + utilityContextMock.EXPECT().CommitContext(gomock.Any()).Return(nil).AnyTimes() + utilityContextMock.EXPECT().ReleaseContext().Return(nil).AnyTimes() return utilityContextMock } -func basePersistenceContextMock(t *testing.T) *modulesMock.MockPersistenceRWContext { - ctrl := gomock.NewController(t) - persistenceContextMock := modulesMock.NewMockPersistenceRWContext(ctrl) - persistenceContextMock.EXPECT().Commit().Return(nil).AnyTimes() - return persistenceContextMock -} +// func basePersistenceContextMock(t *testing.T) *modulesMock.MockPersistenceRWContext { +// ctrl := gomock.NewController(t) +// persistenceContextMock := modulesMock.NewMockPersistenceRWContext(ctrl) +// persistenceContextMock.EXPECT().Commit(gomock.Any()).Return(nil).AnyTimes() +// return persistenceContextMock +// } func baseTelemetryMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockTelemetryModule { ctrl := gomock.NewController(t) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 89541c999..f87c5f58b 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -241,6 +241,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleDecideMessage(m *ConsensusMod // anteHandle is the general handler called for every before every specific HotstuffLeaderMessageHandler handler func (handler *HotstuffLeaderMessageHandler) anteHandle(m *ConsensusModule, msg *typesCons.HotstuffMessage) error { // Basic block metadata validation + if err := m.validateBlockBasic(msg.GetBlock()); err != nil { return err } @@ -336,7 +337,7 @@ func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { return nil, err } - // Apply all the transactions in the block - + // Apply all the transactions in the block - appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) if err != nil { return nil, err diff --git a/consensus/types/errors.go b/consensus/types/errors.go index d90d85151..fb6dd6df4 100644 --- a/consensus/types/errors.go +++ b/consensus/types/errors.go @@ -114,6 +114,7 @@ func DebugHandlingHotstuffMessage(msg *HotstuffMessage) string { // Errors const ( nilBLockError = "block is nil" + blockExistsError = "block exists but should be nil" nilBLockProposalError = "block should never be nil when creating a proposal message" nilBLockVoteError = "block should never be nil when creating a vote message for a proposal" proposalNotValidInPrepareError = "proposal is not valid in the PREPARE step" @@ -152,6 +153,7 @@ const ( var ( ErrNilBlock = errors.New(nilBLockError) + ErrBlockExists = errors.New(blockExistsError) ErrNilBlockProposal = errors.New(nilBLockProposalError) ErrNilBlockVote = errors.New(nilBLockVoteError) ErrProposalNotValidInPrepare = errors.New(proposalNotValidInPrepareError) diff --git a/persistence/block.go b/persistence/block.go index 49de236fc..af9a797da 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -66,6 +66,10 @@ func heightToBytes(height int64) []byte { return heightBytes } -func (p PostgresContext) storeBlock(blockProtoBytes []byte) error { +func (p PostgresContext) commitBlock(blockProtoBytes []byte) error { + // get current height + // get proposer + // get hash + // get transaction return p.DB.Blockstore.Put(heightToBytes(p.Height), blockProtoBytes) } diff --git a/persistence/context.go b/persistence/context.go index 57e15e0ad..35e7ba108 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -16,28 +16,19 @@ func (p PostgresContext) RollbackToSavePoint(bytes []byte) error { } func (p PostgresContext) UpdateAppHash() ([]byte, error) { - if _, err := p.updateStateHash(); err != nil { + if err := p.updateStateHash(); err != nil { return nil, err } - return p.StateHash, nil -} - -func (p PostgresContext) AppHash() ([]byte, error) { - return p.StateHash, nil + return p.stateHash, nil } func (p PostgresContext) Reset() error { panic("TODO: PostgresContext Reset not implemented") } -func (p PostgresContext) Commit() error { +func (p PostgresContext) Commit(quorumCert []byte) error { log.Printf("About to commit context at height %d.\n", p.Height) - // HACK: The data has already been written to the postgres DB, so what should we do here? The idea I have is: - // if _, err := p.updateStateHash(); err != nil { - // return err - // } - ctx := context.TODO() if err := p.DB.Tx.Commit(context.TODO()); err != nil { return err diff --git a/persistence/db.go b/persistence/db.go index 060cec041..8d392d7b5 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -45,7 +45,7 @@ type PostgresContext struct { Height int64 DB PostgresDB - StateHash []byte + stateHash []byte // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get // access to these directly via the postgres module. diff --git a/persistence/debug.go b/persistence/debug.go index 3fdc583a4..14fc5c704 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -44,7 +44,7 @@ func (m *PersistenceModule) showLatestBlockInStore(_ *debug.DebugMessage) { func (m *PersistenceModule) clearState(_ *debug.DebugMessage) { context, err := m.NewRWContext(-1) - defer context.Commit() + defer context.Commit(nil) if err != nil { log.Printf("Error creating new context: %s \n", err) return diff --git a/persistence/genesis.go b/persistence/genesis.go index 727fa4f57..329cdbfa4 100644 --- a/persistence/genesis.go +++ b/persistence/genesis.go @@ -149,7 +149,7 @@ func (m *PersistenceModule) populateGenesisState(state *types.PersistenceGenesis log.Fatalf("an error occurred initializing flags: %s", err.Error()) } - if err = rwContext.Commit(); err != nil { + if err = rwContext.Commit(nil); err != nil { log.Fatalf("an error occurred during commit() on genesis state %s ", err.Error()) } } diff --git a/persistence/state.go b/persistence/state.go index f35f3622b..d689410ee 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -52,30 +52,31 @@ func loadMerkleTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { } // Question: Is this the right approach? -func (p *PostgresContext) updateStateHash() ([]byte, error) { +func (p *PostgresContext) updateStateHash() error { // Update all the merkle trees for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { switch treeType { case appMerkleTree: apps, err := p.getApplicationsUpdatedAtHeight(p.Height) if err != nil { - return nil, typesUtil.NewError(typesUtil.Code(42), "Couldn't figure out apps updated") // TODO_IN_THIS_COMMIT + // TODO_IN_THIS_COMMIT: Update this error + return typesUtil.NewError(typesUtil.Code(42), "Couldn't figure out apps updated") } for _, app := range apps { - appBytes, err := proto.Marshal(app) + appBz, err := proto.Marshal(app) if err != nil { - return nil, err + return err } // An update results in a create/update that is idempotent addrBz, err := hex.DecodeString(app.Address) if err != nil { - return nil, err + return err } - if _, err := p.MerkleTrees[treeType].Update(addrBz, appBytes); err != nil { - return nil, err + if _, err := p.MerkleTrees[treeType].Update(addrBz, appBz); err != nil { + return err } - // TODO_IN_THIS_COMMIT: Add support for `Delete` operations to remove it from the tree } + // TODO_IN_THIS_COMMIT: re default: log.Fatalln("Not handled yet in state commitment update", treeType) } @@ -96,6 +97,6 @@ func (p *PostgresContext) updateStateHash() ([]byte, error) { rootsConcat := bytes.Join(roots, []byte{}) stateHash := sha256.Sum256(rootsConcat) - p.StateHash = stateHash[:] - return p.StateHash, nil + p.stateHash = stateHash[:] + return nil } diff --git a/persistence/test/module_test.go b/persistence/test/module_test.go index 3bee636c5..f5df476b0 100644 --- a/persistence/test/module_test.go +++ b/persistence/test/module_test.go @@ -18,12 +18,13 @@ func TestPersistenceContextParallelReadWrite(t *testing.T) { poolAddress := []byte("address") originalAmount := "15" modifiedAmount := "10" + quorumCert := []byte("quorumCert") // setup a write context, insert a pool and commit it context, err := testPersistenceMod.NewRWContext(0) require.NoError(t, err) require.NoError(t, context.InsertPool(poolName, poolAddress, originalAmount)) - require.NoError(t, context.Commit()) + require.NoError(t, context.Commit(quorumCert)) // verify the insert in the previously committed context worked contextA, err := testPersistenceMod.NewRWContext(0) diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 9f93c5d35..4da6fb211 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -38,32 +38,29 @@ type PersistenceRWContext interface { PersistenceWriteContext } +// TODO: Simplify the interface (reference - https://dave.cheney.net/practical-go/presentations/gophercon-israel.html#_prefer_single_method_interfaces) +// - Add general purpose methods such as `ActorOperation(enum_actor_type, ...)` which can be use like so: `Insert(FISHERMAN, ...)` +// - Use general purpose parameter methods such as `Set(enum_gov_type, ...)` such as `Set(STAKING_ADJUSTMENT, ...)` + // NOTE: There's not really a use case for a write only interface, // but it abstracts and contrasts nicely against the read only context // TODO (andrew) convert address and public key to string not bytes #149 type PersistenceWriteContext interface { - // TODO: Simplify the interface (reference - https://dave.cheney.net/practical-go/presentations/gophercon-israel.html#_prefer_single_method_interfaces) - // - Add general purpose methods such as `ActorOperation(enum_actor_type, ...)` which can be use like so: `Insert(FISHERMAN, ...)` - // - Use general purpose parameter methods such as `Set(enum_gov_type, ...)` such as `Set(STAKING_ADJUSTMENT, ...)` + // Context Operations NewSavePoint([]byte) error RollbackToSavePoint([]byte) error - Commit() error // DISCUSS: Can we consolidate `Reset` and `Release` Reset() error Release() error - // Question: - + // Block / indexer operations UpdateAppHash() ([]byte, error) - AppHash() ([]byte, error) - - // Block Operations - - // Indexer Operations - StoreTransaction(transactionProtoBytes []byte) error // Stores a transaction - CommitTransactions(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error // Writes the block in the SQL database + // Commits the current context (height, hash, transactions, etc...) to finality. + Commit(quorumCert []byte) error + // Indexes the transaction + StoreTransaction(transactionProtoBytes []byte) error // Stores a transaction // Pool Operations AddPoolAmount(name string, amount string) error diff --git a/shared/modules/utility_module.go b/shared/modules/utility_module.go index 8472e4582..5dbf9f887 100644 --- a/shared/modules/utility_module.go +++ b/shared/modules/utility_module.go @@ -21,8 +21,8 @@ type UtilityContext interface { ApplyBlock(height int64, proposer []byte, transactions [][]byte, lastBlockByzantineValidators [][]byte) (appHash []byte, err error) // Context operations - ReleaseContext() - CommitContext() error + ReleaseContext() error + CommitContext(quorumCert []byte) error // Validation operations CheckTransaction(tx []byte) error diff --git a/utility/block.go b/utility/block.go index ef2ab2894..324ee78a8 100644 --- a/utility/block.go +++ b/utility/block.go @@ -34,6 +34,8 @@ var ( func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, transactions [][]byte, lastBlockByzantineValidators [][]byte) ([]byte, error) { u.LatestHeight = latestHeight + u.CurrentProposer = proposerAddress + // begin block lifecycle phase if err := u.BeginBlock(lastBlockByzantineValidators); err != nil { return nil, err @@ -55,8 +57,8 @@ func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, return nil, err } - // TODO: if found, remove transaction from mempool // DISCUSS: What if the context is rolled back or cancelled. Do we add it back to the mempool? + // TODO: if found, remove transaction from mempool // if err := u.Mempool.DeleteTransaction(transaction); err != nil { // return nil, err // } @@ -67,8 +69,14 @@ func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, return nil, err } + // TODO: What if everything above succeeded but updating the app hash failed? + appHash, err := u.Context.UpdateAppHash() + if err != nil { + return nil, typesUtil.ErrAppHash(err) + } + // return the app hash; consensus module will get the validator set directly - return u.GetAppHash() + return appHash, nil } func (u *UtilityContext) BeginBlock(previousBlockByzantineValidators [][]byte) typesUtil.Error { @@ -91,21 +99,9 @@ func (u *UtilityContext) EndBlock(proposer []byte) typesUtil.Error { if err := u.BeginUnstakingMaxPaused(); err != nil { return err } - if _, err := u.Context.UpdateAppHash(); err != nil { - return typesUtil.ErrAppHash(err) - } return nil } -func (u *UtilityContext) GetAppHash() ([]byte, typesUtil.Error) { - // Get the root hash of the merkle state tree for state consensus integrity - appHash, er := u.Context.AppHash() - if er != nil { - return nil, typesUtil.ErrAppHash(er) - } - return appHash, nil -} - // HandleByzantineValidators handles the validators who either didn't sign at all or disagreed with the 2/3+ majority func (u *UtilityContext) HandleByzantineValidators(lastBlockByzantineValidators [][]byte) typesUtil.Error { latestBlockHeight, err := u.GetLatestHeight() @@ -289,14 +285,3 @@ func (u *UtilityContext) SetValidatorMissedBlocks(address []byte, missedBlocks i } return nil } - -func (u *UtilityContext) StoreBlock(blockProtoBytes []byte) error { - // store := u.Store() - - // Store in KV Store - // if err := store.StoreBlock(blockProtoBytes); err != nil { - // return err - // } - - return nil -} diff --git a/utility/context.go b/utility/context.go index 37f26d637..3541bb4e8 100644 --- a/utility/context.go +++ b/utility/context.go @@ -9,13 +9,16 @@ import ( ) type UtilityContext struct { - LatestHeight int64 - Mempool typesUtil.Mempool - Context *Context // IMPROVE: Consider renmaming to PersistenceContext + LatestHeight int64 // IMPROVE: Current height? + CurrentProposer []byte + + Mempool typesUtil.Mempool + Context *Context // IMPROVE: Consider renaming to `PersistenceContext` or `StoreContext` } type Context struct { modules.PersistenceRWContext + // TODO: SavePoints have not been implemented yet SavePointsM map[string]struct{} SavePoints [][]byte } @@ -40,15 +43,16 @@ func (u *UtilityContext) Store() *Context { return u.Context } -func (u *UtilityContext) CommitContext() error { - err := u.Context.PersistenceRWContext.Commit() +func (u *UtilityContext) CommitContext(quorumCert []byte) error { + err := u.Context.PersistenceRWContext.Commit(quorumCert) u.Context = nil return err } -func (u *UtilityContext) ReleaseContext() { - u.Context.Release() +func (u *UtilityContext) ReleaseContext() error { + err := u.Context.Release() u.Context = nil + return err } func (u *UtilityContext) GetLatestHeight() (int64, typesUtil.Error) { diff --git a/utility/test/block_test.go b/utility/test/block_test.go index c1441e405..f4a1420e9 100644 --- a/utility/test/block_test.go +++ b/utility/test/block_test.go @@ -3,12 +3,13 @@ package test import ( "encoding/hex" "fmt" - "github.com/pokt-network/pocket/shared/modules" - "github.com/pokt-network/pocket/shared/test_artifacts" "math" "math/big" "testing" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/pocket/shared/test_artifacts" + typesUtil "github.com/pokt-network/pocket/utility/types" "github.com/stretchr/testify/require" ) @@ -172,19 +173,6 @@ func TestUtilityContext_EndBlock(t *testing.T) { test_artifacts.CleanupTest(ctx) } -func TestUtilityContext_GetAppHash(t *testing.T) { - ctx := NewTestingUtilityContext(t, 0) - - appHashTest, err := ctx.GetAppHash() - require.NoError(t, err) - - appHashSource, er := ctx.Context.AppHash() - require.NoError(t, er) - require.Equal(t, appHashSource, appHashTest, "unexpected appHash") - - test_artifacts.CleanupTest(ctx) -} - func TestUtilityContext_UnstakeValidatorsActorsThatAreReady(t *testing.T) { for _, actorType := range typesUtil.ActorTypes { ctx := NewTestingUtilityContext(t, 1) From 1507de341ddb1a584c895bb3fc69009ad4d59ca1 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sat, 24 Sep 2022 13:35:15 -0700 Subject: [PATCH 19/56] Added a persistence block proto type --- persistence/StateHash.md | 15 +++++++++++++++ persistence/block.go | 29 +++++++++++++++++++++++------ persistence/context.go | 2 +- persistence/proto/block.proto | 13 +++++++++++++ persistence/state.go | 3 +-- persistence/types/base_actor.go | 1 + 6 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 persistence/proto/block.proto diff --git a/persistence/StateHash.md b/persistence/StateHash.md index a930df4fc..bd455f42e 100644 --- a/persistence/StateHash.md +++ b/persistence/StateHash.md @@ -1,3 +1,18 @@ +Remaining tasks: + +1. Simplify interfaces in utility & persistence (make it simple and clear) +2. How do we revert changes to the merkle trees? +3. Draw an end-to-end diagram of everything and the data flow + +## References: + +- https://github.com/cosmos/cosmos-sdk/discussions/9158 +- https://github.com/cosmos/cosmos-sdk/pull/8012 +- https://github.com/cosmos/cosmos-sdk/discussions/8297 +- https://github.com/cosmos/cosmos-sdk/discussions/8297 +- https://paper.dropbox.com/doc/State-commitments-and-storage-review--BpuF08OSksSkLeErjf66jrOAAg-wKl2RINZWD9I0DUmZIFwQ +- https://arxiv.org/pdf/1803.05069.pdf + # This discussion is aimed at: 1. Defining how we should compute the state hash diff --git a/persistence/block.go b/persistence/block.go index af9a797da..5409597ec 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "log" + "github.com/gogo/protobuf/proto" "github.com/pokt-network/pocket/persistence/types" ) @@ -66,10 +67,26 @@ func heightToBytes(height int64) []byte { return heightBytes } -func (p PostgresContext) commitBlock(blockProtoBytes []byte) error { - // get current height - // get proposer - // get hash - // get transaction - return p.DB.Blockstore.Put(heightToBytes(p.Height), blockProtoBytes) +func (p PostgresContext) storeBlock(quorumCert []byte) error { + prevHash, err := p.GetBlockHash(p.Height - 1) + if err != nil { + return err + } + + block := types.Block{ + Height: uint64(p.Height), + Hash: string(p.stateHash), + PrevHash: string(prevHash), + ProposerAddress: []byte("proposer"), // TODO: How should utility context forward this? + QuorumCertificate: quorumCert, + Transactions: nil, // TODO: get this from what was stored via `StoreTransaction` + + } + + blockBz, err := proto.Marshal(&block) + if err != nil { + return err + } + + return p.DB.Blockstore.Put(heightToBytes(p.Height), blockBz) } diff --git a/persistence/context.go b/persistence/context.go index 35e7ba108..9ca2e7f86 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -30,7 +30,7 @@ func (p PostgresContext) Commit(quorumCert []byte) error { log.Printf("About to commit context at height %d.\n", p.Height) ctx := context.TODO() - if err := p.DB.Tx.Commit(context.TODO()); err != nil { + if err := p.DB.Tx.Commit(ctx); err != nil { return err } if err := p.DB.conn.Close(ctx); err != nil { diff --git a/persistence/proto/block.proto b/persistence/proto/block.proto new file mode 100644 index 000000000..0c314690d --- /dev/null +++ b/persistence/proto/block.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; +package persistence; + +option go_package = "github.com/pokt-network/pocket/persistence/types"; + +message Block { + uint64 height = 1; + string hash = 2; + string prevHash = 3; + bytes proposerAddress = 4; + bytes quorumCertificate = 5; + repeated bytes transactions = 6; +} \ No newline at end of file diff --git a/persistence/state.go b/persistence/state.go index d689410ee..88a053138 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -51,7 +51,6 @@ func loadMerkleTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { log.Fatalf("loadMerkleTrees not implemented yet") } -// Question: Is this the right approach? func (p *PostgresContext) updateStateHash() error { // Update all the merkle trees for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { @@ -76,7 +75,7 @@ func (p *PostgresContext) updateStateHash() error { return err } } - // TODO_IN_THIS_COMMIT: re + // TODO_IN_THIS_COMMIT: add support for all the other actors as well default: log.Fatalln("Not handled yet in state commitment update", treeType) } diff --git a/persistence/types/base_actor.go b/persistence/types/base_actor.go index 054d10393..1c73da9a7 100644 --- a/persistence/types/base_actor.go +++ b/persistence/types/base_actor.go @@ -1,5 +1,6 @@ package types +// IMPROVE: Move schema related functions to a separate sub-package import "github.com/pokt-network/pocket/shared/modules" // TECHDEBT: Consider moving this to a protobuf. This struct was created to make testing simple of protocol actors that From f3fe3dfed05304362c5f5b49c4bd67f6a46d9419 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sun, 25 Sep 2022 17:28:19 -0700 Subject: [PATCH 20/56] Rename block persistence proto name --- go.mod | 2 +- persistence/proto/{block.proto => block_persistence.proto} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename persistence/proto/{block.proto => block_persistence.proto} (100%) diff --git a/go.mod b/go.mod index 6e76ac253..f83ae8336 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect + github.com/gogo/protobuf v1.3.2 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/persistence/proto/block.proto b/persistence/proto/block_persistence.proto similarity index 100% rename from persistence/proto/block.proto rename to persistence/proto/block_persistence.proto From 21448113eac5a145369e6fa2e99f65a6eb642f12 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Sun, 25 Sep 2022 19:39:17 -0700 Subject: [PATCH 21/56] Rough non-functional completion of merkle tree flow --- consensus/consensus_tests/utils_test.go | 8 -- consensus/hotstuff_handler.go | 6 +- consensus/hotstuff_leader.go | 3 +- persistence/StateHash.md | 107 ++++++++++++++++++++++++ persistence/block.go | 42 ++++++---- persistence/context.go | 15 +++- persistence/debug.go | 2 +- persistence/genesis.go | 2 +- persistence/state.go | 20 ++++- persistence/test/module_test.go | 3 +- shared/modules/consensus_module.go | 7 +- shared/modules/persistence_module.go | 5 +- utility/context.go | 13 +-- 13 files changed, 188 insertions(+), 45 deletions(-) diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 7f12277a2..ff716ce2c 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -358,7 +358,6 @@ func baseUtilityMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockUti func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { ctrl := gomock.NewController(t) utilityContextMock := modulesMock.NewMockUtilityContext(ctrl) - // persistenceContextMock := basePersistenceContextMock(t) utilityContextMock.EXPECT(). GetProposalTransactions(gomock.Any(), maxTxBytes, gomock.AssignableToTypeOf(emptyByzValidators)). @@ -374,13 +373,6 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { return utilityContextMock } -// func basePersistenceContextMock(t *testing.T) *modulesMock.MockPersistenceRWContext { -// ctrl := gomock.NewController(t) -// persistenceContextMock := modulesMock.NewMockPersistenceRWContext(ctrl) -// persistenceContextMock.EXPECT().Commit(gomock.Any()).Return(nil).AnyTimes() -// return persistenceContextMock -// } - func baseTelemetryMock(t *testing.T, _ modules.EventsChannel) *modulesMock.MockTelemetryModule { ctrl := gomock.NewController(t) telemetryMock := modulesMock.NewMockTelemetryModule(ctrl) diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index aa72b3a26..90b7b3b29 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -17,7 +17,8 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) m.nodeLog(typesCons.DebugHandlingHotstuffMessage(msg)) step := msg.GetStep() - // Liveness & safety checks + + // Pacemaker - Liveness & safety checks if err := m.paceMaker.ValidateMessage(msg); err != nil { // If a replica is not a leader for this round, but has already determined a leader, // and continues to receive NewRound messages, we avoid logging the "message discard" @@ -28,13 +29,14 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) return err } - // Need to execute leader election if there is no leader and we are in a new round. + // Leader Election - Need to execute leader election if there is no leader and we are in a new round. if m.Step == NewRound && m.LeaderId == nil { if err := m.electNextLeader(msg); err != nil { return err } } + // Hotstuff - Handle message if m.isReplica() { replicaHandlers[step](m, msg) } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index f87c5f58b..7529c5e53 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -337,7 +337,8 @@ func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { return nil, err } - // Apply all the transactions in the block - + // OPTIMIZE: Determine if we can avoid the `ApplyBlock` call here + // Apply all the transactions in the block appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) if err != nil { return nil, err diff --git a/persistence/StateHash.md b/persistence/StateHash.md index bd455f42e..cfc54af52 100644 --- a/persistence/StateHash.md +++ b/persistence/StateHash.md @@ -13,6 +13,11 @@ Remaining tasks: - https://paper.dropbox.com/doc/State-commitments-and-storage-review--BpuF08OSksSkLeErjf66jrOAAg-wKl2RINZWD9I0DUmZIFwQ - https://arxiv.org/pdf/1803.05069.pdf +## Open questions: + +1. Review flows +2. How do we revert changes to the merkle trees? + # This discussion is aimed at: 1. Defining how we should compute the state hash @@ -85,3 +90,105 @@ Learnings / Ideas: - Consolidate `UtilActorType` and `persistence.ActorType` - `modules.Actors` interface vs `types.Actor` in persistenceGenesis + +### Context Initialization + +```mermaid +sequenceDiagram + %% autonumber + participant N as Node + participant C as Consensus + participant U as Utility + participant P as Persistence + participant PP as Persistence (PostgresDB) + participant PM as Persistence (MerkleTree) + participant P2P as P2P + + %% Should this be P2P? + N-->>C: HandleMessage(anypb.Any) + critical NewRound Message + C->>+U: NewContext(height) + U->>P: NewRWContext(height) + P->>U: PersistenceRWContext + U->>U: Store persistenceContext + U->>-C: UtilityContext + C->>C: Store utilityContext + Note over C, PM: See 'Block Application' + end + Note over N, P2P: Hotstuff lifecycle + N-->>C: HandleMessage(anypb.Any) + critical Decide Message + Note over C, PM: See 'Block Commit' + end +``` + +### Block Application + +```mermaid +sequenceDiagram + participant C as Consensus + participant U as Utility + participant P as Persistence + participant PP as Persistence (PostgresDB) + participant PM as Persistence (MerkleTree) + + alt as leader + C->>+U: GetProposalTransactions(proposer, maxTxBz, [lastVal]) + U->>U: reap mempool + U->>-C: txs + Note over C, U: Perform replica behaviour + else as replica + C->>+U: ApplyBlock(height, proposer, txs, lastVals) + loop Update DB: for each operation in tx + U->>P: ReadOp | WriteOp + P->>PP: ReadOp | WriteOp + PP->>P: data | ok + P->>U: data | ok + U->>U: validate + U->>P: StoreTransaction(tx) + P->>P: store locally + P->>U: ok + end + U->>+P: UpdateAppHash() + loop for each protocol actor type + P->>PP: GetActorsUpdate(height) + PP->>P: actors + loop Update Tree: for each actor + P->>PM: Update(addr, serialized(actor)) + PM->>P: ok + end + P->>PM: GetRoot() + PM->>P: rootHash + end + P->>P: computeStateHash(rootHashes) + P->>-U: stateHash + U->>-C: hash + end +``` + +### Block Commit + +```mermaid +sequenceDiagram + %% autonumber + participant C as Consensus + participant U as Utility + participant P as Persistence + participant PP as Persistence (PostgresDB) + participant PK as Persistence (Key-Value Store) + + C->>U: CommitContext(quorumCert) + U->>P: Commit(proposerAddr, quorumCert) + P->>P: create typesPer.Block + P->>PP: insertBlock(block) + PP->>P: ok + P->>PK: Put(height, block) + PK->>P: ok + P->>P: commit tx + P->>U: ok + U->>P: Release() + P->>U: ok + C->>U: Release() + U->>C: ok + C->>C: release utilityContext +``` diff --git a/persistence/block.go b/persistence/block.go index 5409597ec..8c035edfb 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -50,43 +50,49 @@ func (p PostgresContext) StoreTransaction(transactionProtoBytes []byte) error { return nil } -func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { +func (p PostgresContext) insertBlock(block *types.Block) error { ctx, tx, err := p.DB.GetCtxAndTxn() if err != nil { return err } - _, err = tx.Exec(ctx, types.InsertBlockQuery(height, hash, proposerAddr, quorumCert)) + _, err = tx.Exec(ctx, types.InsertBlockQuery(block.Height, block.Hash, block.ProposerAddress, block.QuorumCertificate)) return err } -// CLEANUP: Should this be moved to a shared directory? -func heightToBytes(height int64) []byte { - heightBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(heightBytes, uint64(height)) - return heightBytes +func (p PostgresContext) storeBlock(block *types.Block) error { + blockBz, err := proto.Marshal(block) + if err != nil { + return err + } + + return p.DB.Blockstore.Put(heightToBytes(p.Height), blockBz) } -func (p PostgresContext) storeBlock(quorumCert []byte) error { +func (p PostgresContext) getBlock(proposerAddr []byte, quorumCert []byte) (*types.Block, error) { prevHash, err := p.GetBlockHash(p.Height - 1) if err != nil { - return err + return nil, err } - block := types.Block{ + // TODO: get this from the transactions that were store via `StoreTransaction` + txs := make([][]byte, 0) + + block := &types.Block{ Height: uint64(p.Height), Hash: string(p.stateHash), PrevHash: string(prevHash), - ProposerAddress: []byte("proposer"), // TODO: How should utility context forward this? + ProposerAddress: proposerAddr, QuorumCertificate: quorumCert, - Transactions: nil, // TODO: get this from what was stored via `StoreTransaction` - + Transactions: txs, } - blockBz, err := proto.Marshal(&block) - if err != nil { - return err - } + return block, nil +} - return p.DB.Blockstore.Put(heightToBytes(p.Height), blockBz) +// CLEANUP: Should this be moved to a shared directory? +func heightToBytes(height int64) []byte { + heightBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(heightBytes, uint64(height)) + return heightBytes } diff --git a/persistence/context.go b/persistence/context.go index 9ca2e7f86..95a54702e 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -26,9 +26,22 @@ func (p PostgresContext) Reset() error { panic("TODO: PostgresContext Reset not implemented") } -func (p PostgresContext) Commit(quorumCert []byte) error { +func (p PostgresContext) Commit(proposerAddr []byte, quorumCert []byte) error { log.Printf("About to commit context at height %d.\n", p.Height) + block, err := p.getBlock(proposerAddr, quorumCert) + if err != nil { + return err + } + + if err := p.insertBlock(block); err != nil { + return err + } + + if err := p.storeBlock(block); err != nil { + return err + } + ctx := context.TODO() if err := p.DB.Tx.Commit(ctx); err != nil { return err diff --git a/persistence/debug.go b/persistence/debug.go index 14fc5c704..e0e3b6e5e 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -44,7 +44,7 @@ func (m *PersistenceModule) showLatestBlockInStore(_ *debug.DebugMessage) { func (m *PersistenceModule) clearState(_ *debug.DebugMessage) { context, err := m.NewRWContext(-1) - defer context.Commit(nil) + defer context.Commit(nil, nil) if err != nil { log.Printf("Error creating new context: %s \n", err) return diff --git a/persistence/genesis.go b/persistence/genesis.go index 329cdbfa4..feb8dc9b5 100644 --- a/persistence/genesis.go +++ b/persistence/genesis.go @@ -149,7 +149,7 @@ func (m *PersistenceModule) populateGenesisState(state *types.PersistenceGenesis log.Fatalf("an error occurred initializing flags: %s", err.Error()) } - if err = rwContext.Commit(nil); err != nil { + if err = rwContext.Commit(nil, nil); err != nil { log.Fatalf("an error occurred during commit() on genesis state %s ", err.Error()) } } diff --git a/persistence/state.go b/persistence/state.go index 88a053138..5c3913140 100644 --- a/persistence/state.go +++ b/persistence/state.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "encoding/hex" + "fmt" "log" "sort" @@ -75,7 +76,24 @@ func (p *PostgresContext) updateStateHash() error { return err } } - // TODO_IN_THIS_COMMIT: add support for all the other actors as well + case valMerkleTree: + fmt.Println("TODO: valMerkleTree not implemented") + case fishMerkleTree: + fmt.Println("TODO: fishMerkleTree not implemented") + case serviceNodeMerkleTree: + fmt.Println("TODO: serviceNodeMerkleTree not implemented") + case accountMerkleTree: + fmt.Println("TODO: accountMerkleTree not implemented") + case poolMerkleTree: + fmt.Println("TODO: poolMerkleTree not implemented") + case blocksMerkleTree: + // VERY VERY IMPORTANT DISCUSSION: What do we do here provided that `Commit`, which stores the block in the DB and tree + // requires the quorumCert, which we receive at the very end of hotstuff consensus + fmt.Println("TODO: blocksMerkleTree not implemented") + case paramsMerkleTree: + fmt.Println("TODO: paramsMerkleTree not implemented") + case flagsMerkleTree: + fmt.Println("TODO: flagsMerkleTree not implemented") default: log.Fatalln("Not handled yet in state commitment update", treeType) } diff --git a/persistence/test/module_test.go b/persistence/test/module_test.go index f5df476b0..ccb7bd37f 100644 --- a/persistence/test/module_test.go +++ b/persistence/test/module_test.go @@ -18,13 +18,14 @@ func TestPersistenceContextParallelReadWrite(t *testing.T) { poolAddress := []byte("address") originalAmount := "15" modifiedAmount := "10" + proposer := []byte("proposer") quorumCert := []byte("quorumCert") // setup a write context, insert a pool and commit it context, err := testPersistenceMod.NewRWContext(0) require.NoError(t, err) require.NoError(t, context.InsertPool(poolName, poolAddress, originalAmount)) - require.NoError(t, context.Commit(quorumCert)) + require.NoError(t, context.Commit(proposer, quorumCert)) // verify the insert in the previously committed context worked contextA, err := testPersistenceMod.NewRWContext(0) diff --git a/shared/modules/consensus_module.go b/shared/modules/consensus_module.go index cd05f73ec..044b2a3bd 100644 --- a/shared/modules/consensus_module.go +++ b/shared/modules/consensus_module.go @@ -7,16 +7,17 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) -type ValidatorMap map[string]Actor // TODO (Drewsky) deprecate Validator map or populate from persistence module +// TODO(olshansky): deprecate ValidatorMap or populate from persistence module +type ValidatorMap map[string]Actor type ConsensusModule interface { Module - // Consensus Engine + // Consensus Engine Handlers HandleMessage(*anypb.Any) error HandleDebugMessage(*debug.DebugMessage) error - // Consensus State + // Consensus State Accessors CurrentHeight() uint64 AppHash() string // DISCUSS: Why not call this a BlockHash or StateHash? Should it be a []byte or string? ValidatorMap() ValidatorMap // TODO: This needs to be dynamically updated during various operations and network changes. diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 4da6fb211..a4a587bb9 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -9,6 +9,8 @@ import ( type PersistenceModule interface { Module + + // Persistence Context Factory Methods NewRWContext(height int64) (PersistenceRWContext, error) NewReadContext(height int64) (PersistenceReadContext, error) @@ -46,7 +48,6 @@ type PersistenceRWContext interface { // but it abstracts and contrasts nicely against the read only context // TODO (andrew) convert address and public key to string not bytes #149 type PersistenceWriteContext interface { - // Context Operations NewSavePoint([]byte) error RollbackToSavePoint([]byte) error @@ -58,7 +59,7 @@ type PersistenceWriteContext interface { // Block / indexer operations UpdateAppHash() ([]byte, error) // Commits the current context (height, hash, transactions, etc...) to finality. - Commit(quorumCert []byte) error + Commit(proposerAddr []byte, quorumCert []byte) error // Indexes the transaction StoreTransaction(transactionProtoBytes []byte) error // Stores a transaction diff --git a/utility/context.go b/utility/context.go index 3541bb4e8..b0ca4bebc 100644 --- a/utility/context.go +++ b/utility/context.go @@ -9,16 +9,17 @@ import ( ) type UtilityContext struct { - LatestHeight int64 // IMPROVE: Current height? + LatestHeight int64 // IMPROVE: Rename to `currentHeight?` CurrentProposer []byte Mempool typesUtil.Mempool - Context *Context // IMPROVE: Consider renaming to `PersistenceContext` or `StoreContext` + Context *Context // IMPROVE: Rename to `persistenceContext` or `storeContext` or `reversibleContext`? } -type Context struct { +type Context struct { // IMPROVE: Rename to `persistenceContext` or `storeContext`? + // TODO: Since `Context` embeds `PersistenceRWContext`, we don't need to do `u.Context.PersistenceRWContext`, but can call `u.Context` directly modules.PersistenceRWContext - // TODO: SavePoints have not been implemented yet + // TODO/DISCUSS: `SavePoints`` have not been implemented yet SavePointsM map[string]struct{} SavePoints [][]byte } @@ -44,8 +45,8 @@ func (u *UtilityContext) Store() *Context { } func (u *UtilityContext) CommitContext(quorumCert []byte) error { - err := u.Context.PersistenceRWContext.Commit(quorumCert) - u.Context = nil + err := u.Context.PersistenceRWContext.Commit(u.CurrentProposer, quorumCert) + u.Context = nil // DISCUSS: Should we release the context if there was an error here? return err } From d9a1cf5bd52b107982866096cb901c64d87d9c84 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 13:51:55 -0700 Subject: [PATCH 22/56] Remove test_persistence_state_hash --- Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Makefile b/Makefile index 887652723..b174229b8 100644 --- a/Makefile +++ b/Makefile @@ -302,11 +302,6 @@ test_sortition: test_persistence: go test ${VERBOSE_TEST} -p=1 -count=1 ./persistence/... -.PHONY: test_persistence_state_hash -## Run all go unit tests in the Persistence module -test_persistence_state_hash: - go test -run StateHash ${VERBOSE_TEST} -p=1 ./persistence/... - .PHONY: test_p2p_types ## Run p2p subcomponents' tests test_p2p_types: From 798ea1f6904eb1148d05fdab47013b46a6b3b395 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:04:48 -0700 Subject: [PATCH 23/56] Removed explicit MaxBlockBytes variable --- consensus/block.go | 37 +++++++++++++++--------------- consensus/module.go | 19 ++++++++------- shared/test_artifacts/generator.go | 5 ++-- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/consensus/block.go b/consensus/block.go index 48c452b2e..0f43a47e5 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -7,7 +7,23 @@ import ( typesCons "github.com/pokt-network/pocket/consensus/types" ) -// TODO: Add additional basic block metadata validation w/ unit tests +func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { + m.nodeLog(typesCons.CommittingBlock(m.Height, len(block.Transactions))) + + // Commit the utility context + if err := m.UtilityContext.CommitContext(block.BlockHeader.QuorumCertificate); err != nil { + return err + } + // Release the utility context + m.UtilityContext.ReleaseContext() + m.UtilityContext = nil + // Update the last app hash + m.lastAppHash = block.BlockHeader.Hash + + return nil +} + +// TODO: Add unit tests specific to block validation func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { if block == nil && m.Step != NewRound { return typesCons.ErrNilBlock @@ -17,8 +33,8 @@ func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { return typesCons.ErrBlockExists } - if unsafe.Sizeof(*block) > uintptr(m.MaxBlockBytes) { - return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.MaxBlockBytes) + if unsafe.Sizeof(*block) > uintptr(m.consGenesis.MaxBlockBytes) { + return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.consGenesis.MaxBlockBytes) } // If the current block being processed (i.e. voted on) by consensus is non nil, we need to make @@ -56,18 +72,3 @@ func (m *ConsensusModule) refreshUtilityContext() error { m.UtilityContext = utilityContext return nil } - -func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { - m.nodeLog(typesCons.CommittingBlock(m.Height, len(block.Transactions))) - - // Commit and release the context - if err := m.UtilityContext.CommitContext(block.BlockHeader.QuorumCertificate); err != nil { - return err - } - m.UtilityContext.ReleaseContext() - m.UtilityContext = nil - - m.lastAppHash = block.BlockHeader.Hash - - return nil -} diff --git a/consensus/module.go b/consensus/module.go index 68e3874cc..f49e4a97e 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -31,7 +31,9 @@ var _ modules.ConsensusModule = &ConsensusModule{} type ConsensusModule struct { bus modules.Bus privateKey cryptoPocket.Ed25519PrivateKey - consCfg modules.ConsensusConfig + + consCfg *typesCons.ConsensusConfig + consGenesis *typesCons.ConsensusGenesisState // Hotstuff Height uint64 @@ -49,7 +51,7 @@ type ConsensusModule struct { IdToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified // Consensus State - lastAppHash string // TODO: Need to make sure this is populated and updated correctly + lastAppHash string // TODO: Always retrieve this variable from the persistence module and simplify this struct validatorMap typesCons.ValidatorMap // Module Dependencies @@ -62,9 +64,6 @@ type ConsensusModule struct { // TECHDEBT: Move this over to use the txIndexer MessagePool map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage - - // CLEANUP: Access this value from the configs - MaxBlockBytes uint64 } func Create(configPath, genesisPath string, useRandomPK bool) (modules.ConsensusModule, error) { @@ -106,8 +105,9 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus m := &ConsensusModule{ bus: nil, - privateKey: privateKey.(cryptoPocket.Ed25519PrivateKey), - consCfg: cfg, + privateKey: privateKey.(cryptoPocket.Ed25519PrivateKey), + consCfg: cfg, + consGenesis: genesis, Height: 0, Round: 0, @@ -129,9 +129,8 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus paceMaker: paceMaker, leaderElectionMod: leaderElectionMod, - logPrefix: DefaultLogPrefix, - MessagePool: make(map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage), - MaxBlockBytes: genesis.GetMaxBlockBytes(), + logPrefix: DefaultLogPrefix, + MessagePool: make(map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage), } // TODO(olshansky): Look for a way to avoid doing this. diff --git a/shared/test_artifacts/generator.go b/shared/test_artifacts/generator.go index 912e9fe2b..f89c68188 100644 --- a/shared/test_artifacts/generator.go +++ b/shared/test_artifacts/generator.go @@ -2,11 +2,12 @@ package test_artifacts import ( "fmt" + "math/big" + "strconv" + typesPersistence "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/utility/types" - "math/big" - "strconv" "github.com/pokt-network/pocket/shared/crypto" "google.golang.org/protobuf/types/known/timestamppb" From 7cdc00838fa50ff12da7f6636288d8e391fb6c09 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:07:19 -0700 Subject: [PATCH 24/56] Update block.go --- consensus/block.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/consensus/block.go b/consensus/block.go index 0f43a47e5..0ce2851e6 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -33,7 +33,7 @@ func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { return typesCons.ErrBlockExists } - if unsafe.Sizeof(*block) > uintptr(m.consGenesis.MaxBlockBytes) { + if block != nil && unsafe.Sizeof(*block) > uintptr(m.consGenesis.MaxBlockBytes) { return typesCons.ErrInvalidBlockSize(uint64(unsafe.Sizeof(*block)), m.consGenesis.MaxBlockBytes) } @@ -41,7 +41,7 @@ func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { // sure that the data (height, round, step, txs, etc) is the same before we start validating the signatures if m.Block != nil { // DISCUSS: The only difference between blocks from one step to another is the QC, so we need - // to determine where/how to validate this + // to determine where/how to validate this if protoHash(m.Block) != protoHash(block) { log.Println("[TECHDEBT][ERROR] The block being processed is not the same as that received by the consensus module ") } @@ -52,9 +52,8 @@ func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { // Creates a new Utility context and clears/nullifies any previous contexts if they exist func (m *ConsensusModule) refreshUtilityContext() error { - // This is a catch-all to release the previous utility context if it wasn't cleaned up - // in the proper lifecycle (e.g. catch up, error, network partition, etc...). Ideally, this - // should not be called. + // Catch-all structure to release the previous utility context if it wasn't properly cleaned up. + // Ideally, this should not be called. if m.UtilityContext != nil { m.nodeLog(typesCons.NilUtilityContextWarning) if err := m.UtilityContext.ReleaseContext(); err != nil { @@ -68,7 +67,7 @@ func (m *ConsensusModule) refreshUtilityContext() error { if err != nil { return err } - m.UtilityContext = utilityContext + return nil } From e4a63a8879c2af61d4deedadfbefe94a8b32bcea Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:10:48 -0700 Subject: [PATCH 25/56] Fix typo in consensus proto --- consensus/helpers.go | 2 +- consensus/types/proto/hotstuff_types.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/helpers.go b/consensus/helpers.go index 63270826f..eab022ec4 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -26,7 +26,7 @@ const ( ByzantineThreshold = float64(2) / float64(3) HotstuffMessage = "consensus.HotstuffMessage" UtilityMessage = "consensus.UtilityMessage" - Propose = typesCons.HotstuffMessageType_HOTSTUFF_MESAGE_PROPOSE + Propose = typesCons.HotstuffMessageType_HOTSTUFF_MESSAGE_PROPOSE Vote = typesCons.HotstuffMessageType_HOTSTUFF_MESSAGE_VOTE ) diff --git a/consensus/types/proto/hotstuff_types.proto b/consensus/types/proto/hotstuff_types.proto index b1f7c9b4d..c289694c8 100644 --- a/consensus/types/proto/hotstuff_types.proto +++ b/consensus/types/proto/hotstuff_types.proto @@ -18,7 +18,7 @@ enum HotstuffStep { enum HotstuffMessageType { HOTSTUFF_MESSAGE_UNKNOWN = 0; - HOTSTUFF_MESAGE_PROPOSE = 1; + HOTSTUFF_MESSAGE_PROPOSE = 1; HOTSTUFF_MESSAGE_VOTE = 2; } From b2194685c6b0a220b04ee798d4e2980bbede7d6d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:16:29 -0700 Subject: [PATCH 26/56] Updated interface to findHighQC --- consensus/helpers.go | 26 +++++++++++++------------- consensus/hotstuff_leader.go | 3 ++- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/consensus/helpers.go b/consensus/helpers.go index eab022ec4..54a2491c6 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -1,5 +1,6 @@ package consensus +// TODO: Split this file into multiple helpers (e.g. signatures.go, hotstuff_helpers.go, etc...) import ( "encoding/base64" "log" @@ -15,7 +16,6 @@ import ( // These constants and variables are wrappers around the autogenerated protobuf types and were // added to simply make the code in the `consensus` module more readable. - const ( NewRound = typesCons.HotstuffStep_HOTSTUFF_STEP_NEWROUND Prepare = typesCons.HotstuffStep_HOTSTUFF_STEP_PREPARE @@ -23,11 +23,13 @@ const ( Commit = typesCons.HotstuffStep_HOTSTUFF_STEP_COMMIT Decide = typesCons.HotstuffStep_HOTSTUFF_STEP_DECIDE + Propose = typesCons.HotstuffMessageType_HOTSTUFF_MESSAGE_PROPOSE + Vote = typesCons.HotstuffMessageType_HOTSTUFF_MESSAGE_VOTE + ByzantineThreshold = float64(2) / float64(3) - HotstuffMessage = "consensus.HotstuffMessage" - UtilityMessage = "consensus.UtilityMessage" - Propose = typesCons.HotstuffMessageType_HOTSTUFF_MESSAGE_PROPOSE - Vote = typesCons.HotstuffMessageType_HOTSTUFF_MESSAGE_VOTE + + HotstuffMessage = "consensus.HotstuffMessage" + UtilityMessage = "consensus.UtilityMessage" ) var ( @@ -36,22 +38,21 @@ var ( // ** Hotstuff Helpers ** // -// TODO: Make this method functional (i.e. not have the ConsensusModule receiver) +// IMPROVE: Avoid having the `ConsensusModule` be a receiver of this; making it more functional. +// TODO: Add unit tests for quorumCert creation & validation. func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.HotstuffStep, round uint64) (*typesCons.QuorumCertificate, error) { var pss []*typesCons.PartialSignature for _, msg := range m.MessagePool[step] { - // TODO(olshansky): Add tests for this if msg.GetPartialSignature() == nil { m.nodeLog(typesCons.WarnMissingPartialSig(msg)) continue } - // TODO(olshansky): Add tests for this if msg.GetHeight() != height || msg.GetStep() != step || msg.GetRound() != round { m.nodeLog(typesCons.WarnUnexpectedMessageInPool(msg, height, step, round)) continue } - ps := msg.GetPartialSignature() + ps := msg.GetPartialSignature() if ps.Signature == nil || len(ps.Address) == 0 { m.nodeLog(typesCons.WarnIncompletePartialSig(ps, msg)) continue @@ -77,8 +78,8 @@ func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.Hot }, nil } -func (m *ConsensusModule) findHighQC(step typesCons.HotstuffStep) (qc *typesCons.QuorumCertificate) { - for _, m := range m.MessagePool[step] { +func (m *ConsensusModule) findHighQC(msgs []*typesCons.HotstuffMessage) (qc *typesCons.QuorumCertificate) { + for _, m := range msgs { if m.GetQuorumCertificate() == nil { continue } @@ -89,8 +90,7 @@ func (m *ConsensusModule) findHighQC(step typesCons.HotstuffStep) (qc *typesCons return } -func getThresholdSignature( - partialSigs []*typesCons.PartialSignature) (*typesCons.ThresholdSignature, error) { +func getThresholdSignature(partialSigs []*typesCons.PartialSignature) (*typesCons.ThresholdSignature, error) { thresholdSig := new(typesCons.ThresholdSignature) thresholdSig.Signatures = make([]*typesCons.PartialSignature, len(partialSigs)) copy(thresholdSig.Signatures, partialSigs) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 7529c5e53..4aba0298b 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -46,7 +46,8 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM } // Likely to be `nil` if blockchain is progressing well. - highPrepareQC := m.findHighQC(NewRound) // TECHDEBT: How do we validate `highPrepareQC` here? + // TECHDEBT: How do we properly validate `highPrepareQC` here? + highPrepareQC := m.findHighQC(m.MessagePool[NewRound]) // TODO: Add more unit tests for these checks... if highPrepareQC == nil || highPrepareQC.Height < m.Height || highPrepareQC.Round < m.Round { From fa3a051046f895c8fd09752d0111f39ddd4f5323 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:20:24 -0700 Subject: [PATCH 27/56] Reverted changes to the utility module to simplify the PR --- utility/block.go | 53 +++++++++++++++++++++++++++----------- utility/context.go | 28 +++++++++----------- utility/test/block_test.go | 18 ++++++++++--- 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/utility/block.go b/utility/block.go index 324ee78a8..6c09cd8cf 100644 --- a/utility/block.go +++ b/utility/block.go @@ -3,7 +3,7 @@ package utility import ( "math/big" - // TODO (andrew) importing consensus and persistence in this file? + typesCons "github.com/pokt-network/pocket/consensus/types" // TODO (andrew) importing consensus and persistence in this file? typesGenesis "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" @@ -34,8 +34,6 @@ var ( func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, transactions [][]byte, lastBlockByzantineValidators [][]byte) ([]byte, error) { u.LatestHeight = latestHeight - u.CurrentProposer = proposerAddress - // begin block lifecycle phase if err := u.BeginBlock(lastBlockByzantineValidators); err != nil { return nil, err @@ -53,30 +51,22 @@ func (u *UtilityContext) ApplyBlock(latestHeight int64, proposerAddress []byte, if err := u.ApplyTransaction(tx); err != nil { return nil, err } - if err := u.Context.PersistenceRWContext.StoreTransaction(transactionProtoBytes); err != nil { + if err := u.GetPersistenceContext().StoreTransaction(transactionProtoBytes); err != nil { return nil, err } - // DISCUSS: What if the context is rolled back or cancelled. Do we add it back to the mempool? // TODO: if found, remove transaction from mempool + // DISCUSS: What if the context is rolled back or cancelled. Do we add it back to the mempool? // if err := u.Mempool.DeleteTransaction(transaction); err != nil { // return nil, err // } } - // end block lifecycle phase if err := u.EndBlock(proposerAddress); err != nil { return nil, err } - - // TODO: What if everything above succeeded but updating the app hash failed? - appHash, err := u.Context.UpdateAppHash() - if err != nil { - return nil, typesUtil.ErrAppHash(err) - } - - // return the app hash; consensus module will get the validator set directly - return appHash, nil + // return the app hash (consensus module will get the validator set directly + return u.GetAppHash() } func (u *UtilityContext) BeginBlock(previousBlockByzantineValidators [][]byte) typesUtil.Error { @@ -102,6 +92,15 @@ func (u *UtilityContext) EndBlock(proposer []byte) typesUtil.Error { return nil } +func (u *UtilityContext) GetAppHash() ([]byte, typesUtil.Error) { + // Get the root hash of the merkle state tree for state consensus integrity + appHash, er := u.Context.AppHash() + if er != nil { + return nil, typesUtil.ErrAppHash(er) + } + return appHash, nil +} + // HandleByzantineValidators handles the validators who either didn't sign at all or disagreed with the 2/3+ majority func (u *UtilityContext) HandleByzantineValidators(lastBlockByzantineValidators [][]byte) typesUtil.Error { latestBlockHeight, err := u.GetLatestHeight() @@ -285,3 +284,27 @@ func (u *UtilityContext) SetValidatorMissedBlocks(address []byte, missedBlocks i } return nil } + +func (u *UtilityContext) StoreBlock(blockProtoBytes []byte) error { + store := u.Store() + + // Store in KV Store + if err := store.StoreBlock(blockProtoBytes); err != nil { + return err + } + + // Store in SQL Store + // OPTIMIZE: Ideally we'd pass in the block proto struct to utility so we don't + // have to unmarshal it here, but that's a major design decision for the interfaces. + codec := u.Codec() + block := &typesCons.Block{} + if err := codec.Unmarshal(blockProtoBytes, block); err != nil { + return typesUtil.ErrProtoUnmarshal(err) + } + header := block.BlockHeader + if err := store.InsertBlock(uint64(header.Height), header.Hash, header.ProposerAddress, header.QuorumCertificate); err != nil { + return err + } + + return nil +} diff --git a/utility/context.go b/utility/context.go index b0ca4bebc..460b7cfa1 100644 --- a/utility/context.go +++ b/utility/context.go @@ -2,24 +2,19 @@ package utility import ( "encoding/hex" - "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/modules" typesUtil "github.com/pokt-network/pocket/utility/types" ) type UtilityContext struct { - LatestHeight int64 // IMPROVE: Rename to `currentHeight?` - CurrentProposer []byte - - Mempool typesUtil.Mempool - Context *Context // IMPROVE: Rename to `persistenceContext` or `storeContext` or `reversibleContext`? + LatestHeight int64 + Mempool typesUtil.Mempool + Context *Context // IMPROVE: Consider renmaming to PersistenceContext } -type Context struct { // IMPROVE: Rename to `persistenceContext` or `storeContext`? - // TODO: Since `Context` embeds `PersistenceRWContext`, we don't need to do `u.Context.PersistenceRWContext`, but can call `u.Context` directly +type Context struct { modules.PersistenceRWContext - // TODO/DISCUSS: `SavePoints`` have not been implemented yet SavePointsM map[string]struct{} SavePoints [][]byte } @@ -44,16 +39,17 @@ func (u *UtilityContext) Store() *Context { return u.Context } -func (u *UtilityContext) CommitContext(quorumCert []byte) error { - err := u.Context.PersistenceRWContext.Commit(u.CurrentProposer, quorumCert) - u.Context = nil // DISCUSS: Should we release the context if there was an error here? - return err +func (u *UtilityContext) GetPersistenceContext() modules.PersistenceRWContext { + return u.Context.PersistenceRWContext +} + +func (u *UtilityContext) CommitPersistenceContext() error { + return u.Context.PersistenceRWContext.Commit() } -func (u *UtilityContext) ReleaseContext() error { - err := u.Context.Release() +func (u *UtilityContext) ReleaseContext() { + u.Context.Release() u.Context = nil - return err } func (u *UtilityContext) GetLatestHeight() (int64, typesUtil.Error) { diff --git a/utility/test/block_test.go b/utility/test/block_test.go index f4a1420e9..c1441e405 100644 --- a/utility/test/block_test.go +++ b/utility/test/block_test.go @@ -3,13 +3,12 @@ package test import ( "encoding/hex" "fmt" + "github.com/pokt-network/pocket/shared/modules" + "github.com/pokt-network/pocket/shared/test_artifacts" "math" "math/big" "testing" - "github.com/pokt-network/pocket/shared/modules" - "github.com/pokt-network/pocket/shared/test_artifacts" - typesUtil "github.com/pokt-network/pocket/utility/types" "github.com/stretchr/testify/require" ) @@ -173,6 +172,19 @@ func TestUtilityContext_EndBlock(t *testing.T) { test_artifacts.CleanupTest(ctx) } +func TestUtilityContext_GetAppHash(t *testing.T) { + ctx := NewTestingUtilityContext(t, 0) + + appHashTest, err := ctx.GetAppHash() + require.NoError(t, err) + + appHashSource, er := ctx.Context.AppHash() + require.NoError(t, er) + require.Equal(t, appHashSource, appHashTest, "unexpected appHash") + + test_artifacts.CleanupTest(ctx) +} + func TestUtilityContext_UnstakeValidatorsActorsThatAreReady(t *testing.T) { for _, actorType := range typesUtil.ActorTypes { ctx := NewTestingUtilityContext(t, 1) From 752992b15c7b13b01a6f79831b09cb329457ce76 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:49:56 -0700 Subject: [PATCH 28/56] Reverted changes to the utility module interface --- shared/modules/utility_module.go | 9 +++--- shared/types/genesis/validator.go | 49 ------------------------------- 2 files changed, 4 insertions(+), 54 deletions(-) delete mode 100644 shared/types/genesis/validator.go diff --git a/shared/modules/utility_module.go b/shared/modules/utility_module.go index 5dbf9f887..67617399a 100644 --- a/shared/modules/utility_module.go +++ b/shared/modules/utility_module.go @@ -14,15 +14,14 @@ type UtilityModule interface { // operations. type UtilityContext interface { // Block operations - - // Reaps the mempool for transactions that are ready to be proposed in a new block GetProposalTransactions(proposer []byte, maxTransactionBytes int, lastBlockByzantineValidators [][]byte) (transactions [][]byte, err error) - // Applies the transactions to an ephemeral state in the utility & underlying persistence context;similar to `SafeNode` in the Hotstuff whitepaper. ApplyBlock(height int64, proposer []byte, transactions [][]byte, lastBlockByzantineValidators [][]byte) (appHash []byte, err error) + StoreBlock(blockProtoBytes []byte) error // Context operations - ReleaseContext() error - CommitContext(quorumCert []byte) error + ReleaseContext() + GetPersistenceContext() PersistenceRWContext + CommitPersistenceContext() error // Validation operations CheckTransaction(tx []byte) error diff --git a/shared/types/genesis/validator.go b/shared/types/genesis/validator.go deleted file mode 100644 index 17c2af5fd..000000000 --- a/shared/types/genesis/validator.go +++ /dev/null @@ -1,49 +0,0 @@ -package genesis - -// import ( -// "encoding/hex" -// "encoding/json" - -// "google.golang.org/protobuf/encoding/protojson" -// ) - -// // TODO_IN_THIS_COMMIT: See https://github.com/pokt-network/pocket/pull/139/files to remove this shit - -// // HACK: Since the protocol actor protobufs (e.g. validator, fisherman, etc) use `bytes` for some -// // fields (e.g. `address`, `output`, `publicKey`), we need to use a helper struct to unmarshal the -// // the types when they are defined via json (e.g. genesis file, testing configurations, etc...). -// // Alternative solutions could include whole wrapper structs (i.e. duplication of schema definition), -// // using strings instead of bytes (i.e. major change with downstream effects) or avoid defining these -// // types in json altogether (i.e. limitation of usability). -// type JsonBytesLoaderHelper struct { -// Address HexData `json:"address,omitempty"` -// PublicKey HexData `json:"public_key,omitempty"` -// Output HexData `json:"output,omitempty"` -// } - -// type HexData []byte - -// func (h *HexData) UnmarshalJSON(data []byte) error { -// var s string -// if err := json.Unmarshal(data, &s); err != nil { -// return err -// } -// decoded, err := hex.DecodeString(s) -// if err != nil { -// return err -// } -// *h = HexData(decoded) -// return nil -// } - -// func (v *Validator) UnmarshalJSON(data []byte) error { -// var jh JsonBytesLoaderHelper -// json.Unmarshal(data, &jh) - -// protojson.Unmarshal(data, v) -// v.Address = jh.Address -// v.PublicKey = jh.PublicKey -// v.Output = jh.Output - -// return nil -// } From 844688f7c010d9e41cbc267142efa19ff0a50010 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:50:15 -0700 Subject: [PATCH 29/56] Reverted changes to the persistence module interface --- shared/modules/persistence_module.go | 79 +++++++--------------------- 1 file changed, 18 insertions(+), 61 deletions(-) diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index a4a587bb9..edc0e30cc 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -3,14 +3,12 @@ package modules //go:generate mockgen -source=$GOFILE -destination=./mocks/persistence_module_mock.go -aux_files=github.com/pokt-network/pocket/shared/modules=module.go import ( - "github.com/pokt-network/pocket/persistence/kvstore" // Should be moved to shared + "github.com/pokt-network/pocket/persistence/kvstore" "github.com/pokt-network/pocket/shared/debug" ) type PersistenceModule interface { Module - - // Persistence Context Factory Methods NewRWContext(height int64) (PersistenceRWContext, error) NewReadContext(height int64) (PersistenceReadContext, error) @@ -40,28 +38,34 @@ type PersistenceRWContext interface { PersistenceWriteContext } -// TODO: Simplify the interface (reference - https://dave.cheney.net/practical-go/presentations/gophercon-israel.html#_prefer_single_method_interfaces) -// - Add general purpose methods such as `ActorOperation(enum_actor_type, ...)` which can be use like so: `Insert(FISHERMAN, ...)` -// - Use general purpose parameter methods such as `Set(enum_gov_type, ...)` such as `Set(STAKING_ADJUSTMENT, ...)` - // NOTE: There's not really a use case for a write only interface, // but it abstracts and contrasts nicely against the read only context // TODO (andrew) convert address and public key to string not bytes #149 type PersistenceWriteContext interface { + // TODO: Simplify the interface (reference - https://dave.cheney.net/practical-go/presentations/gophercon-israel.html#_prefer_single_method_interfaces) + // - Add general purpose methods such as `ActorOperation(enum_actor_type, ...)` which can be use like so: `Insert(FISHERMAN, ...)` + // - Use general purpose parameter methods such as `Set(enum_gov_type, ...)` such as `Set(STAKING_ADJUSTMENT, ...)` // Context Operations NewSavePoint([]byte) error RollbackToSavePoint([]byte) error - // DISCUSS: Can we consolidate `Reset` and `Release` Reset() error + Commit() error Release() error - // Block / indexer operations - UpdateAppHash() ([]byte, error) - // Commits the current context (height, hash, transactions, etc...) to finality. - Commit(proposerAddr []byte, quorumCert []byte) error - // Indexes the transaction - StoreTransaction(transactionProtoBytes []byte) error // Stores a transaction + AppHash() ([]byte, error) + + // Block Operations + + // Indexer Operations + StoreTransaction(transactionProtoBytes []byte) error + + // Block Operations + // TODO_TEMPORARY: Including two functions for the SQL and KV Store as an interim solution + // until we include the schema as part of the SQL Store because persistence + // currently has no access to the protobuf schema which is the source of truth. + StoreBlock(blockProtoBytes []byte) error // Store the block in the KV Store + InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error // Writes the block in the SQL database // Pool Operations AddPoolAmount(name string, amount string) error @@ -122,29 +126,6 @@ type PersistenceWriteContext interface { // Flag Operations InitFlags() error SetFlag(paramName string, value interface{}, enabled bool) error - - // Tree Operations - - // # Option 1: - - UpdateApplicationsTree([]Actor) error - // UpdateValidatorsTree([]Actor) error - // UpdateServiceNodesTree([]Actor) error - // UpdateFishermanTree([]Actor) error - // UpdateTree([]Actor) error - // UpdateTree([]Other) error - - // # Option 2: - // UpdateActorTree(types.ProtocolActorSchema, []Actor) error - // UpdateTree([]Other) error - - // # Option 3: - // UpdateApplicationsTree([]Application) error - // UpdateValidatorsTree([]Validator) error - // UpdateServiceNodesTree([]ServiceNode) error - // UpdateFishermanTree([]Fisherman) error - // UpdateTree([]FutureActor) error - // UpdateTree([]Other) error } type PersistenceReadContext interface { @@ -223,28 +204,4 @@ type PersistenceReadContext interface { GetIntFlag(paramName string, height int64) (int, bool, error) GetStringFlag(paramName string, height int64) (string, bool, error) GetBytesFlag(paramName string, height int64) ([]byte, bool, error) - - // Tree Operations - - // # Option 1: - - // GetApplicationsUpdatedAtHeight(height int64) ([]Actor, error) - // GetValidatorsUpdatedAtHeight(height int64) ([]Actor, error) - // GetServiceNodesUpdatedAtHeight(height int64) ([]Actor, error) - // GetFishermanUpdatedAtHeight(height int64) ([]Actor, error) - // GetUpdatedAtHeight(height int64) ([]Actor, error) - // GetUpdatedAtHeight(height int64) ([]Actor, error) - // UpdateTree(height int64) ([]Actor, error) - - // # Option 2: - // GetUpdatedAtHeight(types.ProtocolActorSchema, height int64) ([]Actor, error) - // GetUpdatedAtHeight(height int64) ([]Other, error) - - // # Option 3: - // GetApplicationsUpdatedAtHeight(height int64) ([]Application, error) - // GetValidatorsUpdatedAtHeight(height int64) ([]Validator, error) - // GetServiceNodesUpdatedAtHeight(height int64) ([]ServiceNode, error) - // GetFishermanUpdatedAtHeight(height int64) ([]Fisherman, error) - // GetUpdatedAtHeight(height int64) ([]FutureActor, error) - // GetUpdatedAtHeight(height int64) ([]Other, error) } From d5a8a1f2c43437ca72d1183101dcd9d6cf8aaf4d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 14:51:48 -0700 Subject: [PATCH 30/56] Reverted changes to the persistence module files --- persistence/application.go | 51 ----------------------------- persistence/block.go | 42 +++++------------------- persistence/context.go | 26 ++++----------- persistence/db.go | 8 ----- persistence/debug.go | 2 +- persistence/genesis.go | 4 +-- persistence/kvstore/kvstore.go | 35 +++----------------- persistence/module.go | 31 +++--------------- persistence/shared_sql.go | 40 ++-------------------- persistence/test/module_test.go | 4 +-- persistence/types/base_actor.go | 7 +--- persistence/types/protocol_actor.go | 3 -- persistence/types/shared_sql.go | 5 --- shared/indexer/indexer.go | 3 +- 14 files changed, 32 insertions(+), 229 deletions(-) diff --git a/persistence/application.go b/persistence/application.go index 0bd6750bd..0f2b88512 100644 --- a/persistence/application.go +++ b/persistence/application.go @@ -5,66 +5,15 @@ import ( "log" "github.com/pokt-network/pocket/persistence/types" - "google.golang.org/protobuf/proto" - "github.com/pokt-network/pocket/shared/modules" ) -func (p PostgresContext) UpdateApplicationsTree(apps []modules.Actor) error { - for _, app := range apps { - bzAddr, err := hex.DecodeString(app.GetAddress()) - if err != nil { - return err - } - - appBz, err := proto.Marshal(app.(*types.Actor)) - if err != nil { - return err - } - - // OPTIMIZE: This is the only line unique to `Application` - if _, err := p.MerkleTrees[appMerkleTree].Update(bzAddr, appBz); err != nil { - return err - } - } - - return nil -} - -func (p PostgresContext) getApplicationsUpdatedAtHeight(height int64) (apps []*types.Actor, err error) { - // OPTIMIZE: This is the only line unique to `Application` - actors, err := p.GetActorsUpdated(types.ApplicationActor, height) - if err != nil { - return nil, err - } - - apps = make([]*types.Actor, len(actors)) - for _, actor := range actors { - app := &types.Actor{ - ActorType: types.ActorType_App, - Address: actor.Address, - PublicKey: actor.PublicKey, - Chains: actor.Chains, - GenericParam: actor.ActorSpecificParam, - StakedAmount: actor.StakedTokens, - PausedHeight: actor.PausedHeight, - UnstakingHeight: actor.UnstakingHeight, - Output: actor.OutputAddress, - } - apps = append(apps, app) - } - return -} - func (p PostgresContext) GetAppExists(address []byte, height int64) (exists bool, err error) { return p.GetExists(types.ApplicationActor, address, height) } func (p PostgresContext) GetApp(address []byte, height int64) (operator, publicKey, stakedTokens, maxRelays, outputAddress string, pauseHeight, unstakingHeight int64, chains []string, err error) { actor, err := p.GetActor(types.ApplicationActor, address, height) - if err != nil { - return - } operator = actor.Address publicKey = actor.PublicKey stakedTokens = actor.StakedTokens diff --git a/persistence/block.go b/persistence/block.go index 5efa1b71e..aaee96364 100644 --- a/persistence/block.go +++ b/persistence/block.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "log" - "github.com/gogo/protobuf/proto" "github.com/pokt-network/pocket/persistence/types" ) @@ -50,44 +49,21 @@ func (p PostgresContext) StoreTransaction(transactionProtoBytes []byte) error { return nil } -func (p PostgresContext) insertBlock(block *types.Block) error { - ctx, tx, err := p.GetCtxAndTx() - if err != nil { - return err - } - - _, err = tx.Exec(ctx, types.InsertBlockQuery(block.Height, block.Hash, block.ProposerAddress, block.QuorumCertificate)) - return err +func (p PostgresContext) StoreBlock(blockProtoBytes []byte) error { + // INVESTIGATE: Note that we are writing this directly to the blockStore. Depending on how + // the use of the PostgresContext evolves, we may need to write this to `ContextStore` and copy + // over to `BlockStore` when the block is committed. + return p.blockstore.Put(heightToBytes(p.Height), blockProtoBytes) } -func (p PostgresContext) storeBlock(block *types.Block) error { - blockBz, err := proto.Marshal(block) +func (p PostgresContext) InsertBlock(height uint64, hash string, proposerAddr []byte, quorumCert []byte) error { + ctx, tx, err := p.GetCtxAndTx() if err != nil { return err } - return p.blockstore.Put(heightToBytes(p.Height), blockBz) -} - -func (p PostgresContext) getBlock(proposerAddr []byte, quorumCert []byte) (*types.Block, error) { - prevHash, err := p.GetBlockHash(p.Height - 1) - if err != nil { - return nil, err - } - - // TODO: get this from the transactions that were store via `StoreTransaction` - txs := make([][]byte, 0) - - block := &types.Block{ - Height: uint64(p.Height), - Hash: string(p.stateHash), - PrevHash: string(prevHash), - ProposerAddress: proposerAddr, - QuorumCertificate: quorumCert, - Transactions: txs, - } - - return block, nil + _, err = tx.Exec(ctx, types.InsertBlockQuery(height, hash, proposerAddr, quorumCert)) + return err } // CLEANUP: Should this be moved to a shared directory? diff --git a/persistence/context.go b/persistence/context.go index 8224a229a..fd44e3af1 100644 --- a/persistence/context.go +++ b/persistence/context.go @@ -15,39 +15,25 @@ func (p PostgresContext) RollbackToSavePoint(bytes []byte) error { return p.GetTx().Rollback(context.TODO()) } -func (p PostgresContext) UpdateAppHash() ([]byte, error) { - if err := p.updateStateHash(); err != nil { - return nil, err - } - return p.stateHash, nil +func (p PostgresContext) AppHash() ([]byte, error) { + log.Println("TODO: AppHash not implemented") + return []byte("A real app hash, I am not"), nil } func (p PostgresContext) Reset() error { panic("TODO: PostgresContext Reset not implemented") } -func (p PostgresContext) Commit(proposerAddr []byte, quorumCert []byte) error { +func (p PostgresContext) Commit() error { log.Printf("About to commit context at height %d.\n", p.Height) - block, err := p.getBlock(proposerAddr, quorumCert) - if err != nil { - return err - } - - if err := p.insertBlock(block); err != nil { - return err - } - - if err := p.storeBlock(block); err != nil { - return err - } - ctx := context.TODO() - if err := p.GetTx().Commit(ctx); err != nil { + if err := p.GetTx().Commit(context.TODO()); err != nil { return err } if err := p.conn.Close(ctx); err != nil { log.Println("[TODO][ERROR] Implement connection pooling. Error when closing DB connecting...", err) + } return nil } diff --git a/persistence/db.go b/persistence/db.go index b8a974939..3a495596c 100644 --- a/persistence/db.go +++ b/persistence/db.go @@ -7,7 +7,6 @@ import ( "github.com/pokt-network/pocket/persistence/types" - "github.com/celestiaorg/smt" "github.com/jackc/pgconn" "github.com/jackc/pgx/v4" "github.com/pokt-network/pocket/persistence/kvstore" @@ -41,13 +40,6 @@ type PostgresContext struct { conn *pgx.Conn tx pgx.Tx blockstore kvstore.KVStore - - stateHash []byte - // IMPROVE: Depending on how the use of `PostgresContext` evolves, we may be able to get - // access to these directly via the postgres module. - PostgresDB *pgx.Conn - BlockStore kvstore.KVStore // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the persistence module (i.e. not context) - MerkleTrees map[MerkleTree]*smt.SparseMerkleTree // REARCHITECT_IN_THIS_COMMIT: This is a passthrough from the persistence module (i.e. not context) } func (pg *PostgresContext) GetCtxAndTx() (context.Context, pgx.Tx, error) { diff --git a/persistence/debug.go b/persistence/debug.go index e0e3b6e5e..3fdc583a4 100644 --- a/persistence/debug.go +++ b/persistence/debug.go @@ -44,7 +44,7 @@ func (m *PersistenceModule) showLatestBlockInStore(_ *debug.DebugMessage) { func (m *PersistenceModule) clearState(_ *debug.DebugMessage) { context, err := m.NewRWContext(-1) - defer context.Commit(nil, nil) + defer context.Commit() if err != nil { log.Printf("Error creating new context: %s \n", err) return diff --git a/persistence/genesis.go b/persistence/genesis.go index a6eefaa90..1c7e82f14 100644 --- a/persistence/genesis.go +++ b/persistence/genesis.go @@ -149,7 +149,7 @@ func (m *PersistenceModule) populateGenesisState(state *types.PersistenceGenesis log.Fatalf("an error occurred initializing flags: %s", err.Error()) } - if err = rwContext.Commit(nil, nil); err != nil { + if err = rwContext.Commit(); err != nil { log.Fatalf("an error occurred during commit() on genesis state %s ", err.Error()) } } @@ -316,7 +316,7 @@ func (p PostgresContext) GetAllFishermen(height int64) (f []modules.Actor, err e return } -// CONSOLIDATE: Consolidate `types.BaseActor` with `types.Actor` +// TODO (Team) deprecate with interface #163 as #163 is getting large func (p PostgresContext) BaseActorToActor(ba types.BaseActor, actorType types.ActorType) *types.Actor { actor := new(types.Actor) actor.ActorType = actorType diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index 8636e4519..24613afc9 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -1,47 +1,33 @@ package kvstore import ( - "errors" "log" - "github.com/celestiaorg/smt" badger "github.com/dgraph-io/badger/v3" ) +// CLEANUP: move this structure to a shared module type KVStore interface { // Lifecycle methods Stop() error // Accessors // TODO: Add a proper iterator interface + Put(key []byte, value []byte) error + Get(key []byte) ([]byte, error) // TODO: Add pagination for `GetAll` GetAll(prefixKey []byte, descending bool) ([][]byte, error) - Exists(key []byte) (bool, error) ClearAll() error - - // Same interface as in `smt.MapStore`` - Put(key, value []byte) error - Get(key []byte) ([]byte, error) - Delete(key []byte) error } var _ KVStore = &badgerKVStore{} -var _ smt.MapStore = &badgerKVStore{} - -var ( - ErrKVStoreExists = errors.New("kvstore already exists") - ErrKVStoreNotExists = errors.New("kvstore does not exist") -) type badgerKVStore struct { db *badger.DB } -// REFACTOR: Loads or creates a badgerDb at `path`. This may potentially need to be refactored -// into `NewKVStore` and `LoadKVStore` depending on how state sync evolves by leveraging `os.Stat` -// on the file path. -func OpenKVStore(path string) (KVStore, error) { +func NewKVStore(path string) (KVStore, error) { db, err := badger.Open(badger.DefaultOptions(path)) if err != nil { return nil, err @@ -57,7 +43,7 @@ func NewMemKVStore() KVStore { return badgerKVStore{db: db} } -func (store badgerKVStore) Put(key, value []byte) error { +func (store badgerKVStore) Put(key []byte, value []byte) error { tx := store.db.NewTransaction(true) defer tx.Discard() @@ -69,12 +55,6 @@ func (store badgerKVStore) Put(key, value []byte) error { return tx.Commit() } -// CONSOLIDATE: We might be able to remove the `KVStore` interface altogether if we end up using `smt.MapStore` -// A wrapper around `Put` to conform to the `smt.MapStore` interface -func (store *badgerKVStore) Set(key, value []byte) error { - return store.Put(key, value) -} - func (store badgerKVStore) Get(key []byte) ([]byte, error) { tx := store.db.NewTransaction(false) defer tx.Discard() @@ -96,11 +76,6 @@ func (store badgerKVStore) Get(key []byte) ([]byte, error) { return value, nil } -func (store badgerKVStore) Delete(key []byte) error { - log.Fatalf("badgerKVStore.Delete not implemented yet") - return nil -} - func (store badgerKVStore) GetAll(prefix []byte, descending bool) (values [][]byte, err error) { // INVESTIGATE: research `badger.views` for further improvements and optimizations txn := store.db.NewTransaction(false) diff --git a/persistence/module.go b/persistence/module.go index 6761bb22b..33bd827bd 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -7,10 +7,10 @@ import ( "io/ioutil" "log" - "github.com/celestiaorg/smt" + "github.com/pokt-network/pocket/persistence/types" + "github.com/jackc/pgx/v4" "github.com/pokt-network/pocket/persistence/kvstore" - "github.com/pokt-network/pocket/persistence/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/test_artifacts" ) @@ -26,19 +26,10 @@ type PersistenceModule struct { postgresURL string nodeSchema string genesisPath string + blockStore kvstore.KVStore // INVESTIGATE: We may need to create a custom `BlockStore` package in the future // TECHDEBT: Need to implement context pooling (for writes), timeouts (for read & writes), etc... writeContext *PostgresContext // only one write context is allowed at a time - - // The connection to the PostgreSQL database - postgresConn *pgx.Conn - // A reference to the block key-value store - // INVESTIGATE: We may need to create a custom `BlockStore` package in the future. - blockStore kvstore.KVStore - // A mapping of context IDs to persistence contexts - // contexts map[contextId]modules.PersistenceRWContext - // Merkle trees - trees map[MerkleTree]*smt.SparseMerkleTree } const ( @@ -51,10 +42,8 @@ func Create(configPath, genesisPath string) (modules.PersistenceModule, error) { if err != nil { return nil, err } - cfg := c.(*types.PersistenceConfig) g, err := m.InitGenesis(genesisPath) - if err != nil { return nil, err } @@ -80,20 +69,8 @@ func Create(configPath, genesisPath string) (modules.PersistenceModule, error) { genesisPath: genesisPath, blockStore: blockStore, writeContext: nil, - // contexts: make(map[contextId]modules.PersistenceContext), - trees: make(map[MerkleTree]*smt.SparseMerkleTree), } - // DISCUSS_IN_THIS_COMMIT: We've been using the module function pattern, but this making `initializeTrees` - // be able to create and/or load trees outside the scope of the persistence module makes it easier to test. - trees, err := newMerkleTrees() - if err != nil { - return nil, err - } - - // TODO_IN_THIS_COMMIT: load trees from state - persistenceMod.trees = trees - // Determine if we should hydrate the genesis db or use the current state of the DB attached if shouldHydrateGenesis, err := persistenceMod.shouldHydrateGenesisDb(); err != nil { return nil, err @@ -237,7 +214,7 @@ func initializeBlockStore(blockStorePath string) (kvstore.KVStore, error) { if blockStorePath == "" { return kvstore.NewMemKVStore(), nil } - return kvstore.OpenKVStore(blockStorePath) + return kvstore.NewKVStore(blockStorePath) } // TODO(drewsky): Simplify and externalize the logic for whether genesis should be populated and diff --git a/persistence/shared_sql.go b/persistence/shared_sql.go index 8eac77562..9f2c19a00 100644 --- a/persistence/shared_sql.go +++ b/persistence/shared_sql.go @@ -44,36 +44,6 @@ func (p *PostgresContext) GetExists(actorSchema types.ProtocolActorSchema, addre return } -func (p *PostgresContext) GetActorsUpdated(actorSchema types.ProtocolActorSchema, height int64) (actors []types.BaseActor, err error) { - ctx, tx, err := p.GetCtxAndTx() - if err != nil { - return - } - - rows, err := tx.Query(ctx, actorSchema.GetUpdatedAtHeightQuery(height)) - if err != nil { - return nil, err - } - defer rows.Close() - - // OPTIMIZE: Consolidate logic with `GetActor` to reduce code footprint - var addr string - for rows.Next() { - if err = rows.Scan(&addr); err != nil { - return - } - - actor, err := p.GetActor(actorSchema, []byte(addr), height) - if err != nil { - return nil, err - } - - actors = append(actors, actor) - } - - return -} - func (p *PostgresContext) GetActor(actorSchema types.ProtocolActorSchema, address []byte, height int64) (actor types.BaseActor, err error) { ctx, tx, err := p.GetCtxAndTx() if err != nil { @@ -87,16 +57,10 @@ func (p *PostgresContext) GetActor(actorSchema types.ProtocolActorSchema, addres return p.GetChainsForActor(ctx, tx, actorSchema, actor, height) } -// IMPORTANT: Need to consolidate `persistence/types.BaseActor` with `persistence/genesisTypes.Actor` func (p *PostgresContext) GetActorFromRow(row pgx.Row) (actor types.BaseActor, height int64, err error) { err = row.Scan( - &actor.Address, - &actor.PublicKey, - &actor.StakedTokens, - &actor.ActorSpecificParam, - &actor.OutputAddress, - &actor.PausedHeight, - &actor.UnstakingHeight, + &actor.Address, &actor.PublicKey, &actor.StakedTokens, &actor.ActorSpecificParam, + &actor.OutputAddress, &actor.PausedHeight, &actor.UnstakingHeight, &height) return } diff --git a/persistence/test/module_test.go b/persistence/test/module_test.go index ccb7bd37f..3bee636c5 100644 --- a/persistence/test/module_test.go +++ b/persistence/test/module_test.go @@ -18,14 +18,12 @@ func TestPersistenceContextParallelReadWrite(t *testing.T) { poolAddress := []byte("address") originalAmount := "15" modifiedAmount := "10" - proposer := []byte("proposer") - quorumCert := []byte("quorumCert") // setup a write context, insert a pool and commit it context, err := testPersistenceMod.NewRWContext(0) require.NoError(t, err) require.NoError(t, context.InsertPool(poolName, poolAddress, originalAmount)) - require.NoError(t, context.Commit(proposer, quorumCert)) + require.NoError(t, context.Commit()) // verify the insert in the previously committed context worked contextA, err := testPersistenceMod.NewRWContext(0) diff --git a/persistence/types/base_actor.go b/persistence/types/base_actor.go index 1c73da9a7..9514b15eb 100644 --- a/persistence/types/base_actor.go +++ b/persistence/types/base_actor.go @@ -1,6 +1,5 @@ package types -// IMPROVE: Move schema related functions to a separate sub-package import "github.com/pokt-network/pocket/shared/modules" // TECHDEBT: Consider moving this to a protobuf. This struct was created to make testing simple of protocol actors that @@ -61,10 +60,6 @@ func (actor *BaseProtocolActorSchema) GetChainsTableSchema() string { return ProtocolActorChainsTableSchema(actor.chainsHeightConstraintName) } -func (actor *BaseProtocolActorSchema) GetUpdatedAtHeightQuery(height int64) string { - return SelectAtHeight(AddressCol, height, actor.tableName) -} - func (actor *BaseProtocolActorSchema) GetQuery(address string, height int64) string { return Select(AllColsSelector, address, height, actor.tableName) } @@ -155,5 +150,5 @@ func (x *Actor) GetActorTyp() modules.ActorType { if x != nil { return x.GetActorType() } - return ActorType_App // HACK: Is this a hack? + return ActorType_App } diff --git a/persistence/types/protocol_actor.go b/persistence/types/protocol_actor.go index cb1e23fbf..4b57e8e40 100644 --- a/persistence/types/protocol_actor.go +++ b/persistence/types/protocol_actor.go @@ -15,9 +15,6 @@ type ProtocolActorSchema interface { GetActorSpecificColName() string /*** Read/Get Queries ***/ - - // Returns a query to retrieve the addresses of all the Actors updated at that specific height - GetUpdatedAtHeightQuery(height int64) string // Returns a query to retrieve all of a single Actor's attributes. GetQuery(address string, height int64) string // Returns all actors at that height diff --git a/persistence/types/shared_sql.go b/persistence/types/shared_sql.go index 6c053b6de..5502c03d3 100644 --- a/persistence/types/shared_sql.go +++ b/persistence/types/shared_sql.go @@ -69,11 +69,6 @@ func ProtocolActorChainsTableSchema(constraintName string) string { )`, AddressCol, ChainIDCol, HeightCol, DefaultBigInt, constraintName, AddressCol, ChainIDCol, HeightCol) } -func SelectAtHeight(selector string, height int64, tableName string) string { - return fmt.Sprintf(`SELECT %s FROM %s WHERE height=%d`, - selector, tableName, height) -} - func Select(selector, address string, height int64, tableName string) string { return fmt.Sprintf(`SELECT %s FROM %s WHERE address='%s' AND height<=%d ORDER BY height DESC LIMIT 1`, selector, tableName, address, height) diff --git a/shared/indexer/indexer.go b/shared/indexer/indexer.go index 59355787f..559ca1f6e 100644 --- a/shared/indexer/indexer.go +++ b/shared/indexer/indexer.go @@ -5,7 +5,6 @@ package indexer import ( "encoding/hex" "fmt" - "github.com/pokt-network/pocket/shared/codec" "github.com/jordanorelli/lexnum" @@ -112,7 +111,7 @@ type txIndexer struct { } func NewTxIndexer(databasePath string) (TxIndexer, error) { - db, err := kvstore.OpenKVStore(databasePath) + db, err := kvstore.NewKVStore(databasePath) return &txIndexer{ db: db, }, err From 6245080abbffcd6a6c18c61b062ed3e686e14543 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 15:11:54 -0700 Subject: [PATCH 31/56] Using GetCodec().Marshal() where appropriate --- consensus/block.go | 7 +-- consensus/helpers.go | 6 +- consensus/messages.go | 4 +- consensus/types/block.go | 14 ----- consensus/types/errors.go | 3 +- go.mod | 2 +- persistence/state.go | 119 -------------------------------------- 7 files changed, 10 insertions(+), 145 deletions(-) delete mode 100644 consensus/types/block.go delete mode 100644 persistence/state.go diff --git a/consensus/block.go b/consensus/block.go index 0ce2851e6..df5d82268 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -11,7 +11,7 @@ func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { m.nodeLog(typesCons.CommittingBlock(m.Height, len(block.Transactions))) // Commit the utility context - if err := m.UtilityContext.CommitContext(block.BlockHeader.QuorumCertificate); err != nil { + if err := m.UtilityContext.CommitPersistenceContext(); err != nil { return err } // Release the utility context @@ -56,10 +56,7 @@ func (m *ConsensusModule) refreshUtilityContext() error { // Ideally, this should not be called. if m.UtilityContext != nil { m.nodeLog(typesCons.NilUtilityContextWarning) - if err := m.UtilityContext.ReleaseContext(); err != nil { - // Logging an error but allowing the consensus lifecycle to continue - m.nodeLogError("cannot release existing utility context", err) - } + m.UtilityContext.ReleaseContext() m.UtilityContext = nil } diff --git a/consensus/helpers.go b/consensus/helpers.go index 54a2491c6..b59ac6397 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "log" + "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/debug" "google.golang.org/protobuf/proto" @@ -124,7 +125,7 @@ func (m *ConsensusModule) isOptimisticThresholdMet(n int) error { } func protoHash(m proto.Message) string { - b, err := proto.Marshal(m) + b, err := codec.GetCodec().Marshal(m) if err != nil { log.Fatalf("Could not marshal proto message: %v", err) } @@ -146,7 +147,6 @@ func (m *ConsensusModule) sendToNode(msg *typesCons.HotstuffMessage) { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return } - if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.IdToValAddrMap[*m.LeaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { m.nodeLogError(typesCons.ErrSendMessage.Error(), err) return @@ -160,7 +160,6 @@ func (m *ConsensusModule) broadcastToNodes(msg *typesCons.HotstuffMessage) { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return } - if err := m.GetBus().GetP2PModule().Broadcast(anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { m.nodeLogError(typesCons.ErrBroadcastMessage.Error(), err) return @@ -169,6 +168,7 @@ func (m *ConsensusModule) broadcastToNodes(msg *typesCons.HotstuffMessage) { /*** Persistence Helpers ***/ +// TECHDEBT: Integrate this with the `persistence` module or a real mempool. func (m *ConsensusModule) clearMessagesPool() { for _, step := range HotstuffSteps { m.MessagePool[step] = make([]*typesCons.HotstuffMessage, 0) diff --git a/consensus/messages.go b/consensus/messages.go index eb2801275..66fb8d836 100644 --- a/consensus/messages.go +++ b/consensus/messages.go @@ -4,8 +4,8 @@ import ( "log" typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/crypto" - "google.golang.org/protobuf/proto" ) func CreateProposeMessage( @@ -96,5 +96,5 @@ func getSignableBytes(msg *typesCons.HotstuffMessage) ([]byte, error) { Round: msg.GetRound(), Block: msg.GetBlock(), } - return proto.Marshal(msgToSign) + return codec.GetCodec().Marshal(msgToSign) } diff --git a/consensus/types/block.go b/consensus/types/block.go deleted file mode 100644 index f6091b6b3..000000000 --- a/consensus/types/block.go +++ /dev/null @@ -1,14 +0,0 @@ -package types - -import ( - "github.com/pokt-network/pocket/shared/codec" -) - -func (b *Block) Bytes() ([]byte, error) { - codec := codec.GetCodec() - blockProtoBz, err := codec.Marshal(b) - if err != nil { - return nil, err - } - return blockProtoBz, nil -} diff --git a/consensus/types/errors.go b/consensus/types/errors.go index fb6dd6df4..faf9d7002 100644 --- a/consensus/types/errors.go +++ b/consensus/types/errors.go @@ -6,6 +6,7 @@ import ( "fmt" "log" + "github.com/pokt-network/pocket/shared/codec" "google.golang.org/protobuf/proto" ) @@ -235,7 +236,7 @@ func ErrLeaderElection(msg *HotstuffMessage) error { } func protoHash(m proto.Message) string { - b, err := proto.Marshal(m) + b, err := codec.GetCodec().Marshal(m) if err != nil { log.Fatalf("Could not marshal proto message: %v", err) } diff --git a/go.mod b/go.mod index f83ae8336..6e76ac253 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gogo/protobuf v1.3.2 + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/persistence/state.go b/persistence/state.go deleted file mode 100644 index 5c3913140..000000000 --- a/persistence/state.go +++ /dev/null @@ -1,119 +0,0 @@ -package persistence - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "log" - "sort" - - "github.com/celestiaorg/smt" - typesUtil "github.com/pokt-network/pocket/utility/types" - "google.golang.org/protobuf/proto" -) - -type MerkleTree float64 - -// A work-in-progress list of all the trees we need to update to maintain the overall state -const ( - // Actor Merkle Trees - appMerkleTree MerkleTree = iota - valMerkleTree - fishMerkleTree - serviceNodeMerkleTree - accountMerkleTree - poolMerkleTree - - // Data / State Merkle Trees - blocksMerkleTree - paramsMerkleTree - flagsMerkleTree - - // Used for iteration purposes only - see https://stackoverflow.com/a/64178235/768439 - lastMerkleTree -) - -func newMerkleTrees() (map[MerkleTree]*smt.SparseMerkleTree, error) { - // We need a separate Merkle tree for each type of actor or storage - trees := make(map[MerkleTree]*smt.SparseMerkleTree, int(lastMerkleTree)) - - for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { - // TODO_IN_THIS_COMMIT: Rather than using `NewSimpleMap`, use a disk based key-value store - nodeStore := smt.NewSimpleMap() - valueStore := smt.NewSimpleMap() - - trees[treeType] = smt.NewSparseMerkleTree(nodeStore, valueStore, sha256.New()) - } - return trees, nil -} - -func loadMerkleTrees(map[MerkleTree]*smt.SparseMerkleTree, error) { - log.Fatalf("loadMerkleTrees not implemented yet") -} - -func (p *PostgresContext) updateStateHash() error { - // Update all the merkle trees - for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { - switch treeType { - case appMerkleTree: - apps, err := p.getApplicationsUpdatedAtHeight(p.Height) - if err != nil { - // TODO_IN_THIS_COMMIT: Update this error - return typesUtil.NewError(typesUtil.Code(42), "Couldn't figure out apps updated") - } - for _, app := range apps { - appBz, err := proto.Marshal(app) - if err != nil { - return err - } - // An update results in a create/update that is idempotent - addrBz, err := hex.DecodeString(app.Address) - if err != nil { - return err - } - if _, err := p.MerkleTrees[treeType].Update(addrBz, appBz); err != nil { - return err - } - } - case valMerkleTree: - fmt.Println("TODO: valMerkleTree not implemented") - case fishMerkleTree: - fmt.Println("TODO: fishMerkleTree not implemented") - case serviceNodeMerkleTree: - fmt.Println("TODO: serviceNodeMerkleTree not implemented") - case accountMerkleTree: - fmt.Println("TODO: accountMerkleTree not implemented") - case poolMerkleTree: - fmt.Println("TODO: poolMerkleTree not implemented") - case blocksMerkleTree: - // VERY VERY IMPORTANT DISCUSSION: What do we do here provided that `Commit`, which stores the block in the DB and tree - // requires the quorumCert, which we receive at the very end of hotstuff consensus - fmt.Println("TODO: blocksMerkleTree not implemented") - case paramsMerkleTree: - fmt.Println("TODO: paramsMerkleTree not implemented") - case flagsMerkleTree: - fmt.Println("TODO: flagsMerkleTree not implemented") - default: - log.Fatalln("Not handled yet in state commitment update", treeType) - } - } - - // Get the root of each Merkle Tree - roots := make([][]byte, 0) - for treeType := MerkleTree(0); treeType < lastMerkleTree; treeType++ { - roots = append(roots, p.MerkleTrees[treeType].Root()) - } - - // Sort the merkle roots lexicographically - sort.Slice(roots, func(r1, r2 int) bool { - return bytes.Compare(roots[r1], roots[r2]) < 0 - }) - - // Get the state hash - rootsConcat := bytes.Join(roots, []byte{}) - stateHash := sha256.Sum256(rootsConcat) - - p.stateHash = stateHash[:] - return nil -} From 4672e650207ef9dbdcd02b84c1f3fa2bac8ce902 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 15:21:47 -0700 Subject: [PATCH 32/56] Removed additional persistence related code --- persistence/StateHash.md | 194 ---------------------- persistence/proto/block_persistence.proto | 13 -- persistence/state_test.go | 15 -- 3 files changed, 222 deletions(-) delete mode 100644 persistence/StateHash.md delete mode 100644 persistence/proto/block_persistence.proto delete mode 100644 persistence/state_test.go diff --git a/persistence/StateHash.md b/persistence/StateHash.md deleted file mode 100644 index cfc54af52..000000000 --- a/persistence/StateHash.md +++ /dev/null @@ -1,194 +0,0 @@ -Remaining tasks: - -1. Simplify interfaces in utility & persistence (make it simple and clear) -2. How do we revert changes to the merkle trees? -3. Draw an end-to-end diagram of everything and the data flow - -## References: - -- https://github.com/cosmos/cosmos-sdk/discussions/9158 -- https://github.com/cosmos/cosmos-sdk/pull/8012 -- https://github.com/cosmos/cosmos-sdk/discussions/8297 -- https://github.com/cosmos/cosmos-sdk/discussions/8297 -- https://paper.dropbox.com/doc/State-commitments-and-storage-review--BpuF08OSksSkLeErjf66jrOAAg-wKl2RINZWD9I0DUmZIFwQ -- https://arxiv.org/pdf/1803.05069.pdf - -## Open questions: - -1. Review flows -2. How do we revert changes to the merkle trees? - -# This discussion is aimed at: - -1. Defining how we should compute the state hash -2. Identify potential changes needed in the current codebase -3. Propose next steps and actionable on implementation - -## Goals: - -- Define how the state hash will be computed -- Propose the necessary changes in separate tasks -- Implement each of the necessary pieces - -## Non-goals: - -- Choice/decision of Merkle Tree Design & Implementation -- Selection of a key-value store engine - -## Primitives / non-negotiables: - -- We will be using Merkle Trees (and Merkle Proofs) for this design (i.e. not vector commitments) -- We will be using a SQL engine for this (i.e. specifically PostgresSQL) -- We will be using Protobufs (not Flatbuffers, json, yaml or other) for the schema - -## Necessary technical context: - -### DB Engines - -Insert table from here: [Merkle Tree Design & Implementation](https://tikv.org/deep-dive/key-value-engine/b-tree-vs-lsm/#summary) - -- Most **Key-Value Store DB** Engines use **LSM-trees** -> good for writes -- Most **SQL DB** Engines use **B-Trees** -> good for reads - -_Basically all but there can be exceptions_ - -### Addressable Merkle Trees - -State is stored use an Account Based (non UTXO) based Modle - -Insert image from: https://www.horizen.io/blockchain-academy/technology/expert/utxo-vs-account-model/#:~:text=The%20UTXO%20model%20is%20a,constructions%2C%20as%20well%20a%20sharding. - ---- - -### Data Flow - -## Basics: - -1. Get each actor (flag, param, etc...) updated at a certain height (the context's height) -2. Compute the protobuf (the deterministic schema we use as source of truth) -3. Serialize the data struct -4. Update the corresponding merkle tree -5. Compute a state hash from the aggregated roots of all trees as per pokt-network/pocket-network-protocol@main/persistence#562-state-transition-sequence-diagram - -## Q&A - -Q: Can the SQL Engine be changed? -A: Yes - -Q: Can the SQL Engine be removed altogether? -A: Yes, but hard - -Q: Can the protobuf schema change? -A: Yes, but out-of-scope - -Q: Can protobufs be replaced? -A: Maybe, but out-of-scope - ---- - -Learnings / Ideas: - -- Consolidate `UtilActorType` and `persistence.ActorType` -- `modules.Actors` interface vs `types.Actor` in persistenceGenesis - -### Context Initialization - -```mermaid -sequenceDiagram - %% autonumber - participant N as Node - participant C as Consensus - participant U as Utility - participant P as Persistence - participant PP as Persistence (PostgresDB) - participant PM as Persistence (MerkleTree) - participant P2P as P2P - - %% Should this be P2P? - N-->>C: HandleMessage(anypb.Any) - critical NewRound Message - C->>+U: NewContext(height) - U->>P: NewRWContext(height) - P->>U: PersistenceRWContext - U->>U: Store persistenceContext - U->>-C: UtilityContext - C->>C: Store utilityContext - Note over C, PM: See 'Block Application' - end - Note over N, P2P: Hotstuff lifecycle - N-->>C: HandleMessage(anypb.Any) - critical Decide Message - Note over C, PM: See 'Block Commit' - end -``` - -### Block Application - -```mermaid -sequenceDiagram - participant C as Consensus - participant U as Utility - participant P as Persistence - participant PP as Persistence (PostgresDB) - participant PM as Persistence (MerkleTree) - - alt as leader - C->>+U: GetProposalTransactions(proposer, maxTxBz, [lastVal]) - U->>U: reap mempool - U->>-C: txs - Note over C, U: Perform replica behaviour - else as replica - C->>+U: ApplyBlock(height, proposer, txs, lastVals) - loop Update DB: for each operation in tx - U->>P: ReadOp | WriteOp - P->>PP: ReadOp | WriteOp - PP->>P: data | ok - P->>U: data | ok - U->>U: validate - U->>P: StoreTransaction(tx) - P->>P: store locally - P->>U: ok - end - U->>+P: UpdateAppHash() - loop for each protocol actor type - P->>PP: GetActorsUpdate(height) - PP->>P: actors - loop Update Tree: for each actor - P->>PM: Update(addr, serialized(actor)) - PM->>P: ok - end - P->>PM: GetRoot() - PM->>P: rootHash - end - P->>P: computeStateHash(rootHashes) - P->>-U: stateHash - U->>-C: hash - end -``` - -### Block Commit - -```mermaid -sequenceDiagram - %% autonumber - participant C as Consensus - participant U as Utility - participant P as Persistence - participant PP as Persistence (PostgresDB) - participant PK as Persistence (Key-Value Store) - - C->>U: CommitContext(quorumCert) - U->>P: Commit(proposerAddr, quorumCert) - P->>P: create typesPer.Block - P->>PP: insertBlock(block) - PP->>P: ok - P->>PK: Put(height, block) - PK->>P: ok - P->>P: commit tx - P->>U: ok - U->>P: Release() - P->>U: ok - C->>U: Release() - U->>C: ok - C->>C: release utilityContext -``` diff --git a/persistence/proto/block_persistence.proto b/persistence/proto/block_persistence.proto deleted file mode 100644 index 0c314690d..000000000 --- a/persistence/proto/block_persistence.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; -package persistence; - -option go_package = "github.com/pokt-network/pocket/persistence/types"; - -message Block { - uint64 height = 1; - string hash = 2; - string prevHash = 3; - bytes proposerAddress = 4; - bytes quorumCertificate = 5; - repeated bytes transactions = 6; -} \ No newline at end of file diff --git a/persistence/state_test.go b/persistence/state_test.go deleted file mode 100644 index 9570f4c00..000000000 --- a/persistence/state_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package persistence - -import "testing" - -func TestStateHash_InitializeTrees(t *testing.T) { - -} - -func TestStateHash_LoadTrees(t *testing.T) { - -} - -func TestStateHash_ComputeStateHash(t *testing.T) { - -} From 5c83fd5b7f8d305fe7a8c9eb79c704b0bff785af Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 15:21:59 -0700 Subject: [PATCH 33/56] Added initialization flow for AppHash --- shared/docs/flows/AppHash.md | 36 ++++++++++++++++++++++++++++++++++++ shared/docs/flows/README.md | 3 +++ 2 files changed, 39 insertions(+) create mode 100644 shared/docs/flows/AppHash.md create mode 100644 shared/docs/flows/README.md diff --git a/shared/docs/flows/AppHash.md b/shared/docs/flows/AppHash.md new file mode 100644 index 000000000..51a1824b0 --- /dev/null +++ b/shared/docs/flows/AppHash.md @@ -0,0 +1,36 @@ +# AppHash + +## Context Initialization + +```mermaid +sequenceDiagram + %% autonumber + participant N as Node + participant C as Consensus + participant U as Utility + participant P as Persistence + participant PP as Persistence (PostgresDB) + participant PM as Persistence (MerkleTree) + participant P2P as P2P + + %% Should this be P2P? + N-->>C: HandleMessage(anypb.Any) + critical NewRound Message + C->>+U: NewContext(height) + U->>P: NewRWContext(height) + P->>U: PersistenceRWContext + U->>U: Store persistenceContext + U->>-C: UtilityContext + C->>C: Store utilityContext + Note over C, PM: See 'Block Application' + end + Note over N, P2P: Hotstuff lifecycle + N-->>C: HandleMessage(anypb.Any) + critical Decide Message + Note over C, PM: See 'Block Commit' + end +``` + +## Block Application + +## Block Commit diff --git a/shared/docs/flows/README.md b/shared/docs/flows/README.md new file mode 100644 index 000000000..cb4e5d663 --- /dev/null +++ b/shared/docs/flows/README.md @@ -0,0 +1,3 @@ +# Flows + +The purpose of [shared/flows](./) is to document cross-module communication for end-to-end behaviour that has cross-module dependencies and and depends on the interfaces exposed by the interfaces exposed in [shared/modules](../modules). From 8c6a923eb874814cf8eefe5c7f244d9280193e07 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 16:53:05 -0700 Subject: [PATCH 34/56] make develop_test passed --- consensus/CHANGELOG.md | 31 +++++++++++++++++++++++++ consensus/consensus_tests/utils_test.go | 4 ++-- go.mod | 1 - go.sum | 2 -- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/consensus/CHANGELOG.md b/consensus/CHANGELOG.md index 64d55cba0..8dab5b81c 100644 --- a/consensus/CHANGELOG.md +++ b/consensus/CHANGELOG.md @@ -7,8 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.3] - 2022-09-26 + +Consensus logic + +- Pass in a list of messages to `findHighQC` instead of a hotstuff step +- Made `CreateProposeMessage` and `CreateVotemessage` accept explicit values, identifying some bugs along the way +- Made sure to call `applyBlock` when using `highQC` from previous round +- Moved business logic for `prepareAndApplyBlock` into `hotstuff_leader.go` +- Removed `MaxBlockBytes` and storing the consensus genesis type locally as is + +Consensus cleanup + +- Using appropriate getters for protocol types in the hotstuff lifecycle +- Replaced `proto.Marshal` with `codec.GetCodec().Marshal` +- Reorganized and cleaned up the code in `consensus/block.go` +- Consolidated & removed a few `TODO`s throughout the consensus module +- Added TECHDEBT and TODOs that will be require for a real block lifecycle +- Fixed typo in `hotstuff_types.proto` +- Moved the hotstuff handler interface to `consensus/hotstuff_handler.go` + +Consensus testing + +- Improved mock module initialization in `consensus/consensus_tests/utils_test.go` + +General + +- Added a diagram for `AppHash` related `ContextInitialization` +- Added `Makefile` keywords for `TODO` + ## [0.0.0.2] - 2022-08-25 + **Encapsulate structures previously in shared [#163](github.com/pokt-network/pocket/issues/163)** + - Ensured proto structures implement shared interfaces - `ConsensusConfig` uses shared interfaces in order to accept `MockConsensusConfig` in test_artifacts - `ConsensusGenesisState` uses shared interfaces in order to accept `MockConsensusGenesisState` in test_artifacts diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index ff716ce2c..2749968e5 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -367,8 +367,8 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { ApplyBlock(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(appHash, nil). AnyTimes() - utilityContextMock.EXPECT().CommitContext(gomock.Any()).Return(nil).AnyTimes() - utilityContextMock.EXPECT().ReleaseContext().Return(nil).AnyTimes() + utilityContextMock.EXPECT().CommitPersistenceContext().Return(nil).AnyTimes() + utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() return utilityContextMock } diff --git a/go.mod b/go.mod index 6e76ac253..54191cb69 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( ) require ( - github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 github.com/dgraph-io/badger/v3 v3.2103.2 github.com/jackc/pgconn v1.11.0 github.com/jordanorelli/lexnum v0.0.0-20141216151731-460eeb125754 diff --git a/go.sum b/go.sum index 8c2cca931..d064b7f46 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884 h1:iRNKw2WmAbVgGMNYzDH5Y2yY3+jyxwEK9Hc5pwIjZAE= -github.com/celestiaorg/smt v0.2.1-0.20220414134126-dba215ccb884/go.mod h1:/sdYDakowo/XaxS2Fl7CBqtuf/O2uTqF2zmAUFAtAiw= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= From 62a51ea37f692616e2b00adfeff511d83c13be14 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 17:10:03 -0700 Subject: [PATCH 35/56] Added call to StoreBlock back --- consensus/block.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/consensus/block.go b/consensus/block.go index df5d82268..a0c2d2929 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -5,11 +5,29 @@ import ( "unsafe" typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/shared/codec" ) func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { m.nodeLog(typesCons.CommittingBlock(m.Height, len(block.Transactions))) + // Store the block in the KV store + codec := codec.GetCodec() + blockProtoBytes, err := codec.Marshal(block) + if err != nil { + return err + } + + // IMPROVE(olshansky): temporary solution. `ApplyBlock` above applies the + // transactions to the postgres database, and this stores it in the KV store upon commitment. + // Instead of calling this directly, an alternative solution is to store the block metadata in + // the persistence context and have `CommitPersistenceContext` do this under the hood. However, + // additional `Block` metadata will need to be passed through and may change when we merkle the + // state hash. + if err := m.UtilityContext.StoreBlock(blockProtoBytes); err != nil { + return err + } + // Commit the utility context if err := m.UtilityContext.CommitPersistenceContext(); err != nil { return err From 8f61b6f3e165bfebbafbf696148f5c4bd294f663 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 17:24:27 -0700 Subject: [PATCH 36/56] Avoiding anypb in a few places in the consensus code --- consensus/consensus_tests/utils_test.go | 8 +++++--- consensus/helpers.go | 5 ++--- consensus/module.go | 12 ++++++++---- shared/codec/codec.go | 4 ++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 2749968e5..10f4758d2 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/debug" "github.com/pokt-network/pocket/shared/test_artifacts" @@ -25,7 +26,6 @@ import ( "github.com/pokt-network/pocket/shared/modules" modulesMock "github.com/pokt-network/pocket/shared/modules/mocks" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) @@ -226,10 +226,12 @@ func WaitForNetworkConsensusMessages( ) (messages []*anypb.Any, err error) { includeFilter := func(m *anypb.Any) bool { - var hotstuffMessage typesCons.HotstuffMessage - err := anypb.UnmarshalTo(m, &hotstuffMessage, proto.UnmarshalOptions{}) + msg, err := codec.GetCodec().FromAny(m) require.NoError(t, err) + hotstuffMessage, ok := msg.(*typesCons.HotstuffMessage) + require.True(t, ok) + return hotstuffMessage.Type == hotstuffMsgType && hotstuffMessage.Step == step } diff --git a/consensus/helpers.go b/consensus/helpers.go index b59ac6397..d089d0338 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -12,7 +12,6 @@ import ( typesCons "github.com/pokt-network/pocket/consensus/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - "google.golang.org/protobuf/types/known/anypb" ) // These constants and variables are wrappers around the autogenerated protobuf types and were @@ -142,7 +141,7 @@ func (m *ConsensusModule) sendToNode(msg *typesCons.HotstuffMessage) { } m.nodeLog(typesCons.SendingMessage(msg, *m.LeaderId)) - anyConsensusMessage, err := anypb.New(msg) + anyConsensusMessage, err := codec.GetCodec().ToAny(msg) if err != nil { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return @@ -155,7 +154,7 @@ func (m *ConsensusModule) sendToNode(msg *typesCons.HotstuffMessage) { func (m *ConsensusModule) broadcastToNodes(msg *typesCons.HotstuffMessage) { m.nodeLog(typesCons.BroadcastingMessage(msg)) - anyConsensusMessage, err := anypb.New(msg) + anyConsensusMessage, err := codec.GetCodec().ToAny(msg) if err != nil { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return diff --git a/consensus/module.go b/consensus/module.go index f49e4a97e..4c8e7d94f 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -8,9 +8,9 @@ import ( "github.com/pokt-network/pocket/consensus/leader_election" typesCons "github.com/pokt-network/pocket/consensus/types" + "github.com/pokt-network/pocket/shared/codec" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/test_artifacts" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" @@ -220,11 +220,15 @@ func (m *ConsensusModule) SetBus(pocketBus modules.Bus) { func (m *ConsensusModule) HandleMessage(message *anypb.Any) error { switch message.MessageName() { case HotstuffMessage: - var hotstuffMessage typesCons.HotstuffMessage - if err := anypb.UnmarshalTo(message, &hotstuffMessage, proto.UnmarshalOptions{}); err != nil { + msg, err := codec.GetCodec().FromAny(message) + if err != nil { return err } - if err := m.handleHotstuffMessage(&hotstuffMessage); err != nil { + hotstuffMessage, ok := msg.(*typesCons.HotstuffMessage) + if !ok { + return fmt.Errorf("failed to cast message to HotstuffMessage") + } + if err := m.handleHotstuffMessage(hotstuffMessage); err != nil { return err } case UtilityMessage: diff --git a/shared/codec/codec.go b/shared/codec/codec.go index ac7765673..a41fe2e0b 100644 --- a/shared/codec/codec.go +++ b/shared/codec/codec.go @@ -5,6 +5,8 @@ import ( "google.golang.org/protobuf/types/known/anypb" ) +// TODO: Use generics in place of `proto.Message` in the interface below +// so every caller does not need to do in place casting. type Codec interface { // TODO (Team) move to shared. Possibly rename Marshal(proto.Message) ([]byte, error) Unmarshal([]byte, proto.Message) error @@ -14,6 +16,8 @@ type Codec interface { // TODO (Team) move to shared. Possibly rename var _ Codec = &ProtoCodec{} +// TODO: Need to define a type like `type ProtoAny anypb.Any` so that we are +// mixing protobufs and a centralized codec structure throughout the codebase. type ProtoCodec struct{} func (p *ProtoCodec) Marshal(message proto.Message) ([]byte, error) { From 1200203ecd519be875a82a097764410a30970439 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 26 Sep 2022 17:31:24 -0700 Subject: [PATCH 37/56] Fixed unit tests --- consensus/consensus_tests/utils_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 10f4758d2..b5e3cf71a 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -371,6 +371,7 @@ func baseUtilityContextMock(t *testing.T) *modulesMock.MockUtilityContext { AnyTimes() utilityContextMock.EXPECT().CommitPersistenceContext().Return(nil).AnyTimes() utilityContextMock.EXPECT().ReleaseContext().Return().AnyTimes() + utilityContextMock.EXPECT().StoreBlock(gomock.Any()).Return(nil).AnyTimes() return utilityContextMock } From 637092221f397b2178f79298a85a77abbdbe8823 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 14:52:04 -0700 Subject: [PATCH 38/56] Added shouldElectNextLeader --- consensus/hotstuff_handler.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index 90b7b3b29..b27e65ba1 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -29,8 +29,7 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) return err } - // Leader Election - Need to execute leader election if there is no leader and we are in a new round. - if m.Step == NewRound && m.LeaderId == nil { + if m.shouldElectNextLeader() { if err := m.electNextLeader(msg); err != nil { return err } @@ -45,3 +44,8 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) return nil } + +func (m *ConsensusModule) shouldElectNextLeader() bool { + // Execute leader election if there is no leader and we are in a new round + return m.Step == NewRound && m.LeaderId == nil +} From 05e011884b757532a8ab492aa997b9deef4580d5 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:38:39 -0700 Subject: [PATCH 39/56] Added a comment for getSignableBytes --- consensus/messages.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consensus/messages.go b/consensus/messages.go index 66fb8d836..7ff8355c8 100644 --- a/consensus/messages.go +++ b/consensus/messages.go @@ -89,6 +89,8 @@ func getMessageSignature(msg *typesCons.HotstuffMessage, privKey crypto.PrivateK } // Signature only over subset of fields in HotstuffMessage +// For reference, see section 4.3 of the the hotstuff whitepaper, partial signatures are +// computed over `tsignr(hm.type, m.viewNumber , m.nodei)`. https://arxiv.org/pdf/1803.05069.pdf func getSignableBytes(msg *typesCons.HotstuffMessage) ([]byte, error) { msgToSign := &typesCons.HotstuffMessage{ Height: msg.GetHeight(), From 45da8225beeb731fd4dec084d0d7d94cfe249110 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:41:59 -0700 Subject: [PATCH 40/56] Check for nil in CreateProposeMessage --- consensus/messages.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/consensus/messages.go b/consensus/messages.go index 7ff8355c8..b2a06013e 100644 --- a/consensus/messages.go +++ b/consensus/messages.go @@ -15,6 +15,10 @@ func CreateProposeMessage( block *typesCons.Block, qc *typesCons.QuorumCertificate, ) (*typesCons.HotstuffMessage, error) { + if block == nil { + return nil, typesCons.ErrNilBlockVote + } + msg := &typesCons.HotstuffMessage{ Type: Propose, Height: height, From 2410521eb059a70a702fe2cabfd2fc3e97d587d3 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:44:02 -0700 Subject: [PATCH 41/56] Revert order of pacemaker defer and telemetry event emitting --- consensus/hotstuff_leader.go | 10 +++++----- consensus/hotstuff_replica.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 4aba0298b..2b7642bc4 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -24,8 +24,8 @@ var ( /*** Prepare Step ***/ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -93,8 +93,8 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM /*** PreCommit Step ***/ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -137,8 +137,8 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo /*** Commit Step ***/ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -181,8 +181,8 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus /*** Decide Step ***/ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -230,8 +230,8 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusMod } func (handler *HotstuffLeaderMessageHandler) HandleDecideMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index a4eda9951..0cc440389 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -24,8 +24,8 @@ var ( /*** NewRound Step ***/ func (handler *HotstuffReplicaMessageHandler) HandleNewRoundMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -43,8 +43,8 @@ func (handler *HotstuffReplicaMessageHandler) HandleNewRoundMessage(m *Consensus /*** Prepare Step ***/ func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -78,8 +78,8 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrepareMessage(m *ConsensusM /*** PreCommit Step ***/ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -107,8 +107,8 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *Consensu /*** Commit Step ***/ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) @@ -136,8 +136,8 @@ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusMo /*** Decide Step ***/ func (handler *HotstuffReplicaMessageHandler) HandleDecideMessage(m *ConsensusModule, msg *typesCons.HotstuffMessage) { - handler.emitTelemetryEvent(m, msg) defer m.paceMaker.RestartTimer() + handler.emitTelemetryEvent(m, msg) if err := handler.anteHandle(m, msg); err != nil { m.nodeLogError(typesCons.ErrHotstuffValidation.Error(), err) From a478516c7ef643989679b527cf1ca240e53596a0 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:47:30 -0700 Subject: [PATCH 42/56] s/IdToValAddrMap/idToValAddrMap --- consensus/helpers.go | 6 +++--- consensus/module.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/consensus/helpers.go b/consensus/helpers.go index d089d0338..fb4a57c1d 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -146,7 +146,7 @@ func (m *ConsensusModule) sendToNode(msg *typesCons.HotstuffMessage) { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return } - if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.IdToValAddrMap[*m.LeaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { + if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.idToValAddrMap[*m.LeaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { m.nodeLogError(typesCons.ErrSendMessage.Error(), err) return } @@ -201,10 +201,10 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) er if m.LeaderId != nil && *m.LeaderId == m.NodeId { m.logPrefix = "LEADER" - m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.IdToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) + m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) } else { m.logPrefix = "REPLICA" - m.nodeLog(typesCons.ElectedNewLeader(m.IdToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) + m.nodeLog(typesCons.ElectedNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) } return nil diff --git a/consensus/module.go b/consensus/module.go index 4c8e7d94f..6ec334946 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -48,7 +48,7 @@ type ConsensusModule struct { LeaderId *typesCons.NodeId NodeId typesCons.NodeId ValAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified - IdToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified + idToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified // Consensus State lastAppHash string // TODO: Always retrieve this variable from the persistence module and simplify this struct @@ -120,7 +120,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus NodeId: valIdMap[address], LeaderId: nil, ValAddrToIdMap: valIdMap, - IdToValAddrMap: idValMap, + idToValAddrMap: idValMap, lastAppHash: "", validatorMap: valMap, From 8faaf21d575304f6785060c7c748dfc00b81cc2b Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:47:44 -0700 Subject: [PATCH 43/56] s/ValAddrToIdMap/valAddrToIdMap --- consensus/hotstuff_leader.go | 4 ++-- consensus/hotstuff_replica.go | 4 ++-- consensus/module.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 2b7642bc4..8e948d5d8 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -293,7 +293,7 @@ func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessag address := partialSig.GetAddress() validator, ok := m.validatorMap[address] if !ok { - return typesCons.ErrMissingValidator(address, m.ValAddrToIdMap[address]) + return typesCons.ErrMissingValidator(address, m.valAddrToIdMap[address]) } pubKey := validator.GetPublicKey() if isSignatureValid(msg, pubKey, partialSig.GetSignature()) { @@ -301,7 +301,7 @@ func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessag } return typesCons.ErrValidatingPartialSig( - address, m.ValAddrToIdMap[address], msg, pubKey) + address, m.valAddrToIdMap[address], msg, pubKey) } // TODO: This is just a placeholder at the moment for indexing hotstuff messages ONLY. diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 0cc440389..4fb925bff 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -262,13 +262,13 @@ func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertific for _, partialSig := range qc.ThresholdSignature.Signatures { validator, ok := m.validatorMap[partialSig.Address] if !ok { - m.nodeLogError(typesCons.ErrMissingValidator(partialSig.Address, m.ValAddrToIdMap[partialSig.Address]).Error(), nil) + m.nodeLogError(typesCons.ErrMissingValidator(partialSig.Address, m.valAddrToIdMap[partialSig.Address]).Error(), nil) continue } // TODO(olshansky): Every call to `IsSignatureValid` does a serialization and should be optimized. We can // just serialize `Message` once and verify each signature without re-serializing every time. if !isSignatureValid(msgToJustify, validator.GetPublicKey(), partialSig.Signature) { - m.nodeLog(typesCons.WarnInvalidPartialSigInQC(partialSig.Address, m.ValAddrToIdMap[partialSig.Address])) + m.nodeLog(typesCons.WarnInvalidPartialSigInQC(partialSig.Address, m.valAddrToIdMap[partialSig.Address])) continue } numValid++ diff --git a/consensus/module.go b/consensus/module.go index 6ec334946..195568c79 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -47,7 +47,7 @@ type ConsensusModule struct { // Leader Election LeaderId *typesCons.NodeId NodeId typesCons.NodeId - ValAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified + valAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified idToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified // Consensus State @@ -119,7 +119,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus NodeId: valIdMap[address], LeaderId: nil, - ValAddrToIdMap: valIdMap, + valAddrToIdMap: valIdMap, idToValAddrMap: idValMap, lastAppHash: "", From 25971b773c63b2504362abcb6ce4f542f3a46c3e Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:48:16 -0700 Subject: [PATCH 44/56] s/NodeId/nodeId --- consensus/debugging.go | 2 +- consensus/helpers.go | 8 ++++---- consensus/module.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/consensus/debugging.go b/consensus/debugging.go index 0419575ba..319d1848b 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -31,7 +31,7 @@ func (m *ConsensusModule) GetNodeState() typesCons.ConsensusNodeState { leaderId = *m.LeaderId } return typesCons.ConsensusNodeState{ - NodeId: m.NodeId, + NodeId: m.nodeId, Height: m.Height, Round: uint8(m.Round), Step: uint8(m.Step), diff --git a/consensus/helpers.go b/consensus/helpers.go index fb4a57c1d..fd99734df 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -177,7 +177,7 @@ func (m *ConsensusModule) clearMessagesPool() { /*** Leader Election Helpers ***/ func (m *ConsensusModule) isLeader() bool { - return m.LeaderId != nil && *m.LeaderId == m.NodeId + return m.LeaderId != nil && *m.LeaderId == m.nodeId } func (m *ConsensusModule) isReplica() bool { @@ -199,7 +199,7 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) er m.LeaderId = &leaderId - if m.LeaderId != nil && *m.LeaderId == m.NodeId { + if m.LeaderId != nil && *m.LeaderId == m.nodeId { m.logPrefix = "LEADER" m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) } else { @@ -214,10 +214,10 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) er // TODO(#164): Remove this once we have a proper logging system. func (m *ConsensusModule) nodeLog(s string) { - log.Printf("[%s][%d] %s\n", m.logPrefix, m.NodeId, s) + log.Printf("[%s][%d] %s\n", m.logPrefix, m.nodeId, s) } // TODO(#164): Remove this once we have a proper logging system. func (m *ConsensusModule) nodeLogError(s string, err error) { - log.Printf("[ERROR][%s][%d] %s: %v\n", m.logPrefix, m.NodeId, s, err) + log.Printf("[ERROR][%s][%d] %s: %v\n", m.logPrefix, m.nodeId, s, err) } diff --git a/consensus/module.go b/consensus/module.go index 195568c79..ea17e5484 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -46,7 +46,7 @@ type ConsensusModule struct { // Leader Election LeaderId *typesCons.NodeId - NodeId typesCons.NodeId + nodeId typesCons.NodeId valAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified idToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified @@ -117,7 +117,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus HighPrepareQC: nil, LockedQC: nil, - NodeId: valIdMap[address], + nodeId: valIdMap[address], LeaderId: nil, valAddrToIdMap: valIdMap, idToValAddrMap: idValMap, From 472473ce17d78e238cfb098028ae4c8fb511a50b Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:48:28 -0700 Subject: [PATCH 45/56] s/LeaderId/leaderId --- consensus/debugging.go | 4 ++-- consensus/helpers.go | 18 +++++++++--------- consensus/hotstuff_handler.go | 4 ++-- consensus/module.go | 4 ++-- consensus/pacemaker.go | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/consensus/debugging.go b/consensus/debugging.go index 319d1848b..94f09bf18 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -27,8 +27,8 @@ func (m *ConsensusModule) HandleDebugMessage(debugMessage *debug.DebugMessage) e func (m *ConsensusModule) GetNodeState() typesCons.ConsensusNodeState { leaderId := typesCons.NodeId(0) - if m.LeaderId != nil { - leaderId = *m.LeaderId + if m.leaderId != nil { + leaderId = *m.leaderId } return typesCons.ConsensusNodeState{ NodeId: m.nodeId, diff --git a/consensus/helpers.go b/consensus/helpers.go index fd99734df..b1d2a6569 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -135,18 +135,18 @@ func protoHash(m proto.Message) string { func (m *ConsensusModule) sendToNode(msg *typesCons.HotstuffMessage) { // TODO(olshansky): This can happen due to a race condition with the pacemaker. - if m.LeaderId == nil { + if m.leaderId == nil { m.nodeLogError(typesCons.ErrNilLeaderId.Error(), nil) return } - m.nodeLog(typesCons.SendingMessage(msg, *m.LeaderId)) + m.nodeLog(typesCons.SendingMessage(msg, *m.leaderId)) anyConsensusMessage, err := codec.GetCodec().ToAny(msg) if err != nil { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return } - if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.idToValAddrMap[*m.LeaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { + if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.idToValAddrMap[*m.leaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { m.nodeLogError(typesCons.ErrSendMessage.Error(), err) return } @@ -177,7 +177,7 @@ func (m *ConsensusModule) clearMessagesPool() { /*** Leader Election Helpers ***/ func (m *ConsensusModule) isLeader() bool { - return m.LeaderId != nil && *m.LeaderId == m.nodeId + return m.leaderId != nil && *m.leaderId == m.nodeId } func (m *ConsensusModule) isReplica() bool { @@ -186,7 +186,7 @@ func (m *ConsensusModule) isReplica() bool { func (m *ConsensusModule) clearLeader() { m.logPrefix = DefaultLogPrefix - m.LeaderId = nil + m.leaderId = nil } func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) error { @@ -197,14 +197,14 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) er return err } - m.LeaderId = &leaderId + m.leaderId = &leaderId - if m.LeaderId != nil && *m.LeaderId == m.nodeId { + if m.leaderId != nil && *m.leaderId == m.nodeId { m.logPrefix = "LEADER" - m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) + m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.idToValAddrMap[*m.leaderId], *m.leaderId, m.Height, m.Round)) } else { m.logPrefix = "REPLICA" - m.nodeLog(typesCons.ElectedNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) + m.nodeLog(typesCons.ElectedNewLeader(m.idToValAddrMap[*m.leaderId], *m.leaderId, m.Height, m.Round)) } return nil diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index b27e65ba1..6624917ed 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -23,7 +23,7 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) // If a replica is not a leader for this round, but has already determined a leader, // and continues to receive NewRound messages, we avoid logging the "message discard" // because it creates unnecessary spam. - if !(m.LeaderId != nil && !m.isLeader() && step == NewRound) { + if !(m.leaderId != nil && !m.isLeader() && step == NewRound) { m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) } return err @@ -47,5 +47,5 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) func (m *ConsensusModule) shouldElectNextLeader() bool { // Execute leader election if there is no leader and we are in a new round - return m.Step == NewRound && m.LeaderId == nil + return m.Step == NewRound && m.leaderId == nil } diff --git a/consensus/module.go b/consensus/module.go index ea17e5484..3b9bdbfa4 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -45,7 +45,7 @@ type ConsensusModule struct { LockedQC *typesCons.QuorumCertificate // Highest QC for which replica voted COMMIT // Leader Election - LeaderId *typesCons.NodeId + leaderId *typesCons.NodeId nodeId typesCons.NodeId valAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified idToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified @@ -118,7 +118,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus LockedQC: nil, nodeId: valIdMap[address], - LeaderId: nil, + leaderId: nil, valAddrToIdMap: valIdMap, idToValAddrMap: idValMap, diff --git a/consensus/pacemaker.go b/consensus/pacemaker.go index c2655e315..0b9e5ae17 100644 --- a/consensus/pacemaker.go +++ b/consensus/pacemaker.go @@ -140,7 +140,7 @@ func (p *paceMaker) ValidateMessage(m *typesCons.HotstuffMessage) error { // TODO(olshansky): Add tests for this. When we catch up to a later step, the leader is still the same. // However, when we catch up to a later round, the leader at the same height will be different. - if p.consensusMod.Round != m.Round || p.consensusMod.LeaderId == nil { + if p.consensusMod.Round != m.Round || p.consensusMod.leaderId == nil { p.consensusMod.electNextLeader(m) } From a5edce54132141bf6147fa6cea29591321cd95ec Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:48:45 -0700 Subject: [PATCH 46/56] s/MessagePool/messagePool --- consensus/helpers.go | 6 +++--- consensus/hotstuff_leader.go | 14 +++++++------- consensus/module.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/consensus/helpers.go b/consensus/helpers.go index b1d2a6569..8ca304851 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -42,7 +42,7 @@ var ( // TODO: Add unit tests for quorumCert creation & validation. func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.HotstuffStep, round uint64) (*typesCons.QuorumCertificate, error) { var pss []*typesCons.PartialSignature - for _, msg := range m.MessagePool[step] { + for _, msg := range m.messagePool[step] { if msg.GetPartialSignature() == nil { m.nodeLog(typesCons.WarnMissingPartialSig(msg)) continue @@ -112,7 +112,7 @@ func isSignatureValid(msg *typesCons.HotstuffMessage, pubKeyString string, signa } func (m *ConsensusModule) didReceiveEnoughMessageForStep(step typesCons.HotstuffStep) error { - return m.isOptimisticThresholdMet(len(m.MessagePool[step])) + return m.isOptimisticThresholdMet(len(m.messagePool[step])) } func (m *ConsensusModule) isOptimisticThresholdMet(n int) error { @@ -170,7 +170,7 @@ func (m *ConsensusModule) broadcastToNodes(msg *typesCons.HotstuffMessage) { // TECHDEBT: Integrate this with the `persistence` module or a real mempool. func (m *ConsensusModule) clearMessagesPool() { for _, step := range HotstuffSteps { - m.MessagePool[step] = make([]*typesCons.HotstuffMessage, 0) + m.messagePool[step] = make([]*typesCons.HotstuffMessage, 0) } } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 8e948d5d8..9000e4135 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -47,7 +47,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM // Likely to be `nil` if blockchain is progressing well. // TECHDEBT: How do we properly validate `highPrepareQC` here? - highPrepareQC := m.findHighQC(m.MessagePool[NewRound]) + highPrepareQC := m.findHighQC(m.messagePool[NewRound]) // TODO: Add more unit tests for these checks... if highPrepareQC == nil || highPrepareQC.Height < m.Height || highPrepareQC.Round < m.Round { @@ -71,7 +71,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM } m.Step = Prepare - m.MessagePool[NewRound] = nil + m.messagePool[NewRound] = nil prepareProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Prepare, m.Block, highPrepareQC) if err != nil { @@ -115,7 +115,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo m.Step = PreCommit m.HighPrepareQC = prepareQC - m.MessagePool[Prepare] = nil + m.messagePool[Prepare] = nil preCommitProposeMessage, err := CreateProposeMessage(m.Height, m.Round, PreCommit, m.Block, prepareQC) if err != nil { @@ -159,7 +159,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus m.Step = Commit m.LockedQC = preCommitQC - m.MessagePool[PreCommit] = nil + m.messagePool[PreCommit] = nil commitProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Commit, m.Block, preCommitQC) if err != nil { @@ -202,7 +202,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleCommitMessage(m *ConsensusMod } m.Step = Decide - m.MessagePool[Commit] = nil + m.messagePool[Commit] = nil decideProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Decide, m.Block, commitQC) if err != nil { @@ -309,14 +309,14 @@ func (m *ConsensusModule) validatePartialSignature(msg *typesCons.HotstuffMessag // and does not recursively determine the size of all the underlying elements // Add proper tests and implementation once the mempool is implemented. func (m *ConsensusModule) tempIndexHotstuffMessage(msg *typesCons.HotstuffMessage) { - if m.consCfg.GetMaxMempoolBytes() < uint64(unsafe.Sizeof(m.MessagePool)) { + if m.consCfg.GetMaxMempoolBytes() < uint64(unsafe.Sizeof(m.messagePool)) { m.nodeLogError(typesCons.DisregardHotstuffMessage, typesCons.ErrConsensusMempoolFull) return } // Only the leader needs to aggregate consensus related messages. step := msg.GetStep() - m.MessagePool[step] = append(m.MessagePool[step], msg) + m.messagePool[step] = append(m.messagePool[step], msg) } // This is a helper function intended to be called by a leader/validator during a view change diff --git a/consensus/module.go b/consensus/module.go index 3b9bdbfa4..62cfa3d86 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -63,7 +63,7 @@ type ConsensusModule struct { logPrefix string // TECHDEBT: Move this over to use the txIndexer - MessagePool map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage + messagePool map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage } func Create(configPath, genesisPath string, useRandomPK bool) (modules.ConsensusModule, error) { @@ -130,7 +130,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus leaderElectionMod: leaderElectionMod, logPrefix: DefaultLogPrefix, - MessagePool: make(map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage), + messagePool: make(map[typesCons.HotstuffStep][]*typesCons.HotstuffMessage), } // TODO(olshansky): Look for a way to avoid doing this. From 28f1ba011f46dd957cd4ce45047c5909d6669b52 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:51:06 -0700 Subject: [PATCH 47/56] Only a few remaining exported fields in ConsensusModule based on reflections --- consensus/debugging.go | 8 ++++---- consensus/helpers.go | 18 +++++++++--------- consensus/hotstuff_handler.go | 4 ++-- consensus/hotstuff_leader.go | 4 ++-- consensus/hotstuff_replica.go | 6 +++--- consensus/module.go | 12 ++++++------ consensus/pacemaker.go | 8 ++++---- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/consensus/debugging.go b/consensus/debugging.go index 94f09bf18..e233af946 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -27,8 +27,8 @@ func (m *ConsensusModule) HandleDebugMessage(debugMessage *debug.DebugMessage) e func (m *ConsensusModule) GetNodeState() typesCons.ConsensusNodeState { leaderId := typesCons.NodeId(0) - if m.leaderId != nil { - leaderId = *m.leaderId + if m.LeaderId != nil { + leaderId = *m.LeaderId } return typesCons.ConsensusNodeState{ NodeId: m.nodeId, @@ -48,8 +48,8 @@ func (m *ConsensusModule) resetToGenesis(_ *debug.DebugMessage) { m.Step = 0 m.Block = nil - m.HighPrepareQC = nil - m.LockedQC = nil + m.highPrepareQC = nil + m.lockedQC = nil m.clearLeader() m.clearMessagesPool() diff --git a/consensus/helpers.go b/consensus/helpers.go index 8ca304851..a81c60205 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -135,18 +135,18 @@ func protoHash(m proto.Message) string { func (m *ConsensusModule) sendToNode(msg *typesCons.HotstuffMessage) { // TODO(olshansky): This can happen due to a race condition with the pacemaker. - if m.leaderId == nil { + if m.LeaderId == nil { m.nodeLogError(typesCons.ErrNilLeaderId.Error(), nil) return } - m.nodeLog(typesCons.SendingMessage(msg, *m.leaderId)) + m.nodeLog(typesCons.SendingMessage(msg, *m.LeaderId)) anyConsensusMessage, err := codec.GetCodec().ToAny(msg) if err != nil { m.nodeLogError(typesCons.ErrCreateConsensusMessage.Error(), err) return } - if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.idToValAddrMap[*m.leaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { + if err := m.GetBus().GetP2PModule().Send(cryptoPocket.AddressFromString(m.idToValAddrMap[*m.LeaderId]), anyConsensusMessage, debug.PocketTopic_CONSENSUS_MESSAGE_TOPIC); err != nil { m.nodeLogError(typesCons.ErrSendMessage.Error(), err) return } @@ -177,7 +177,7 @@ func (m *ConsensusModule) clearMessagesPool() { /*** Leader Election Helpers ***/ func (m *ConsensusModule) isLeader() bool { - return m.leaderId != nil && *m.leaderId == m.nodeId + return m.LeaderId != nil && *m.LeaderId == m.nodeId } func (m *ConsensusModule) isReplica() bool { @@ -186,7 +186,7 @@ func (m *ConsensusModule) isReplica() bool { func (m *ConsensusModule) clearLeader() { m.logPrefix = DefaultLogPrefix - m.leaderId = nil + m.LeaderId = nil } func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) error { @@ -197,14 +197,14 @@ func (m *ConsensusModule) electNextLeader(message *typesCons.HotstuffMessage) er return err } - m.leaderId = &leaderId + m.LeaderId = &leaderId - if m.leaderId != nil && *m.leaderId == m.nodeId { + if m.LeaderId != nil && *m.LeaderId == m.nodeId { m.logPrefix = "LEADER" - m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.idToValAddrMap[*m.leaderId], *m.leaderId, m.Height, m.Round)) + m.nodeLog(typesCons.ElectedSelfAsNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) } else { m.logPrefix = "REPLICA" - m.nodeLog(typesCons.ElectedNewLeader(m.idToValAddrMap[*m.leaderId], *m.leaderId, m.Height, m.Round)) + m.nodeLog(typesCons.ElectedNewLeader(m.idToValAddrMap[*m.LeaderId], *m.LeaderId, m.Height, m.Round)) } return nil diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index 6624917ed..b27e65ba1 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -23,7 +23,7 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) // If a replica is not a leader for this round, but has already determined a leader, // and continues to receive NewRound messages, we avoid logging the "message discard" // because it creates unnecessary spam. - if !(m.leaderId != nil && !m.isLeader() && step == NewRound) { + if !(m.LeaderId != nil && !m.isLeader() && step == NewRound) { m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) } return err @@ -47,5 +47,5 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) func (m *ConsensusModule) shouldElectNextLeader() bool { // Execute leader election if there is no leader and we are in a new round - return m.Step == NewRound && m.leaderId == nil + return m.Step == NewRound && m.LeaderId == nil } diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 9000e4135..c985e9d15 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -114,7 +114,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrepareMessage(m *ConsensusMo } m.Step = PreCommit - m.HighPrepareQC = prepareQC + m.highPrepareQC = prepareQC m.messagePool[Prepare] = nil preCommitProposeMessage, err := CreateProposeMessage(m.Height, m.Round, PreCommit, m.Block, prepareQC) @@ -158,7 +158,7 @@ func (handler *HotstuffLeaderMessageHandler) HandlePrecommitMessage(m *Consensus } m.Step = Commit - m.LockedQC = preCommitQC + m.lockedQC = preCommitQC m.messagePool[PreCommit] = nil commitProposeMessage, err := CreateProposeMessage(m.Height, m.Round, Commit, m.Block, preCommitQC) diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 4fb925bff..d0e5b7b9f 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -94,7 +94,7 @@ func (handler *HotstuffReplicaMessageHandler) HandlePrecommitMessage(m *Consensu } m.Step = Commit - m.HighPrepareQC = quorumCert // INVESTIGATE: Why are we never using this for validation? + m.highPrepareQC = quorumCert // INVESTIGATE: Why are we never using this for validation? preCommitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, PreCommit, m.Block, m.privateKey) if err != nil { @@ -123,7 +123,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleCommitMessage(m *ConsensusMo } m.Step = Decide - m.LockedQC = quorumCert // DISCUSS: How does the replica recover if it's locked? Replica `formally` agrees on the QC while the rest of the network `verbally` agrees on the QC. + m.lockedQC = quorumCert // DISCUSS: How does the replica recover if it's locked? Replica `formally` agrees on the QC while the rest of the network `verbally` agrees on the QC. commitVoteMessage, err := CreateVoteMessage(m.Height, m.Round, Commit, m.Block, m.privateKey) if err != nil { @@ -198,7 +198,7 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error } } - lockedQC := m.LockedQC + lockedQC := m.lockedQC justifyQC := quorumCert // Safety: not locked diff --git a/consensus/module.go b/consensus/module.go index 62cfa3d86..6b5e5af42 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -41,11 +41,11 @@ type ConsensusModule struct { Step typesCons.HotstuffStep Block *typesCons.Block // The current block being proposed / voted on; it has not been committed to finality - HighPrepareQC *typesCons.QuorumCertificate // Highest QC for which replica voted PRECOMMIT - LockedQC *typesCons.QuorumCertificate // Highest QC for which replica voted COMMIT + highPrepareQC *typesCons.QuorumCertificate // Highest QC for which replica voted PRECOMMIT + lockedQC *typesCons.QuorumCertificate // Highest QC for which replica voted COMMIT // Leader Election - leaderId *typesCons.NodeId + LeaderId *typesCons.NodeId nodeId typesCons.NodeId valAddrToIdMap typesCons.ValAddrToIdMap // TODO: This needs to be updated every time the ValMap is modified idToValAddrMap typesCons.IdToValAddrMap // TODO: This needs to be updated every time the ValMap is modified @@ -114,11 +114,11 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus Step: NewRound, Block: nil, - HighPrepareQC: nil, - LockedQC: nil, + highPrepareQC: nil, + lockedQC: nil, nodeId: valIdMap[address], - leaderId: nil, + LeaderId: nil, valAddrToIdMap: valIdMap, idToValAddrMap: idValMap, diff --git a/consensus/pacemaker.go b/consensus/pacemaker.go index 0b9e5ae17..9be4a6cb4 100644 --- a/consensus/pacemaker.go +++ b/consensus/pacemaker.go @@ -140,7 +140,7 @@ func (p *paceMaker) ValidateMessage(m *typesCons.HotstuffMessage) error { // TODO(olshansky): Add tests for this. When we catch up to a later step, the leader is still the same. // However, when we catch up to a later round, the leader at the same height will be different. - if p.consensusMod.Round != m.Round || p.consensusMod.leaderId == nil { + if p.consensusMod.Round != m.Round || p.consensusMod.LeaderId == nil { p.consensusMod.electNextLeader(m) } @@ -178,7 +178,7 @@ func (p *paceMaker) InterruptRound() { p.consensusMod.nodeLog(typesCons.PacemakerInterrupt(p.consensusMod.Height, p.consensusMod.Step, p.consensusMod.Round)) p.consensusMod.Round++ - p.startNextView(p.consensusMod.HighPrepareQC, false) + p.startNextView(p.consensusMod.highPrepareQC, false) } func (p *paceMaker) NewHeight() { @@ -188,8 +188,8 @@ func (p *paceMaker) NewHeight() { p.consensusMod.Round = 0 p.consensusMod.Block = nil - p.consensusMod.HighPrepareQC = nil - p.consensusMod.LockedQC = nil + p.consensusMod.highPrepareQC = nil + p.consensusMod.lockedQC = nil p.startNextView(nil, false) // TODO(design): We are omitting CommitQC and TimeoutQC here. From a7207b980ce9e1cb0871d0b33810388966ca8a9d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 17:52:02 -0700 Subject: [PATCH 48/56] Add TODO in sequence diagram flow --- shared/docs/flows/AppHash.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared/docs/flows/AppHash.md b/shared/docs/flows/AppHash.md index 51a1824b0..c17f3cd41 100644 --- a/shared/docs/flows/AppHash.md +++ b/shared/docs/flows/AppHash.md @@ -33,4 +33,8 @@ sequenceDiagram ## Block Application +TODO(olshansky): Add a sequenceDiagram here. + ## Block Commit + +TODO(olshansky): Add a sequenceDiagram here. From a8e5f3dae04d54da3745c502e14ab27801578f69 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 18:09:48 -0700 Subject: [PATCH 49/56] Added isNodeLockedOnPastQC --- consensus/hotstuff_replica.go | 24 +++++++++++++++++++++--- consensus/types/errors.go | 6 ++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index d0e5b7b9f..752b7614b 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -5,6 +5,7 @@ import ( "fmt" consensusTelemetry "github.com/pokt-network/pocket/consensus/telemetry" + "github.com/pokt-network/pocket/consensus/types" typesCons "github.com/pokt-network/pocket/consensus/types" ) @@ -214,10 +215,10 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error return nil } - // Liveness: node is locked on a QC from the past. + // Liveness: is node is locked on a QC from the past? // DISCUSS: Where should additional logic be added to unlock the node? - if justifyQC.Height > lockedQC.Height || (justifyQC.Height == lockedQC.Height && justifyQC.Round > lockedQC.Round) { - return typesCons.ErrNodeIsLockedOnPastQC + if isLocked, err := isNodeLockedOnPastQC(justifyQC, lockedQC); isLocked { + return err } return typesCons.ErrUnhandledProposalCase @@ -280,6 +281,23 @@ func (m *ConsensusModule) validateQuorumCertificate(qc *typesCons.QuorumCertific return nil } +func isNodeLockedOnPastQC(justifyQC, lockedQC *types.QuorumCertificate) (bool, error) { + if isLockedOnPastHeight(justifyQC, lockedQC) { + return true, types.ErrNodeLockedPastHeight + } else if isLockedOnCurrHeightAndPastRound(justifyQC, lockedQC) { + return true, types.ErrNodeLockedPastHeight + } + return false, nil +} + +func isLockedOnPastHeight(justifyQC, lockedQC *types.QuorumCertificate) bool { + return justifyQC.Height > lockedQC.Height +} + +func isLockedOnCurrHeightAndPastRound(justifyQC, lockedQC *types.QuorumCertificate) bool { + return justifyQC.Height == lockedQC.Height && justifyQC.Round > lockedQC.Round +} + func qcToHotstuffMessage(qc *typesCons.QuorumCertificate) *typesCons.HotstuffMessage { return &typesCons.HotstuffMessage{ Height: qc.Height, diff --git a/consensus/types/errors.go b/consensus/types/errors.go index faf9d7002..5368ee1bf 100644 --- a/consensus/types/errors.go +++ b/consensus/types/errors.go @@ -124,7 +124,8 @@ const ( nilBlockInQCError = "QC must contain a non nil block" nilThresholdSigInQCError = "QC must contains a non nil threshold signature" notEnoughSignaturesError = "did not receive enough partial signature" - nodeIsLockedOnPastQCError = "node is locked on a QC from the past" + nodeIsLockedOnPastHeightQCError = "node is locked on a QC from a past height" + nodeIsLockedOnPastRoundQCError = "node is locked on a QC from a past round" unhandledProposalCaseError = "unhandled proposal validation check" unnecessaryPartialSigForNewRoundError = "newRound messages do not need a partial signature" unnecessaryPartialSigForLeaderProposalError = "leader proposals do not need a partial signature" @@ -163,7 +164,8 @@ var ( ErrNilBlockInQC = errors.New(nilBlockInQCError) ErrNilThresholdSigInQC = errors.New(nilThresholdSigInQCError) ErrNotEnoughSignatures = errors.New(notEnoughSignaturesError) - ErrNodeIsLockedOnPastQC = errors.New(nodeIsLockedOnPastQCError) + ErrNodeLockedPastHeight = errors.New(nodeIsLockedOnPastHeightQCError) + ErrNodeLockedPastRound = errors.New(nodeIsLockedOnPastRoundQCError) ErrUnhandledProposalCase = errors.New(unhandledProposalCaseError) ErrUnnecessaryPartialSigForNewRound = errors.New(unnecessaryPartialSigForNewRoundError) ErrUnnecessaryPartialSigForLeaderProposal = errors.New(unnecessaryPartialSigForLeaderProposalError) From e39f0f8b17311bc6db86b66b0cb482898bc9914e Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 18:11:33 -0700 Subject: [PATCH 50/56] Logging error when refreshUtilityContext fails --- consensus/hotstuff_leader.go | 1 + consensus/hotstuff_replica.go | 1 + 2 files changed, 2 insertions(+) diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index c985e9d15..b7dad2466 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -42,6 +42,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM // Clear the previous utility context, if it exists, and create a new one if err := m.refreshUtilityContext(); err != nil { + m.nodeLogError("Could not refresh utility context", err) return } diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 752b7614b..9da379a61 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -35,6 +35,7 @@ func (handler *HotstuffReplicaMessageHandler) HandleNewRoundMessage(m *Consensus // Clear the previous utility context, if it exists, and create a new one if err := m.refreshUtilityContext(); err != nil { + m.nodeLogError("Could not refresh utility context", err) return } From a6990d217861bfe25ea34d2ff643f1d8cace93fe Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 18:23:56 -0700 Subject: [PATCH 51/56] Added shouldLogHotstuffDiscardMessage --- consensus/hotstuff_handler.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index b27e65ba1..536777a1c 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -20,10 +20,7 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) // Pacemaker - Liveness & safety checks if err := m.paceMaker.ValidateMessage(msg); err != nil { - // If a replica is not a leader for this round, but has already determined a leader, - // and continues to receive NewRound messages, we avoid logging the "message discard" - // because it creates unnecessary spam. - if !(m.LeaderId != nil && !m.isLeader() && step == NewRound) { + if m.shouldLogHotstuffDiscardMessage() { m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) } return err @@ -49,3 +46,10 @@ func (m *ConsensusModule) shouldElectNextLeader() bool { // Execute leader election if there is no leader and we are in a new round return m.Step == NewRound && m.LeaderId == nil } + +func (m *ConsensusModule) shouldLogHotstuffDiscardMessage(step typesCons.HotstuffStep) bool { + // If a replica is not a leader for this round, but has already determined a leader, + // and continues to receive NewRound messages, we avoid logging the "message discard" + // because it creates unnecessary spam. + return !(m.LeaderId != nil && !m.isLeader() && step == NewRound) +} From 8a41361cb93007896bfd8222782819bc415b2f37 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 18:32:50 -0700 Subject: [PATCH 52/56] Added shouldPrepareNewBlock --- consensus/hotstuff_handler.go | 2 +- consensus/hotstuff_leader.go | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/consensus/hotstuff_handler.go b/consensus/hotstuff_handler.go index 536777a1c..274a1b45b 100644 --- a/consensus/hotstuff_handler.go +++ b/consensus/hotstuff_handler.go @@ -20,7 +20,7 @@ func (m *ConsensusModule) handleHotstuffMessage(msg *typesCons.HotstuffMessage) // Pacemaker - Liveness & safety checks if err := m.paceMaker.ValidateMessage(msg); err != nil { - if m.shouldLogHotstuffDiscardMessage() { + if m.shouldLogHotstuffDiscardMessage(step) { m.nodeLog(typesCons.WarnDiscardHotstuffMessage(msg, err.Error())) } return err diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index b7dad2466..1e1cefa7d 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -51,7 +51,7 @@ func (handler *HotstuffLeaderMessageHandler) HandleNewRoundMessage(m *ConsensusM highPrepareQC := m.findHighQC(m.messagePool[NewRound]) // TODO: Add more unit tests for these checks... - if highPrepareQC == nil || highPrepareQC.Height < m.Height || highPrepareQC.Round < m.Round { + if m.shouldPrepareNewBlock(highPrepareQC) { // Leader prepares a new block if `highPrepareQC` is not applicable block, err := m.prepareAndApplyBlock() if err != nil { @@ -362,3 +362,23 @@ func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { return block, nil } + +// Return true if this node, the leader, should prepare a new block +func (m *ConsensusModule) shouldPrepareNewBlock(highPrepareQC *typesCons.QuorumCertificate) bool { + if highPrepareQC == nil { + m.nodeLog("Preparing a new block - no highPrepareQC found") + return true + } else if m.isHighPrepareQCFromPast(highPrepareQC) { + m.nodeLog("Preparing a new block - highPrepareQC is from the past") + return true + } else if highPrepareQC.Block == nil { + m.nodeLog("[WARN] Preparing a new block - highPrepareQC SHOULD be used but block is nil") + return true + } + return false +} + +// The `highPrepareQC` is from the past so we can safely ignore it +func (m *ConsensusModule) isHighPrepareQCFromPast(highPrepareQC *typesCons.QuorumCertificate) bool { + return highPrepareQC.Height < m.Height || highPrepareQC.Round < m.Round +} From 7493dd39059f99ddf55e29e1d453456c93105bb8 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Wed, 28 Sep 2022 18:48:09 -0700 Subject: [PATCH 53/56] Added TODO for 256 --- consensus/helpers.go | 2 +- consensus/module.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/consensus/helpers.go b/consensus/helpers.go index a81c60205..a9b3b5caa 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -39,7 +39,7 @@ var ( // ** Hotstuff Helpers ** // // IMPROVE: Avoid having the `ConsensusModule` be a receiver of this; making it more functional. -// TODO: Add unit tests for quorumCert creation & validation. +// TODO: Add unit tests for all quorumCert creation & validation logic... func (m *ConsensusModule) getQuorumCertificate(height uint64, step typesCons.HotstuffStep, round uint64) (*typesCons.QuorumCertificate, error) { var pss []*typesCons.PartialSignature for _, msg := range m.messagePool[step] { diff --git a/consensus/module.go b/consensus/module.go index 6b5e5af42..23a1fc881 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -27,7 +27,7 @@ var _ modules.PacemakerConfig = &typesCons.PacemakerConfig{} var _ modules.ConsensusConfig = &typesCons.ConsensusConfig{} var _ modules.ConsensusModule = &ConsensusModule{} -// TODO: Do not export the `ConsensusModule` struct or the fields inside of it. +// TODO(#256): Do not export the `ConsensusModule` struct or the fields inside of it. type ConsensusModule struct { bus modules.Bus privateKey cryptoPocket.Ed25519PrivateKey From 3f413bd06099802d568f6f54ad07cbf04f59648d Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 3 Oct 2022 20:56:55 -0700 Subject: [PATCH 54/56] Added SetUtilityMod and made utilityContext private --- consensus/block.go | 17 ++++++++-------- consensus/consensus_tests/pacemaker_test.go | 22 +++++++++++---------- consensus/consensus_tests/utils_test.go | 8 ++++++-- consensus/hotstuff_leader.go | 4 ++-- consensus/hotstuff_replica.go | 2 +- consensus/module.go | 9 +++++++-- 6 files changed, 36 insertions(+), 26 deletions(-) diff --git a/consensus/block.go b/consensus/block.go index ae4ca7dca..a70c6cecc 100644 --- a/consensus/block.go +++ b/consensus/block.go @@ -4,7 +4,6 @@ import ( "log" "unsafe" - typesCons "github.com/pokt-network/pocket/consensus/types" "github.com/pokt-network/pocket/shared/codec" ) @@ -30,12 +29,12 @@ func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { } // Commit and release the context - if err := m.UtilityContext.CommitPersistenceContext(); err != nil { + if err := m.utilityContext.CommitPersistenceContext(); err != nil { return err } - m.UtilityContext.ReleaseContext() - m.UtilityContext = nil + m.utilityContext.ReleaseContext() + m.utilityContext = nil m.lastAppHash = block.BlockHeader.Hash @@ -43,7 +42,7 @@ func (m *ConsensusModule) commitBlock(block *typesCons.Block) error { } func (m *ConsensusModule) storeBlock(block *typesCons.Block, blockProtoBytes []byte) error { - store := m.UtilityContext.GetPersistenceContext() + store := m.utilityContext.GetPersistenceContext() // Store in KV Store if err := store.StoreBlock(blockProtoBytes); err != nil { return err @@ -88,10 +87,10 @@ func (m *ConsensusModule) validateBlockBasic(block *typesCons.Block) error { func (m *ConsensusModule) refreshUtilityContext() error { // Catch-all structure to release the previous utility context if it wasn't properly cleaned up. // Ideally, this should not be called. - if m.UtilityContext != nil { + if m.utilityContext != nil { m.nodeLog(typesCons.NilUtilityContextWarning) - m.UtilityContext.ReleaseContext() - m.UtilityContext = nil + m.utilityContext.ReleaseContext() + m.utilityContext = nil } utilityContext, err := m.GetBus().GetUtilityModule().NewContext(int64(m.Height)) @@ -99,6 +98,6 @@ func (m *ConsensusModule) refreshUtilityContext() error { return err } - m.UtilityContext = utilityContext + m.utilityContext = utilityContext return nil } diff --git a/consensus/consensus_tests/pacemaker_test.go b/consensus/consensus_tests/pacemaker_test.go index 3090d24ca..80bf64fc4 100644 --- a/consensus/consensus_tests/pacemaker_test.go +++ b/consensus/consensus_tests/pacemaker_test.go @@ -157,7 +157,7 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { Transactions: emptyTxs, } - leaderConsensusMod := GetConsensusModImplementation(leader) + leaderConsensusMod := GetConsensusModElem(leader) leaderConsensusMod.FieldByName("Block").Set(reflect.ValueOf(block)) // Set all nodes to the same STEP and HEIGHT BUT different ROUNDS @@ -166,18 +166,20 @@ func TestPacemakerCatchupSameStepDifferentRounds(t *testing.T) { utilityContext, err := pocketNode.GetBus().GetUtilityModule().NewContext(int64(testHeight)) require.NoError(t, err) - consensusModImpl := GetConsensusModImplementation(pocketNode) - consensusModImpl.FieldByName("Height").SetUint(testHeight) - consensusModImpl.FieldByName("Step").SetInt(testStep) - consensusModImpl.FieldByName("LeaderId").Set(reflect.Zero(reflect.TypeOf(&leaderId))) // This is re-elected during paceMaker catchup - consensusModImpl.FieldByName("UtilityContext").Set(reflect.ValueOf(utilityContext)) + consensusModElem := GetConsensusModElem(pocketNode) + consensusModElem.FieldByName("Height").SetUint(testHeight) + consensusModElem.FieldByName("Step").SetInt(testStep) + consensusModElem.FieldByName("LeaderId").Set(reflect.Zero(reflect.TypeOf(&leaderId))) // This is re-elected during paceMaker catchup + + consensusModImpl := GetConsensusModImpl(pocketNode) + consensusModImpl.MethodByName("SetUtilityContext").Call([]reflect.Value{reflect.ValueOf(utilityContext)}) } // Set the leader to be in the highest round. - GetConsensusModImplementation(pocketNodes[1]).FieldByName("Round").SetUint(uint64(leaderRound - 2)) - GetConsensusModImplementation(pocketNodes[2]).FieldByName("Round").SetUint(uint64(leaderRound - 3)) - GetConsensusModImplementation(pocketNodes[leaderId]).FieldByName("Round").SetUint(uint64(leaderRound)) - GetConsensusModImplementation(pocketNodes[4]).FieldByName("Round").SetUint(uint64(leaderRound - 4)) + GetConsensusModElem(pocketNodes[1]).FieldByName("Round").SetUint(uint64(leaderRound - 2)) + GetConsensusModElem(pocketNodes[2]).FieldByName("Round").SetUint(uint64(leaderRound - 3)) + GetConsensusModElem(pocketNodes[leaderId]).FieldByName("Round").SetUint(uint64(leaderRound)) + GetConsensusModElem(pocketNodes[4]).FieldByName("Round").SetUint(uint64(leaderRound - 4)) prepareProposal := &typesCons.HotstuffMessage{ Type: consensus.Propose, diff --git a/consensus/consensus_tests/utils_test.go b/consensus/consensus_tests/utils_test.go index 9a2473541..3bda5de07 100644 --- a/consensus/consensus_tests/utils_test.go +++ b/consensus/consensus_tests/utils_test.go @@ -181,13 +181,17 @@ func StartAllTestPocketNodes(t *testing.T, pocketNodes IdToNodeMapping) { // define the interfaces used for debug/development. The latter will probably scale more but will // require more effort and pollute the source code with debugging information. func GetConsensusNodeState(node *shared.Node) typesCons.ConsensusNodeState { - return reflect.ValueOf(node.GetBus().GetConsensusModule()).MethodByName("GetNodeState").Call([]reflect.Value{})[0].Interface().(typesCons.ConsensusNodeState) + return GetConsensusModImpl(node).MethodByName("GetNodeState").Call([]reflect.Value{})[0].Interface().(typesCons.ConsensusNodeState) } -func GetConsensusModImplementation(node *shared.Node) reflect.Value { +func GetConsensusModElem(node *shared.Node) reflect.Value { return reflect.ValueOf(node.GetBus().GetConsensusModule()).Elem() } +func GetConsensusModImpl(node *shared.Node) reflect.Value { + return reflect.ValueOf(node.GetBus().GetConsensusModule()) +} + /*** Debug/Development Message Helpers ***/ func TriggerNextView(t *testing.T, node *shared.Node) { diff --git a/consensus/hotstuff_leader.go b/consensus/hotstuff_leader.go index 1e1cefa7d..54f41e0dd 100644 --- a/consensus/hotstuff_leader.go +++ b/consensus/hotstuff_leader.go @@ -334,14 +334,14 @@ func (m *ConsensusModule) prepareAndApplyBlock() (*typesCons.Block, error) { lastByzValidators := make([][]byte, 0) // Reap the mempool for transactions to be applied in this block - txs, err := m.UtilityContext.GetProposalTransactions(m.privateKey.Address(), maxTxBytes, lastByzValidators) + txs, err := m.utilityContext.GetProposalTransactions(m.privateKey.Address(), maxTxBytes, lastByzValidators) if err != nil { return nil, err } // OPTIMIZE: Determine if we can avoid the `ApplyBlock` call here // Apply all the transactions in the block - appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) + appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), m.privateKey.Address(), txs, lastByzValidators) if err != nil { return nil, err } diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index 9da379a61..a71119307 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -231,7 +231,7 @@ func (m *ConsensusModule) applyBlock(block *typesCons.Block) error { lastByzValidators := make([][]byte, 0) // Apply all the transactions in the block and get the appHash - appHash, err := m.UtilityContext.ApplyBlock(int64(m.Height), block.BlockHeader.ProposerAddress, block.Transactions, lastByzValidators) + appHash, err := m.utilityContext.ApplyBlock(int64(m.Height), block.BlockHeader.ProposerAddress, block.Transactions, lastByzValidators) if err != nil { return err } diff --git a/consensus/module.go b/consensus/module.go index 5259fc0cf..0f7f9f9f7 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -63,7 +63,7 @@ type ConsensusModule struct { validatorMap typesCons.ValidatorMap // Module Dependencies - UtilityContext modules.UtilityContext + utilityContext modules.UtilityContext paceMaker Pacemaker leaderElectionMod leader_election.LeaderElectionModule @@ -133,7 +133,7 @@ func Create(configPath, genesisPath string, useRandomPK bool) (modules.Consensus lastAppHash: "", validatorMap: valMap, - UtilityContext: nil, + utilityContext: nil, paceMaker: paceMaker, leaderElectionMod: leaderElectionMod, @@ -262,6 +262,11 @@ func (m *ConsensusModule) ValidatorMap() modules.ValidatorMap { return typesCons.ValidatorMapToModulesValidatorMap(m.validatorMap) } +// TODO(#256): Currently only used for testing purposes +func (m *ConsensusModule) SetUtilityContext(utilityContext modules.UtilityContext) { + m.utilityContext = utilityContext +} + // TODO: Populate the entire state from the persistence module: validator set, quorum cert, last block hash, etc... func (m *ConsensusModule) loadPersistedState() error { persistenceContext, err := m.GetBus().GetPersistenceModule().NewReadContext(-1) // Unknown height From 317e677fad6274b7cd20f3fd44bf2ed5afc7ab06 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 3 Oct 2022 21:30:10 -0700 Subject: [PATCH 55/56] Updated a comment --- consensus/hotstuff_replica.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/hotstuff_replica.go b/consensus/hotstuff_replica.go index a71119307..e55e85585 100644 --- a/consensus/hotstuff_replica.go +++ b/consensus/hotstuff_replica.go @@ -216,7 +216,7 @@ func (m *ConsensusModule) validateProposal(msg *typesCons.HotstuffMessage) error return nil } - // Liveness: is node is locked on a QC from the past? + // Liveness: is node locked on a QC from the past? // DISCUSS: Where should additional logic be added to unlock the node? if isLocked, err := isNodeLockedOnPastQC(justifyQC, lockedQC); isLocked { return err From 070ba2a0b2ed05ef6231a0c6aeadd8a28e5fc352 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 4 Oct 2022 16:57:13 -0700 Subject: [PATCH 56/56] Added a TODO for 283 --- consensus/module.go | 1 + 1 file changed, 1 insertion(+) diff --git a/consensus/module.go b/consensus/module.go index 0f7f9f9f7..917cd88d4 100644 --- a/consensus/module.go +++ b/consensus/module.go @@ -63,6 +63,7 @@ type ConsensusModule struct { validatorMap typesCons.ValidatorMap // Module Dependencies + // TODO(#283): Improve how `utilityContext` is managed utilityContext modules.UtilityContext paceMaker Pacemaker leaderElectionMod leader_election.LeaderElectionModule