[Tooling] Flaky tests - Issue #192

.github/workflows/main.yml

@@ -18,6 +18,7 @@ env:
   # Even though we can test against multiple versions, this one is considered a target version.
   TARGET_GOLANG_VERSION: "1.18"
   PROTOC_VERSION: "3.19.4"
+  PUSH_TO_DOCKERHUB: false
 
 jobs:
   test-multiple-go-versions:
@@ -101,17 +102,20 @@ jobs:
             type=raw,value=latest,enable={{is_default_branch}}${{ matrix.imageType == 'dev' && ',suffix=-dev' || '' }}
       # Turned off until we set up docker hub
       - name: Login to  DockerHub
+        if: ${{ env.PUSH_TO_DOCKERHUB != 'false' }}
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to GitHub Container Registry
+        if: ${{ env.PUSH_TO_DOCKERHUB != 'false' }}
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push Docker image
+        if: ${{ env.PUSH_TO_DOCKERHUB != 'false' }}
         uses: docker/build-push-action@v3
         with:
           push: true

app/pocket/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"log"
 
+	"github.com/benbjohnson/clock"
 	"github.com/pokt-network/pocket/shared"
 )
 
@@ -21,7 +22,7 @@ func main() {
 		log.Printf("Version flag currently unused %s\n", version)
 		return
 	}
-	pocketNode, err := shared.Create(*configFilename, *genesisFilename)
+	pocketNode, err := shared.Create(*configFilename, *genesisFilename, clock.New())
 	if err != nil {
 		log.Fatalf("Failed to create pocket node: %s", err)
 	}

consensus/consensus_tests/hotstuff_test.go

@@ -3,7 +3,9 @@ package consensus_tests
 import (
 	"fmt"
 	"testing"
+	"time"
 
+	"github.com/benbjohnson/clock"
 	"github.com/pokt-network/pocket/consensus"
 	typesCons "github.com/pokt-network/pocket/consensus/types"
 	"github.com/pokt-network/pocket/shared/modules"
@@ -15,24 +17,33 @@ func TestHotstuff4Nodes1BlockHappyPath(t *testing.T) {
 	numNodes := 4
 	configs, genesisStates := GenerateNodeConfigs(t, numNodes)
 
+	clockMock := clock.NewMock()
+	go timeReminder(clockMock, 100*time.Millisecond)
+
 	// Create & start test pocket nodes
 	testChannel := make(modules.EventsChannel, 100)
-	pocketNodes := CreateTestConsensusPocketNodes(t, configs, genesisStates, testChannel)
+	pocketNodes := CreateTestConsensusPocketNodes(t, configs, genesisStates, clockMock, testChannel)
 	StartAllTestPocketNodes(t, pocketNodes)
 
 	// Debug message to start consensus by triggering first view change
 	for _, pocketNode := range pocketNodes {
 		TriggerNextView(t, pocketNode)
 	}
 
+	advanceTime(clockMock, 10*time.Millisecond)
+
 	// NewRound
-	newRoundMessages, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.NewRound, consensus.Propose, numNodes, 1000)
+	newRoundMessages, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.NewRound, consensus.Propose, numNodes, 1000)
 	require.NoError(t, err)
-	for _, pocketNode := range pocketNodes {
+	for nodeId, pocketNode := range pocketNodes {
 		nodeState := GetConsensusNodeState(pocketNode)
-		require.Equal(t, uint64(1), nodeState.Height)
-		require.Equal(t, uint8(consensus.NewRound), nodeState.Step)
-		require.Equal(t, uint8(0), nodeState.Round)
+		assertNodeConsensusView(t, nodeId,
+			typesCons.ConsensusNodeState{
+				Height: 1,
+				Step:   uint8(consensus.NewRound),
+				Round:  0,
+			},
+			nodeState)
 		require.Equal(t, false, nodeState.IsLeader)
 	}
 	for _, message := range newRoundMessages {
@@ -44,96 +55,136 @@ func TestHotstuff4Nodes1BlockHappyPath(t *testing.T) {
 	leaderId := typesCons.NodeId(2)
 	leader := pocketNodes[leaderId]
 
+	advanceTime(clockMock, 10*time.Millisecond)
+
 	// Prepare
-	prepareProposal, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.Prepare, consensus.Propose, 1, 1000)
+	prepareProposal, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.Prepare, consensus.Propose, 1, 1000)
 	require.NoError(t, err)
-	for _, pocketNode := range pocketNodes {
+	for nodeId, pocketNode := range pocketNodes {
 		nodeState := GetConsensusNodeState(pocketNode)
-		require.Equal(t, uint64(1), nodeState.Height)
-		require.Equal(t, uint8(consensus.Prepare), nodeState.Step)
-		require.Equal(t, uint8(0), nodeState.Round)
+		assertNodeConsensusView(t, nodeId,
+			typesCons.ConsensusNodeState{
+				Height: 1,
+				Step:   uint8(consensus.Prepare),
+				Round:  0,
+			},
+			nodeState)
 		require.Equal(t, leaderId, nodeState.LeaderId, fmt.Sprintf("%d should be the current leader", leaderId))
 	}
 	for _, message := range prepareProposal {
 		P2PBroadcast(t, pocketNodes, message)
 	}
 
+	advanceTime(clockMock, 10*time.Millisecond)
+
 	// Precommit
-	prepareVotes, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.Prepare, consensus.Vote, numNodes, 1000)
+	prepareVotes, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.Prepare, consensus.Vote, numNodes, 1000)
 	require.NoError(t, err)
 	for _, vote := range prepareVotes {
 		P2PSend(t, leader, vote)
 	}
 
-	preCommitProposal, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.PreCommit, consensus.Propose, 1, 1000)
+	advanceTime(clockMock, 10*time.Millisecond)
+
+	preCommitProposal, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.PreCommit, consensus.Propose, 1, 1000)
 	require.NoError(t, err)
-	for _, pocketNode := range pocketNodes {
+	for nodeId, pocketNode := range pocketNodes {
 		nodeState := GetConsensusNodeState(pocketNode)
-		require.Equal(t, uint64(1), nodeState.Height)
-		require.Equal(t, uint8(consensus.PreCommit), nodeState.Step)
-		require.Equal(t, uint8(0), nodeState.Round)
+		assertNodeConsensusView(t, nodeId,
+			typesCons.ConsensusNodeState{
+				Height: 1,
+				Step:   uint8(consensus.PreCommit),
+				Round:  0,
+			},
+			nodeState)
 		require.Equal(t, leaderId, nodeState.LeaderId, fmt.Sprintf("%d should be the current leader", leaderId))
 	}
 	for _, message := range preCommitProposal {
 		P2PBroadcast(t, pocketNodes, message)
 	}
 
+	advanceTime(clockMock, 10*time.Millisecond)
+
 	// Commit
-	preCommitVotes, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.PreCommit, consensus.Vote, numNodes, 1000)
+	preCommitVotes, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.PreCommit, consensus.Vote, numNodes, 1000)
 	require.NoError(t, err)
 	for _, vote := range preCommitVotes {
 		P2PSend(t, leader, vote)
 	}
 
-	commitProposal, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.Commit, consensus.Propose, 1, 1000)
+	advanceTime(clockMock, 10*time.Millisecond)
+
+	commitProposal, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.Commit, consensus.Propose, 1, 1000)
 	require.NoError(t, err)
-	for _, pocketNode := range pocketNodes {
+	for nodeId, pocketNode := range pocketNodes {
 		nodeState := GetConsensusNodeState(pocketNode)
-		require.Equal(t, uint64(1), nodeState.Height)
-		require.Equal(t, uint8(consensus.Commit), nodeState.Step)
-		require.Equal(t, uint8(0), nodeState.Round)
+		assertNodeConsensusView(t, nodeId,
+			typesCons.ConsensusNodeState{
+				Height: 1,
+				Step:   uint8(consensus.Commit),
+				Round:  0,
+			},
+			nodeState)
 		require.Equal(t, leaderId, nodeState.LeaderId, fmt.Sprintf("%d should be the current leader", leaderId))
 	}
 	for _, message := range commitProposal {
 		P2PBroadcast(t, pocketNodes, message)
 	}
 
+	advanceTime(clockMock, 10*time.Millisecond)
+
 	// Decide
-	commitVotes, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.Commit, consensus.Vote, numNodes, 1000)
+	commitVotes, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.Commit, consensus.Vote, numNodes, 1000)
 	require.NoError(t, err)
 	for _, vote := range commitVotes {
 		P2PSend(t, leader, vote)
 	}
 
-	decideProposal, err := WaitForNetworkConsensusMessages(t, testChannel, consensus.Decide, consensus.Propose, 1, 1000)
+	advanceTime(clockMock, 10*time.Millisecond)
+
+	decideProposal, err := WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.Decide, consensus.Propose, 1, 1000)
 	require.NoError(t, err)
 	for pocketId, pocketNode := range pocketNodes {
 		nodeState := GetConsensusNodeState(pocketNode)
 		// Leader has already committed the block and hence moved to the next height.
 		if pocketId == leaderId {
-			require.Equal(t, uint64(2), nodeState.Height)
-			require.Equal(t, uint8(consensus.NewRound), nodeState.Step)
-			require.Equal(t, uint8(0), nodeState.Round)
+			assertNodeConsensusView(t, pocketId,
+				typesCons.ConsensusNodeState{
+					Height: 2,
+					Step:   uint8(consensus.NewRound),
+					Round:  0,
+				},
+				nodeState)
 			require.Equal(t, nodeState.LeaderId, typesCons.NodeId(0), "Leader should be empty")
 			continue
 		}
-		require.Equal(t, uint64(1), nodeState.Height)
-		require.Equal(t, uint8(consensus.Decide), nodeState.Step)
-		require.Equal(t, uint8(0), nodeState.Round)
+		assertNodeConsensusView(t, pocketId,
+			typesCons.ConsensusNodeState{
+				Height: 1,
+				Step:   uint8(consensus.Decide),
+				Round:  0,
+			},
+			nodeState)
 		require.Equal(t, leaderId, nodeState.LeaderId, fmt.Sprintf("%d should be the current leader", leaderId))
 	}
 	for _, message := range decideProposal {
 		P2PBroadcast(t, pocketNodes, message)
 	}
 
+	advanceTime(clockMock, 10*time.Millisecond)
+
 	// Block has been committed and new round has begun
-	_, err = WaitForNetworkConsensusMessages(t, testChannel, consensus.NewRound, consensus.Propose, numNodes, 1000)
+	_, err = WaitForNetworkConsensusMessages(t, clockMock, testChannel, consensus.NewRound, consensus.Propose, numNodes, 1000)
 	require.NoError(t, err)
-	for _, pocketNode := range pocketNodes {
+	for pocketId, pocketNode := range pocketNodes {
 		nodeState := GetConsensusNodeState(pocketNode)
-		require.Equal(t, uint64(2), nodeState.Height)
-		require.Equal(t, uint8(consensus.NewRound), nodeState.Step)
-		require.Equal(t, uint8(0), nodeState.Round)
+		assertNodeConsensusView(t, pocketId,
+			typesCons.ConsensusNodeState{
+				Height: 2,
+				Step:   uint8(consensus.NewRound),
+				Round:  0,
+			},
+			nodeState)
 		require.Equal(t, nodeState.LeaderId, typesCons.NodeId(0), "Leader should be empty")
 	}
 }