feat(internal/database): replace chaindb/badgerdb with pebbledb

cmd/gossamer/README.md

@@ -14,6 +14,7 @@ cd gossamer
 ### Compile
 
 To put the binary in ./bin, run:
+
 ```bash
 make build
 ```
@@ -62,7 +63,7 @@ The node configuration can be modified in the `config.toml` file.
 ### Start the node
 
 ```bash
-gossamer --basepath /tmp/gossamer --key alice 
+gossamer --basepath /tmp/gossamer --key alice
 ```
 
 **Note: The `init` command is optional. If the node is not initialised, it will be initialised with the default configuration.**
@@ -108,19 +109,22 @@ This subcommand provides capabilities that are similar to
 [Parity's Subkey utility](https://docs.substrate.io/v3/tools/subkey).
 
 The account command supports following arguments:
+
 - `generate` - generates a new key pair; specify `--scheme ed25519`, `--scheme secp256k1`, or `--scheme sr25519` (default)
 - `list` - lists the keys in the Gossamer keystore
 - `import` - imports a key from a keystore file
 - `import-raw` - imports a raw key from a keystore file
 
 Supported flags:
+
 - `keystore-path` - path to the Gossamer keystore
 - `keystore-file` - path to the keystore file
 - `chain` - path to the human-readable chain-spec file
 - `--scheme` - `ed25519`, `secp256k1`, or `sr25519` (default)
 - `--password` - allows the user to provide a password to either encrypt a generated key or unlock the Gossamer keystore
 
 Examples:
+
 - `gossamer account generate --scheme ed25519` - generates an `ed25519` key pair
 - `gossamer account list` - lists the keys in the Gossamer keystore
 - `gossamer account import --keystore-file keystore.json` - imports a key from a keystore file
@@ -145,6 +149,7 @@ represent the Gossamer default configuration.
 - `--output-path` - path to the file where the compiled chain-spec should be written
 
 Examples:
+
 - `gossamer build-spec --chain chain-spec.json --output-path compiled-chain-spec.json` - compiles a human-readable
   chain-spec into a format that Gossamer can consume
 - `gossamer build-spec --chain chain-spec.json --raw --output-path compiled-chain-spec.json` - compiles a human-readable
@@ -166,6 +171,7 @@ of a JSON file. The input for this subcommand can be retrieved from
 - `--chain` - path to the human-readable chain-spec file
 
 Examples:
+
 - `gossamer import-state --first-slot 1 --header header.json --state state.json --chain chain-spec.json` - seeds Gossamer
   storage with key-value pairs from a JSON file
 
@@ -185,6 +191,7 @@ What follows is a list that describes the services and capabilities that inform
 
 #### State
 
+// TODO: update the `State` description to pebble db
 This service is a wrapper around an instance of [`chaindb`](https://github.com/ChainSafe/chaindb), a key-value database
 that is built on top of [BadgerDB](https://github.com/dgraph-io/badger) from [Dgraph](https://dgraph.io/). The state
 service provides storage capabilities for the other Gossamer services - each service is assigned a prefix that is added
@@ -271,4 +278,4 @@ capabilities are defined in the [dot/telemetry](../../dot/telemetry) package and
 The default listening address for Prometheus metrics is `localhost:9876`, and Gossamer allows the user to configure this parameter with the
 `--metrics-address` command-line parameter. The Gossamer telemetry server publishes telemetry data that is compatible with
 [Polkadot Telemetry](https://github.com/paritytech/substrate-telemetry) and
-[its helpful UI](https://telemetry.polkadot.io/).
+[its helpful UI](https://telemetry.polkadot.io/).

dot/core/service.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/ChainSafe/gossamer/dot/network"
 	"github.com/ChainSafe/gossamer/dot/types"
@@ -100,6 +101,26 @@ func NewService(cfg *Config) (*Service, error) {
 // Start starts the core service
 func (s *Service) Start() error {
 	go s.handleBlocksAsync()
+
+	go func() {
+		time.Sleep(30 * time.Second)
+		header, err := s.blockState.BestBlockHeader()
+		if err != nil {
+			panic(err)
+		}
+
+		ba := &network.BlockAnnounceMessage{
+			ParentHash:     header.ParentHash,
+			Number:         header.Number,
+			StateRoot:      header.StateRoot,
+			ExtrinsicsRoot: header.ExtrinsicsRoot,
+			Digest:         header.Digest,
+			BestBlock:      true,
+		}
+
+		s.net.GossipMessage(ba)
+	}()
+
 	return nil
 }
 

dot/network/discovery.go

@@ -28,7 +28,7 @@ var (
 	startDHTTimeout             = time.Second * 10
 	initialAdvertisementTimeout = time.Millisecond
 	tryAdvertiseTimeout         = time.Second * 30
-	connectToPeersTimeout       = time.Minute * 5
+	connectToPeersTimeout       = time.Minute
 	findPeersTimeout            = time.Minute
 )
 
@@ -183,7 +183,7 @@ func (d *discovery) checkPeerCount() {
 		case <-d.ctx.Done():
 			return
 		case <-ticker.C:
-			if len(d.h.Network().Peers()) > d.minPeers {
+			if len(d.h.Network().Peers()) >= d.maxPeers {
 				continue
 			}
 

dot/network/errors.go

@@ -8,6 +8,8 @@ import (
 )
 
 var (
+	ErrReceivedEmptyMessage = errors.New("received empty message")
+
 	errCannotValidateHandshake       = errors.New("failed to validate handshake")
 	errMessageTypeNotValid           = errors.New("message type is not valid")
 	errInvalidHandshakeForPeer       = errors.New("peer previously sent invalid handshake")

dot/network/message.go

@@ -17,6 +17,9 @@ import (
 	"github.com/ChainSafe/gossamer/pkg/scale"
 )
 
+// MaxBlocksInResponse is maximum number of block data a BlockResponse message can contain
+const MaxBlocksInResponse = 128
+
 type MessageType byte
 
 // Message types for notifications protocol messages. Used internally to map message to protocol.
@@ -44,6 +47,9 @@ const (
 	RequestedDataReceipt       = byte(4)
 	RequestedDataMessageQueue  = byte(8)
 	RequestedDataJustification = byte(16)
+	BootstrapRequestData       = RequestedDataHeader +
+		RequestedDataBody +
+		RequestedDataJustification
 )
 
 var _ Message = (*BlockRequestMessage)(nil)
@@ -354,3 +360,56 @@ func (cm *ConsensusMessage) Hash() (common.Hash, error) {
 	}
 	return common.Blake2bHash(encMsg)
 }
+
+func NewBlockRequest(startingBlock variadic.Uint32OrHash, amount uint32,
+	requestedData byte, direction SyncDirection) *BlockRequestMessage {
+	return &BlockRequestMessage{
+		RequestedData: requestedData,
+		StartingBlock: startingBlock,
+		Direction:     direction,
+		Max:           &amount,
+	}
+}
+
+func NewAscendingBlockRequests(startNumber, targetNumber uint, requestedData byte) []*BlockRequestMessage {
+	if startNumber > targetNumber {
+		return []*BlockRequestMessage{}
+	}
+
+	diff := targetNumber - startNumber
+
+	// start and end block are the same, just request 1 block
+	if diff == 0 {
+		return []*BlockRequestMessage{
+			NewBlockRequest(*variadic.MustNewUint32OrHash(uint32(startNumber)), 1, requestedData, Ascending),
+		}
+	}
+
+	numRequests := diff / MaxBlocksInResponse
+	// we should check if the diff is in the maxResponseSize bounds
+	// otherwise we should increase the numRequests by one, take this
+	// example, we want to sync from 0 to 259, the diff is 259
+	// then the num of requests is 2 (uint(259)/uint(128)) however two requests will
+	// retrieve only 256 blocks (each request can retrieve a max of 128 blocks), so we should
+	// create one more request to retrieve those missing blocks, 3 in this example.
+	missingBlocks := diff % MaxBlocksInResponse
+	if missingBlocks != 0 {
+		numRequests++
+	}
+
+	reqs := make([]*BlockRequestMessage, numRequests)
+	for i := uint(0); i < numRequests; i++ {
+		max := uint32(MaxBlocksInResponse)
+
+		lastIteration := numRequests - 1
+		if i == lastIteration && missingBlocks != 0 {
+			max = uint32(missingBlocks)
+		}
+
+		start := variadic.MustNewUint32OrHash(startNumber)
+		reqs[i] = NewBlockRequest(*start, max, requestedData, Ascending)
+		startNumber += uint(max)
+	}
+
+	return reqs
+}

dot/network/message_test.go

@@ -422,3 +422,122 @@ func TestDecodeConsensusMessage(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, encMsg, encodedMessage)
 }
+
+func TestAscendingBlockRequest(t *testing.T) {
+	one := uint32(1)
+	three := uint32(3)
+	maxResponseSize := uint32(MaxBlocksInResponse)
+	cases := map[string]struct {
+		startNumber, targetNumber   uint
+		expectedBlockRequestMessage []*BlockRequestMessage
+	}{
+		"start_greater_than_target": {
+			startNumber:                 10,
+			targetNumber:                0,
+			expectedBlockRequestMessage: []*BlockRequestMessage{},
+		},
+
+		"no_difference_between_start_and_target": {
+			startNumber:  10,
+			targetNumber: 10,
+			expectedBlockRequestMessage: []*BlockRequestMessage{
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(10)),
+					Direction:     Ascending,
+					Max:           &one,
+				},
+			},
+		},
+
+		"requesting_128_blocks": {
+			startNumber:  0,
+			targetNumber: 128,
+			expectedBlockRequestMessage: []*BlockRequestMessage{
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(0)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+			},
+		},
+
+		"requesting_4_chunks_of_128_blocks": {
+			startNumber:  0,
+			targetNumber: 512, // 128 * 4
+			expectedBlockRequestMessage: []*BlockRequestMessage{
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(0)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(128)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(256)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(384)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+			},
+		},
+
+		"requesting_4_chunks_of_128_plus_3_blocks": {
+			startNumber:  0,
+			targetNumber: (128 * 4) + 3,
+			expectedBlockRequestMessage: []*BlockRequestMessage{
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(0)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(128)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(256)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(384)),
+					Direction:     Ascending,
+					Max:           &maxResponseSize,
+				},
+				{
+					RequestedData: BootstrapRequestData,
+					StartingBlock: *variadic.MustNewUint32OrHash(uint32(512)),
+					Direction:     Ascending,
+					Max:           &three,
+				},
+			},
+		},
+	}
+
+	for tname, tt := range cases {
+		tt := tt
+
+		t.Run(tname, func(t *testing.T) {
+			requests := NewAscendingBlockRequests(tt.startNumber, tt.targetNumber, BootstrapRequestData)
+			require.Equal(t, tt.expectedBlockRequestMessage, requests)
+		})
+	}
+}

dot/network/service.go

@@ -605,6 +605,11 @@ func (s *Service) NetworkState() common.NetworkState {
 	}
 }
 
+// AllConnectedPeersID returns all the connected to the node instance
+func (s *Service) AllConnectedPeersID() []peer.ID {
+	return s.host.p2pHost.Network().Peers()
+}
+
 // Peers returns information about connected peers needed for the rpc server
 func (s *Service) Peers() []common.PeerInfo {
 	var peers []common.PeerInfo

dot/network/sync.go

@@ -4,16 +4,10 @@
 package network
 
 import (
-	"time"
-
 	libp2pnetwork "github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
-var (
-	BlockRequestTimeout = time.Second * 20
-)
-
 // handleSyncStream handles streams with the <protocol-id>/sync/2 protocol ID
 func (s *Service) handleSyncStream(stream libp2pnetwork.Stream) {
 	if stream == nil {

dot/node_integration_test.go

@@ -57,7 +57,6 @@ func TestNewNode(t *testing.T) {
 	initConfig.Account.Key = "alice"
 	initConfig.Core.Role = common.FullNodeRole
 	initConfig.Core.WasmInterpreter = wazero_runtime.Name
-
 	initConfig.Log.Digest = "critical"
 
 	networkConfig := &network.Config{
@@ -86,7 +85,7 @@ func TestNewNode(t *testing.T) {
 	assert.NoError(t, err)
 
 	mockServiceRegistry := NewMockServiceRegisterer(ctrl)
-	mockServiceRegistry.EXPECT().RegisterService(gomock.Any()).Times(8)
+	mockServiceRegistry.EXPECT().RegisterService(gomock.Any()).Times(9)
 
 	m := NewMocknodeBuilderIface(ctrl)
 	m.EXPECT().isNodeInitialised(initConfig.BasePath).Return(nil)
@@ -325,6 +324,7 @@ func TestStartStopNode(t *testing.T) {
 	config.ChainSpec = genFile
 	config.Core.GrandpaAuthority = false
 	config.Core.BabeAuthority = false
+	config.Network.MinPeers = 0
 
 	err := InitNode(config)
 	require.NoError(t, err)