From 337ee8f413a6b66e6fb65deac8f86c6366ffa6c3 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 15:15:54 +0100 Subject: [PATCH 01/82] refactor: move peer and url conversion to `p2p/utils` pkg --- {libp2p/network => p2p/utils}/peer_conversion.go | 2 +- {libp2p/network => p2p/utils}/url_conversion.go | 8 +++----- {libp2p/network => p2p/utils}/url_conversion_test.go | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) rename {libp2p/network => p2p/utils}/peer_conversion.go (99%) rename {libp2p/network => p2p/utils}/url_conversion.go (96%) rename {libp2p/network => p2p/utils}/url_conversion_test.go (99%) diff --git a/libp2p/network/peer_conversion.go b/p2p/utils/peer_conversion.go similarity index 99% rename from libp2p/network/peer_conversion.go rename to p2p/utils/peer_conversion.go index 51316dca8..b0ba7b9ce 100644 --- a/libp2p/network/peer_conversion.go +++ b/p2p/utils/peer_conversion.go @@ -1,4 +1,4 @@ -package network +package utils import ( "fmt" diff --git a/libp2p/network/url_conversion.go b/p2p/utils/url_conversion.go similarity index 96% rename from libp2p/network/url_conversion.go rename to p2p/utils/url_conversion.go index b8d16056a..0a4e54fc4 100644 --- a/libp2p/network/url_conversion.go +++ b/p2p/utils/url_conversion.go @@ -1,4 +1,4 @@ -package network +package utils import ( "crypto/rand" @@ -137,7 +137,7 @@ func getPeerIP(hostname string) (net.IP, error) { return peerIP, nil } - // CONSIDER: using a `/dns<4 or 6>/` multiaddr instead of resolving here. + // CONSIDERATION: using a `/dns<4 or 6>/` multiaddr instead of resolving here. // I attempted using `/dns4/.../tcp/...` and go this error: // > failed to listen on any addresses: [can only dial TCP over IPv4 or IPv6] addrs, err := net.LookupHost(hostname) @@ -145,9 +145,7 @@ func getPeerIP(hostname string) (net.IP, error) { return nil, newResolvePeerIPErr(hostname, err) } - // CONSIDER: which address(es) should we use when multiple - // are provided in a DNS response? - // CONSIDER: preferring IPv6 responses when resolving DNS. + // CONSIDERATION: preferring IPv6 responses when resolving DNS. // Return first address which is a parsable IP address. var ( validIPs []net.IP diff --git a/libp2p/network/url_conversion_test.go b/p2p/utils/url_conversion_test.go similarity index 99% rename from libp2p/network/url_conversion_test.go rename to p2p/utils/url_conversion_test.go index 9dff3468b..44e894a4f 100644 --- a/libp2p/network/url_conversion_test.go +++ b/p2p/utils/url_conversion_test.go @@ -1,4 +1,4 @@ -package network +package utils import ( "fmt" From 206b647a3800f32ce89a29cd45fb26c53150064d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 17 Mar 2023 14:14:22 +0100 Subject: [PATCH 02/82] refactor: move and rename `network.setupHost` to `utils.PopulateLibp2pHost` --- libp2p/network/network.go | 35 -------------------------- p2p/utils/host.go | 52 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 p2p/utils/host.go diff --git a/libp2p/network/network.go b/libp2p/network/network.go index 511313a8c..1e358275d 100644 --- a/libp2p/network/network.go +++ b/libp2p/network/network.go @@ -217,41 +217,6 @@ func (p2pNet *libp2pNetwork) setupCurrentHeightProvider() providers.CurrentHeigh return currentHeightProviderModule.(providers.CurrentHeightProvider) } -// setupHost iterates through peers in given `pstore`, converting peer info for -// use with libp2p and adding it to the underlying libp2p host's peerstore. -// (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/host#Host) -// (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/peerstore#Peerstore) -func (p2pNet *libp2pNetwork) setupHost() error { - for _, peer := range p2pNet.pstore.GetPeerList() { - pubKey, err := Libp2pPublicKeyFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer public key, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - libp2pPeer, err := Libp2pAddrInfoFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer info, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - - p2pNet.host.Peerstore().AddAddrs(libp2pPeer.ID, libp2pPeer.Addrs, defaultPeerTTL) - if err := p2pNet.host.Peerstore().AddPubKey(libp2pPeer.ID, pubKey); err != nil { - return fmt.Errorf( - "adding peer public key, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - } - return nil -} - // setup initializes p2pNet.pstore using the PeerstoreProvider // and CurrentHeightProvider registered on the bus, if preseent. func (p2pNet *libp2pNetwork) setup() (err error) { diff --git a/p2p/utils/host.go b/p2p/utils/host.go new file mode 100644 index 000000000..5b456b8dd --- /dev/null +++ b/p2p/utils/host.go @@ -0,0 +1,52 @@ +package utils + +import ( + "fmt" + "time" + + libp2pHost "github.com/libp2p/go-libp2p/core/host" + + "github.com/pokt-network/pocket/shared/p2p" +) + +const ( + week = time.Hour * 24 * 7 + // TECHDEBT: consider more carefully and parameterize. + // TECHDEBT: unexport after consolidation of P2P modules + DefaultPeerTTL = 2 * week +) + +// PopulateLibp2pHost iterates through peers in given `pstore`, converting peer +// info for use with libp2p and adding it to the underlying libp2p host's peerstore. +// (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/host#Host) +// (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/peerstore#Peerstore) +func PopulateLibp2pHost(host libp2pHost.Host, pstore p2p.Peerstore) error { + for _, peer := range pstore.GetPeerList() { + pubKey, err := Libp2pPublicKeyFromPeer(peer) + if err != nil { + return fmt.Errorf( + "converting peer public key, pokt address: %s: %w", + peer.GetAddress(), + err, + ) + } + libp2pPeer, err := Libp2pAddrInfoFromPeer(peer) + if err != nil { + return fmt.Errorf( + "converting peer info, pokt address: %s: %w", + peer.GetAddress(), + err, + ) + } + + host.Peerstore().AddAddrs(libp2pPeer.ID, libp2pPeer.Addrs, DefaultPeerTTL) + if err := host.Peerstore().AddPubKey(libp2pPeer.ID, pubKey); err != nil { + return fmt.Errorf( + "adding peer public key, pokt address: %s: %w", + peer.GetAddress(), + err, + ) + } + } + return nil +} From d8e6fa5acd2abbdfe6b42c684b9c5884bd16f2ad Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 9 Mar 2023 13:49:08 +0100 Subject: [PATCH 03/82] chore: move protocol package into P2P module (cherry picked from commit 655e61a9595ee20a2edb35520b05f8ee62221b39) --- libp2p/module.go | 2 +- libp2p/network/network.go | 2 +- {libp2p => p2p}/protocol/protocol.go | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {libp2p => p2p}/protocol/protocol.go (100%) diff --git a/libp2p/module.go b/libp2p/module.go index 4bdf3c73a..4b64e5549 100644 --- a/libp2p/module.go +++ b/libp2p/module.go @@ -34,8 +34,8 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/pokt-network/pocket/libp2p/network" - "github.com/pokt-network/pocket/libp2p/protocol" "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/p2p/protocol" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/configs/types" diff --git a/libp2p/network/network.go b/libp2p/network/network.go index 1e358275d..550e13bbf 100644 --- a/libp2p/network/network.go +++ b/libp2p/network/network.go @@ -8,7 +8,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" libp2pHost "github.com/libp2p/go-libp2p/core/host" - "github.com/pokt-network/pocket/libp2p/protocol" + "github.com/pokt-network/pocket/p2p/protocol" "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/current_height_provider" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" diff --git a/libp2p/protocol/protocol.go b/p2p/protocol/protocol.go similarity index 100% rename from libp2p/protocol/protocol.go rename to p2p/protocol/protocol.go From 09f4ac89d78398593186c63e79a9de2463ad9e82 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 17 Mar 2023 14:20:21 +0100 Subject: [PATCH 04/82] refactor: consolidate libp2p & P2P modules --- libp2p/module.go | 374 ---------------------------------------- p2p/event_handler.go | 2 +- p2p/module.go | 401 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 320 insertions(+), 457 deletions(-) delete mode 100644 libp2p/module.go diff --git a/libp2p/module.go b/libp2p/module.go deleted file mode 100644 index 4b64e5549..000000000 --- a/libp2p/module.go +++ /dev/null @@ -1,374 +0,0 @@ -/* -TECHDEBT: This module currently imports types from the "legacy" P2P module. - -Migration path: - 1. Redefine P2P concrete types in terms of interfaces - - PeersManager (raintree/peersManager) - - Peer (p2p/types/NetworkPeer) - - AddrBook (p2p/types/AddrBook) - - AddrBookMap (p2p/types/NetworkPeer) - - rainTreeNetwork doesn't depend on any concrete p2p types - 2. Simplify libp2p module implementation - - Transport likely reduces to nothing - - Network interface can be simplified - - Consider renaming network as it functions more like a "router" - (NB: could be replaced in future iterations with a "raintree pubsub router") - 3. Remove "legacy" P2P module & rename libp2p module directory (possibly object names as well) - - P2PModule interface can be simplified - - Clean up TECHDEBT introduced in debug CLI and node startup -*/ -package libp2p - -import ( - "context" - "fmt" - "io" - "time" - - "github.com/libp2p/go-libp2p" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/libp2p/go-libp2p/core/host" - libp2pNetwork "github.com/libp2p/go-libp2p/core/network" - "github.com/multiformats/go-multiaddr" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" - - "github.com/pokt-network/pocket/libp2p/network" - "github.com/pokt-network/pocket/logger" - "github.com/pokt-network/pocket/p2p/protocol" - typesP2P "github.com/pokt-network/pocket/p2p/types" - "github.com/pokt-network/pocket/runtime/configs" - "github.com/pokt-network/pocket/runtime/configs/types" - "github.com/pokt-network/pocket/shared/crypto" - "github.com/pokt-network/pocket/shared/messaging" - "github.com/pokt-network/pocket/shared/modules" - "github.com/pokt-network/pocket/shared/modules/base_modules" -) - -var _ modules.P2PModule = &libp2pModule{} - -type libp2pModule struct { - base_modules.IntegratableModule - - logger *modules.Logger - cfg *configs.P2PConfig - identity libp2p.Option - listenAddrs libp2p.Option - // host encapsulates libp2p peerstore & connection manager - host host.Host - // pubsub is used for broadcast communication - // (i.e. multiple, unidentified receivers) - pubsub *pubsub.PubSub - // topic similar to pubsub but received messages are filtered by a "topic" string. - // Published messages are also given the respective topic before broadcast. - topic *pubsub.Topic - // subscription provides an interface to continuously read messages from. - subscription *pubsub.Subscription - network typesP2P.Network -} - -var ( - // TECHDEBT: configure timeouts. Consider security exposure vs. real-world conditions). - // TECHDEBT: parameterize and expose via config. - // readStreamTimeout is the duration to wait for a read operation on a - // stream to complete, after which the stream is closed ("timed out"). - readStreamTimeoutDuration = time.Second * 10 -) - -func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { - return new(libp2pModule).Create(bus, options...) -} - -func (mod *libp2pModule) GetModuleName() string { - return modules.P2PModuleName -} - -func (_ *libp2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { - logger.Global.Debug().Msg("Creating libp2p-backed network module") - mod := &libp2pModule{ - cfg: bus.GetRuntimeMgr().GetConfig().P2P, - logger: logger.Global.CreateLoggerForModule(modules.P2PModuleName), - } - - // MUST call before referencing mod.bus to ensure != nil. - bus.RegisterModule(mod) - - for _, option := range options { - option(mod) - } - - // TECHDEBT: investigate any unnecessary - // key exposure / duplication in memory - privateKey, err := crypto.NewLibP2PPrivateKey(mod.cfg.PrivateKey) - if err != nil { - return nil, fmt.Errorf("loading private key: %w", err) - } - - mod.identity = libp2p.Identity(privateKey) - - // INCOMPLETE: support RainTree network - if mod.cfg.UseRainTree { - return nil, fmt.Errorf("%s", "raintree is not yet compatible with libp2p") - } - - switch mod.cfg.ConnectionType { - case types.ConnectionType_TCPConnection: - addr, err := mod.getMultiaddr() - if err != nil { - return nil, fmt.Errorf("parsing multiaddr from config: %w", err) - } - mod.listenAddrs = libp2p.ListenAddrs(addr) - case types.ConnectionType_EmptyConnection: - mod.listenAddrs = libp2p.NoListenAddrs - default: - return nil, fmt.Errorf( - // DISCUSS: rename to "transport protocol" instead. - "unsupported connection type: %s: %w", - mod.cfg.ConnectionType, - err, - ) - } - - return mod, nil -} - -func (mod *libp2pModule) Start() error { - // IMPROVE: receive context in interface methods. - ctx := context.Background() - - // TECHDEBT: metrics integration. - var err error - opts := []libp2p.Option{ - mod.identity, - // INCOMPLETE(#544): add transport security! - } - - // Disable unused libp2p relay and ping services in client debug mode. - // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#DisableRelay - // and https://pkg.go.dev/github.com/libp2p/go-libp2p#Ping) - if mod.isClientDebugMode() { - opts = append(opts, - libp2p.DisableRelay(), - libp2p.Ping(false), - libp2p.NoListenAddrs, - ) - } else { - opts = append(opts, mod.listenAddrs) - } - - // Represents a libp2p network node, `libp2p.New` configures - // and starts listening according to options. - // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) - mod.host, err = libp2p.New(opts...) - if err != nil { - return fmt.Errorf("unable to create libp2p host: %w", err) - } - - listenAddrLogEvent := mod.logger.Info() - for i, addr := range host.InfoFromHost(mod.host).Addrs { - listenAddrLogEvent.Str(fmt.Sprintf("listen_addr_%d", i), addr.String()) - } - listenAddrLogEvent.Msg("Listening for incoming connections...") - - // TECHDEBT: use RandomSub or GossipSub once we're on more stable ground. - // IMPROVE: consider supporting multiple router types via config. - mod.pubsub, err = pubsub.NewFloodSub(ctx, mod.host) - if err != nil { - return fmt.Errorf("unable to create pubsub: %w", err) - } - - // Topic is used to `#Publish` messages. - mod.topic, err = mod.pubsub.Join(protocol.DefaultTopicStr) - if err != nil { - return fmt.Errorf("unable to join pubsub topic: %w", err) - } - - // Subscription is notified when a new message is received on the topic. - mod.subscription, err = mod.topic.Subscribe() - if err != nil { - return fmt.Errorf("subscribing to pubsub topic: %w", err) - } - - mod.network, err = network.NewLibp2pNetwork(mod.GetBus(), mod.logger, mod.host, mod.topic) - if err != nil { - return fmt.Errorf("creating network: %w", err) - } - - // Don't handle streams or read from the subscription in client debug mode. - if !mod.isClientDebugMode() { - mod.host.SetStreamHandler(protocol.PoktProtocolID, mod.handleStream) - go mod.readFromSubscription(ctx) - } - return nil -} - -func (mod *libp2pModule) Stop() error { - return mod.host.Close() -} - -func (mod *libp2pModule) Broadcast(msg *anypb.Any) error { - c := &messaging.PocketEnvelope{ - Content: msg, - } - //TECHDEBT: use shared/codec for marshalling - data, err := proto.MarshalOptions{Deterministic: true}.Marshal(c) - if err != nil { - return err - } - mod.logger.Info().Msg("broadcasting message to network") - - return mod.network.NetworkBroadcast(data) -} - -func (mod *libp2pModule) Send(addr crypto.Address, msg *anypb.Any) error { - c := &messaging.PocketEnvelope{ - Content: msg, - } - //TECHDEBT: use shared/codec for marshalling - data, err := proto.MarshalOptions{Deterministic: true}.Marshal(c) - if err != nil { - return err - } - - return mod.network.NetworkSend(data, addr) -} - -func (mod *libp2pModule) GetAddress() (crypto.Address, error) { - privateKey, err := crypto.NewPrivateKey(mod.cfg.PrivateKey) - if err != nil { - return nil, err - } - - return privateKey.Address(), nil -} - -// HandleEvent implements the respective `modules.Module` interface method. -func (mod *libp2pModule) HandleEvent(msg *anypb.Any) error { - return nil -} - -func (mod *libp2pModule) isClientDebugMode() bool { - return mod.GetBus().GetRuntimeMgr().GetConfig().ClientDebugMode -} - -// handleStream is called each time a peer establishes a new stream with this -// module's libp2p `host.Host`. -func (mod *libp2pModule) handleStream(stream libp2pNetwork.Stream) { - peer, err := network.PeerFromLibp2pStream(stream) - if err != nil { - mod.logger.Error().Err(err). - Str("address", peer.GetAddress().String()). - Msg("parsing remote peer public key") - - if err = stream.Close(); err != nil { - mod.logger.Error().Err(err) - } - } - - if err := mod.network.AddPeer(peer); err != nil { - mod.logger.Error().Err(err). - Str("address", peer.GetAddress().String()). - Msg("adding remote peer to address book") - } - - go mod.readStream(stream) -} - -// readStream is intended to be called in a goroutine. It continuously reads from -// the given stream for handling at the network level. Used for handling "direct" -// messages (i.e. one specific target node). -func (mod *libp2pModule) readStream(stream libp2pNetwork.Stream) { - closeStream := func() { - if err := stream.Close(); err != nil { - mod.logger.Error().Err(err) - } - } - - // NB: time out if no data is sent to free resources. - if err := stream.SetReadDeadline(newReadStreamDeadline()); err != nil { - mod.logger.Error().Err(err).Msg("setting stream read deadline") - // TODO: abort if we can't set a read deadline? - } - - data, err := io.ReadAll(stream) - if err != nil { - mod.logger.Error().Err(err).Msg("reading from stream") - closeStream() - // NB: abort this goroutine - // TODO: signal this somewhere? - return - } - defer closeStream() - - mod.handleNetworkData(data) -} - -// readFromSubscription is intended to be called in a goroutine. It continuously -// reads from the subscribed topic in preparation for handling at the network level. -// Used for handling "broadcast" messages (i.e. no specific target node). -func (mod *libp2pModule) readFromSubscription(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - default: - msg, err := mod.subscription.Next(ctx) - if err != nil { - mod.logger.Error().Err(err). - Bool("TODO", true). - Msg("reading from subscription") - } - - // NB: ignore messages from self - if msg.ReceivedFrom == mod.host.ID() { - continue - } - - mod.handleNetworkData(msg.Data) - } - } -} - -func (mod *libp2pModule) handleNetworkData(data []byte) { - appMsgData, err := mod.network.HandleNetworkData(data) - if err != nil { - mod.logger.Error().Err(err).Msg("handling network data") - return - } - - // There was no error, but we don't need to forward this to the app-specific bus. - // For example, the message has already been handled by the application. - if appMsgData == nil { - return - } - - networkMessage := messaging.PocketEnvelope{} - if err := proto.Unmarshal(appMsgData, &networkMessage); err != nil { - mod.logger.Error().Err(err). - Bool("TODO", true). - Msg("Error decoding network message") - return - } - - event := messaging.PocketEnvelope{ - Content: networkMessage.Content, - } - - mod.GetBus().PublishEventToBus(&event) -} - -// getMultiaddr returns a multiaddr constructed from the `hostname` and `port` -// in the P2P config which pas provided upon creation. -func (mod *libp2pModule) getMultiaddr() (multiaddr.Multiaddr, error) { - // TECHDEBT: as soon as we add support for multiple transports - // (i.e. not just TCP), we'll need to do something else. - return network.Libp2pMultiaddrFromServiceURL(fmt.Sprintf( - "%s:%d", mod.cfg.Hostname, mod.cfg.Port, - )) -} - -// newReadStreamDeadline returns a future deadline -// based on the read stream timeout duration. -func newReadStreamDeadline() time.Time { - return time.Now().Add(readStreamTimeoutDuration) -} diff --git a/p2p/event_handler.go b/p2p/event_handler.go index 606511ecf..7ea596eef 100644 --- a/p2p/event_handler.go +++ b/p2p/event_handler.go @@ -24,7 +24,7 @@ func (m *p2pModule) HandleEvent(event *anypb.Any) error { } oldPeerList := m.network.GetPeerstore().GetPeerList() - updatedPeerstore, err := m.peerstoreProvider.GetStakedPeerstoreAtHeight(consensusNewHeightEvent.Height) + updatedPeerstore, err := m.pstoreProvider.GetStakedPeerstoreAtHeight(consensusNewHeightEvent.Height) if err != nil { return err } diff --git a/p2p/module.go b/p2p/module.go index 24f7c5768..09c61b736 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -1,119 +1,149 @@ package p2p import ( - "log" + "context" + "fmt" + "io" + "time" + + "github.com/libp2p/go-libp2p" + pubsub "github.com/libp2p/go-libp2p-pubsub" + libp2pHost "github.com/libp2p/go-libp2p/core/host" + libp2pNetwork "github.com/libp2p/go-libp2p/core/network" + "github.com/multiformats/go-multiaddr" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "github.com/pokt-network/pocket/libp2p/network" "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/p2p/protocol" "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/current_height_provider" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" persABP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/persistence" "github.com/pokt-network/pocket/p2p/raintree" - "github.com/pokt-network/pocket/p2p/stdnetwork" - "github.com/pokt-network/pocket/p2p/transport" typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" + "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/runtime/configs/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" "github.com/pokt-network/pocket/telemetry" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" ) +const readStreamTimeoutDuration = time.Second * 10 + +// TECHDEBT: configure timeouts. Consider security exposure vs. real-world conditions). +// TECHDEBT: parameterize and expose via config. +// readStreamTimeout is the duration to wait for a read operation on a +// stream to complete, after which the stream is closed ("timed out"). var _ modules.P2PModule = &p2pModule{} type p2pModule struct { base_modules.IntegratableModule - listener typesP2P.Transport - address cryptoPocket.Address - - logger *modules.Logger - - network typesP2P.Network - - peerstoreProvider providers.PeerstoreProvider + address cryptoPocket.Address + logger *modules.Logger + cfg *configs.P2PConfig + bootstrapNodes []string currentHeightProvider providers.CurrentHeightProvider - - bootstrapNodes []string + pstoreProvider providers.PeerstoreProvider + identity libp2p.Option + listenAddrs libp2p.Option + // host represents a libp2p network node, it encapsulates a libp2p peerstore + // & connection manager. `libp2p.New` configures and starts listening + // according to options. + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) + host libp2pHost.Host + // pubsub is used for broadcast communication + // (i.e. multiple, unidentified receivers) + pubsub *pubsub.PubSub + // topic similar to pubsub but received messages are filtered by a "topic" string. + // Published messages are also given the respective topic before broadcast. + topic *pubsub.Topic + // subscription provides an interface to continuously read messages from. + subscription *pubsub.Subscription + network typesP2P.Network } func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { return new(p2pModule).Create(bus, options...) } -func (*p2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { - log.Println("Creating network module") - m := &p2pModule{} +// WithHostOption associates an existing (i.e. "started") libp2p `host.Host` +// with this module, instead of creating a new one on `#Start()`. +func WithHostOption(host libp2pHost.Host) modules.ModuleOption { + return func(m modules.InitializableModule) { + mod, ok := m.(*p2pModule) + if ok { + mod.host = host + mod.logger.Debug().Msg("using host provided via `WithHostOption`") + } + } +} - for _, option := range options { - option(m) +func (m *p2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { + logger.Global.Debug().Msg("Creating libp2p-backed network module") + *m = p2pModule{ + cfg: bus.GetRuntimeMgr().GetConfig().P2P, + logger: logger.Global.CreateLoggerForModule(modules.P2PModuleName), } + // MUST call before referencing m.bus to ensure != nil. bus.RegisterModule(m) + m.setupDependencies() - runtimeMgr := bus.GetRuntimeMgr() - cfg := runtimeMgr.GetConfig() - p2pCfg := cfg.P2P + for _, option := range options { + option(m) + } if err := m.configureBootstrapNodes(); err != nil { return nil, err } - privateKey, err := cryptoPocket.NewPrivateKey(p2pCfg.PrivateKey) + // TECHDEBT: investigate any unnecessary + // key exposure / duplication in memory + privateKey, err := cryptoPocket.NewPrivateKey(m.cfg.PrivateKey) if err != nil { - return nil, err + return nil, fmt.Errorf("parsing private key as pocket key: %w", err) } m.address = privateKey.Address() - m.setupDependencies() + libp2pPrivKey, err := cryptoPocket.NewLibP2PPrivateKey(m.cfg.PrivateKey) + if err != nil { + return nil, fmt.Errorf("parsing private key as libp2p key: %w", err) + } + m.identity = libp2p.Identity(libp2pPrivKey) - if !cfg.ClientDebugMode { - l, err := transport.CreateListener(p2pCfg) + switch m.cfg.ConnectionType { + case types.ConnectionType_TCPConnection: + addr, err := m.getMultiaddr() if err != nil { - return nil, err + return nil, fmt.Errorf("parsing multiaddr from config: %w", err) } - m.listener = l + m.listenAddrs = libp2p.ListenAddrs(addr) + case types.ConnectionType_EmptyConnection: + m.listenAddrs = libp2p.NoListenAddrs + default: + return nil, fmt.Errorf( + // TECHDEBT: rename to "transport protocol" instead. + "unsupported connection type: %s: %w", + m.cfg.ConnectionType, + err, + ) } return m, nil } - -func (m *p2pModule) setupDependencies() { - pstoreProvider, err := m.GetBus().GetModulesRegistry().GetModule(peerstore_provider.ModuleName) - if err != nil { - pstoreProvider = persABP.NewPersistencePeerstoreProvider(m.GetBus()) - } - m.peerstoreProvider = pstoreProvider.(providers.PeerstoreProvider) - - currentHeightProvider, err := m.GetBus().GetModulesRegistry().GetModule(current_height_provider.ModuleName) - if err != nil { - currentHeightProvider = m.GetBus().GetConsensusModule() - } - m.currentHeightProvider = currentHeightProvider.(providers.CurrentHeightProvider) -} - func (m *p2pModule) GetModuleName() string { return modules.P2PModuleName } -func (m *p2pModule) Start() error { - m.logger = logger.Global.CreateLoggerForModule(m.GetModuleName()) - m.logger.Info().Msg("Starting network module") - - cfg := m.GetBus().GetRuntimeMgr().GetConfig() - - // TODO: pass down logger - if cfg.P2P.UseRainTree { - m.network = raintree.NewRainTreeNetwork(m.address, m.GetBus(), m.peerstoreProvider, m.currentHeightProvider) - } else { - m.network = stdnetwork.NewNetwork(m.GetBus(), m.peerstoreProvider, m.currentHeightProvider) - } - - if cfg.ClientDebugMode { - return nil - } +func (m *p2pModule) Start() (err error) { + // TECHDEBT(#595): receive context in interface methods. + ctx := context.Background() m.GetBus(). GetTelemetryModule(). @@ -123,31 +153,54 @@ func (m *p2pModule) Start() error { telemetry.P2P_NODE_STARTED_TIMESERIES_METRIC_DESCRIPTION, ) - go func() { - for { - data, err := m.listener.ReadAll() - if err != nil { - m.logger.Error().Err(err).Msg("Error reading data from connection") - continue - } - go m.handleNetworkMessage(data) - } - }() + if err = m.startHost(); err != nil { + return fmt.Errorf("starting libp2pHost: %w", err) + } + + listenAddrLogEvent := m.logger.Info() + for i, addr := range libp2pHost.InfoFromHost(m.host).Addrs { + listenAddrLogEvent.Str(fmt.Sprintf("listen_addr_%d", i), addr.String()) + } + listenAddrLogEvent.Msg("Listening for incoming connections...") + + // TECHDEBT: use RandomSub or GossipSub once we're on more stable ground. + // IMPROVE: consider supporting multiple router types via config. + m.pubsub, err = pubsub.NewFloodSub(ctx, m.host) + if err != nil { + return fmt.Errorf("unable to create pubsub: %w", err) + } + + // Topic is used to `#Publish` messages. + m.topic, err = m.pubsub.Join(protocol.DefaultTopicStr) + if err != nil { + return fmt.Errorf("unable to join pubsub topic: %w", err) + } + + // Subscription is notified when a new message is received on the topic. + m.subscription, err = m.topic.Subscribe() + if err != nil { + return fmt.Errorf("subscribing to pubsub topic: %w", err) + } + + if err := m.startNetwork(); err != nil { + return fmt.Errorf("creating network: %w", err) + } + + // Don't handle streams or read from the subscription in client debug mode. + if !m.isClientDebugMode() { + m.host.SetStreamHandler(protocol.PoktProtocolID, m.handleStream) + go m.readFromSubscription(ctx) + } m.GetBus(). GetTelemetryModule(). GetTimeSeriesAgent(). CounterIncrement(telemetry.P2P_NODE_STARTED_TIMESERIES_METRIC_NAME) - return nil } func (m *p2pModule) Stop() error { - m.logger.Info().Msg("Stopping network module") - if err := m.listener.Close(); err != nil { - return err - } - return nil + return m.host.Close() } func (m *p2pModule) Broadcast(msg *anypb.Any) error { @@ -177,15 +230,181 @@ func (m *p2pModule) Send(addr cryptoPocket.Address, msg *anypb.Any) error { return m.network.NetworkSend(data, addr) } -// TECHDEBT: Define what the node identity is throughout the codebase +// TECHDEBT(#348): Define what the node identity is throughout the codebase func (m *p2pModule) GetAddress() (cryptoPocket.Address, error) { return m.address, nil } -func (m *p2pModule) handleNetworkMessage(networkMsgData []byte) { - appMsgData, err := m.network.HandleNetworkData(networkMsgData) +func (m *p2pModule) setupDependencies() { + m.setupCurrentHeightProvider() + m.setupPeerstoreProvider() +} + +// setupPeerstoreProvider attempts to retrieve the peerstore provider from the +// bus, if one is registered, otherwise returns a new `persistencePeerstoreProvider`. +func (m *p2pModule) setupPeerstoreProvider() { + m.logger.Debug().Msg("setupPeerstoreProvider") + pstoreProviderModule, err := m.GetBus().GetModulesRegistry().GetModule(peerstore_provider.ModuleName) + if pstoreProviderModule != nil { + m.logger.Debug().Msg("loaded persistence peerstore...") + } if err != nil { - m.logger.Error().Err(err).Msg("Error handling raw data") + m.logger.Debug().Msg("NewPersistencePeerstore...") + pstoreProviderModule = persABP.NewPersistencePeerstoreProvider(m.GetBus()) + } + + var ok bool + m.pstoreProvider, ok = pstoreProviderModule.(providers.PeerstoreProvider) + if !ok { + m.logger.Fatal().Msgf("unknown peerstore provider type: %T", pstoreProviderModule) + } +} + +// setupCurrentHeightProvider attempts to retrieve the current height provider +// from the bus registry, falls back to the consensus module if none is registered. +func (m *p2pModule) setupCurrentHeightProvider() { + currentHeightProviderModule, err := m.GetBus().GetModulesRegistry().GetModule(current_height_provider.ModuleName) + if err != nil { + currentHeightProviderModule = m.GetBus().GetConsensusModule() + } + + var ok bool + m.currentHeightProvider, ok = currentHeightProviderModule.(providers.CurrentHeightProvider) + if !ok { + m.logger.Fatal().Msgf("unexpected current height provider type: %T", currentHeightProviderModule) + } +} + +func (m *p2pModule) startNetwork() (err error) { + if m.cfg.UseRainTree { + m.network = raintree.NewRainTreeNetwork( + m.address, + m.GetBus(), + m.pstoreProvider, + m.currentHeightProvider, + ) + } else { + m.network, err = network.NewLibp2pNetwork(m.GetBus(), m.logger, m.host, m.topic) + } + return err +} + +func (m *p2pModule) startHost() (err error) { + // Return early if host has already been started (e.g. via `WithHostOption`) + if m.host != nil { + return nil + } + opts := []libp2p.Option{ + // Explicitly specify supported transport security options (noise, TLS) + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.3#DefaultSecurity) + libp2p.DefaultSecurity, + m.identity, + } + + // Disable unused libp2p relay and ping services in client debug mode. + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#DisableRelay + // and https://pkg.go.dev/github.com/libp2p/go-libp2p#Ping) + if m.isClientDebugMode() { + opts = append(opts, + libp2p.DisableRelay(), + libp2p.Ping(false), + libp2p.NoListenAddrs, + ) + } else { + opts = append(opts, m.listenAddrs) + } + + m.host, err = libp2p.New(opts...) + if err != nil { + return fmt.Errorf("unable to create libp2p host: %w", err) + } + return nil +} + +func (m *p2pModule) isClientDebugMode() bool { + return m.GetBus().GetRuntimeMgr().GetConfig().ClientDebugMode +} + +// handleStream is called each time a peer establishes a new stream with this +// module's libp2p `host.Host`. +func (m *p2pModule) handleStream(stream libp2pNetwork.Stream) { + peer, err := utils.PeerFromLibp2pStream(stream) + if err != nil { + m.logger.Error().Err(err). + Str("address", peer.GetAddress().String()). + Msg("parsing remote peer public key") + + if err = stream.Reset(); err != nil { + m.logger.Error().Err(err).Msg("resetting stream") + } + } + + if err := m.network.AddPeer(peer); err != nil { + m.logger.Error().Err(err). + Str("address", peer.GetAddress().String()). + Msg("adding remote peer to address book") + } + + go m.readStream(stream) +} + +// readStream is intended to be called in a goroutine. It continuously reads from +// the given stream for handling at the network level. Used for handling "direct" +// messages (i.e. one specific target node). +func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { + // Time out if no data is sent to free resources. + if err := stream.SetReadDeadline(newReadStreamDeadline()); err != nil { + // NB: tests using libp2p's `mocknet` rely on this not returning an error. + // `SetReadDeadline` not supported by `mocknet` streams. + m.logger.Debug().Err(err).Msg("setting stream read deadline") + } + + data, err := io.ReadAll(stream) + if err != nil { + m.logger.Error().Err(err).Msg("reading from stream") + if err := stream.Reset(); err != nil { + m.logger.Debug().Err(err).Msg("resetting stream") + } + return + } + + m.handleNetworkData(data) + + if err := stream.CloseRead(); err != nil { + m.logger.Debug().Err(err).Msg("closing read stream") + } +} + +// readFromSubscription is intended to be called in a goroutine. It continuously +// reads from the subscribed topic in preparation for handling at the network level. +// Used for handling "broadcast" messages (i.e. no specific target node). +func (m *p2pModule) readFromSubscription(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + default: + msg, err := m.subscription.Next(ctx) + if err != nil { + m.logger.Error().Err(err). + Bool("TODO", true). + Msg("reading from subscription") + } + + // Ignore messages from self + if msg.ReceivedFrom == m.host.ID() { + continue + } + + m.handleNetworkData(msg.Data) + } + } +} + +func (m *p2pModule) handleNetworkData(data []byte) { + appMsgData, err := m.network.HandleNetworkData(data) + if err != nil { + m.logger.Error().Err(err).Msg("handling network data") return } @@ -197,7 +416,9 @@ func (m *p2pModule) handleNetworkMessage(networkMsgData []byte) { networkMessage := messaging.PocketEnvelope{} if err := proto.Unmarshal(appMsgData, &networkMessage); err != nil { - m.logger.Error().Err(err).Msg("Error decoding network message") + m.logger.Error().Err(err). + Bool("TODO", true). + Msg("Error decoding network message") return } @@ -207,3 +428,19 @@ func (m *p2pModule) handleNetworkMessage(networkMsgData []byte) { m.GetBus().PublishEventToBus(&event) } + +// getMultiaddr returns a multiaddr constructed from the `hostname` and `port` +// in the P2P config which pas provided upon creation. +func (m *p2pModule) getMultiaddr() (multiaddr.Multiaddr, error) { + // TECHDEBT: as soon as we add support for multiple transports + // (i.e. not just TCP), we'll need to do something else. + return utils.Libp2pMultiaddrFromServiceURL(fmt.Sprintf( + "%s:%d", m.cfg.Hostname, m.cfg.Port, + )) +} + +// newReadStreamDeadline returns a future deadline +// based on the read stream timeout duration. +func newReadStreamDeadline() time.Time { + return time.Now().Add(readStreamTimeoutDuration) +} From ec98eadd1de088dd6e898208f4a814e452c13b06 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 20 Mar 2023 14:24:30 +0100 Subject: [PATCH 05/82] refactor: consolidate libp2p and stdnetwork networks --- libp2p/network/network.go | 235 -------------------------------------- p2p/module.go | 8 +- p2p/stdnetwork/network.go | 27 +++-- p2p/utils/host.go | 27 +++++ 4 files changed, 50 insertions(+), 247 deletions(-) delete mode 100644 libp2p/network/network.go diff --git a/libp2p/network/network.go b/libp2p/network/network.go deleted file mode 100644 index 550e13bbf..000000000 --- a/libp2p/network/network.go +++ /dev/null @@ -1,235 +0,0 @@ -package network - -import ( - "context" - "fmt" - "time" - - pubsub "github.com/libp2p/go-libp2p-pubsub" - libp2pHost "github.com/libp2p/go-libp2p/core/host" - - "github.com/pokt-network/pocket/p2p/protocol" - "github.com/pokt-network/pocket/p2p/providers" - "github.com/pokt-network/pocket/p2p/providers/current_height_provider" - "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" - persABP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/persistence" - typesP2P "github.com/pokt-network/pocket/p2p/types" - "github.com/pokt-network/pocket/shared/crypto" - "github.com/pokt-network/pocket/shared/modules" - "github.com/pokt-network/pocket/shared/modules/base_modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" -) - -const ( - week = time.Hour * 24 * 7 - // TECHDEBT: consider more carefully and parameterize. - defaultPeerTTL = 2 * week -) - -var _ typesP2P.Network = &libp2pNetwork{} - -type libp2pNetwork struct { - base_modules.IntegratableModule - - logger *modules.Logger - host libp2pHost.Host - topic *pubsub.Topic - pstore sharedP2P.Peerstore -} - -func NewLibp2pNetwork( - bus modules.Bus, - logger *modules.Logger, - host libp2pHost.Host, - topic *pubsub.Topic, -) (typesP2P.Network, error) { - p2pNet := &libp2pNetwork{ - logger: logger, - host: host, - topic: topic, - } - - p2pNet.SetBus(bus) - err := p2pNet.setup() - return p2pNet, err -} - -// NetworkBroadcast uses the configured pubsub router to broadcast data to peers. -func (p2pNet *libp2pNetwork) NetworkBroadcast(data []byte) error { - // IMPROVE: receive context in interface methods. - ctx := context.Background() - - return p2pNet.topic.Publish(ctx, data) -} - -// NetworkSend connects sends data directly to the specified peer. -func (p2pNet *libp2pNetwork) NetworkSend(data []byte, poktAddr crypto.Address) error { - // IMPROVE: receive context in interface methods. - ctx := context.Background() - - selfPoktAddr, err := p2pNet.GetBus().GetP2PModule().GetAddress() - if err != nil { - return fmt.Errorf( - "sending to poktAddr: %s: %w", - poktAddr, - err, - ) - } - - // Don't send to self. - if selfPoktAddr.Equals(poktAddr) { - return nil - } - - peer := p2pNet.pstore.GetPeer(poktAddr) - if peer == nil { - // This should not happen. - return fmt.Errorf( - "peer not found in address book, pokt address: %s: %w", - poktAddr, - err, - ) - } - - peerAddrInfo, err := Libp2pAddrInfoFromPeer(peer) - if err != nil { - return fmt.Errorf("parsing peer multiaddr: %w", err) - } - - stream, err := p2pNet.host.NewStream(ctx, peerAddrInfo.ID, protocol.PoktProtocolID) - if err != nil { - return fmt.Errorf( - "opening peer stream, pokt address: %s: %w", - poktAddr, - err, - ) - } - - if _, err := stream.Write(data); err != nil { - return fmt.Errorf( - "writing to stream, peer address: %s: %w", - poktAddr, - err, - ) - } - defer func() { - // Close the stream so that peer receives EOF. - if err := stream.Close(); err != nil { - p2pNet.logger.Error().Err(err).Msg(fmt.Sprintf( - "closing peer stream, pokt address: %s", poktAddr, - )) - } - }() - return nil -} - -// This function was added to specifically support the RainTree implementation. -// Handles the raw data received from the network and returns the data to be processed -// by the application layer. -func (p2pNet *libp2pNetwork) HandleNetworkData(data []byte) ([]byte, error) { - return data, nil -} - -func (p2pNet *libp2pNetwork) GetPeerstore() sharedP2P.Peerstore { - return p2pNet.pstore -} - -func (p2pNet *libp2pNetwork) AddPeer(peer sharedP2P.Peer) error { - if err := p2pNet.pstore.AddPeer(peer); err != nil { - return fmt.Errorf( - "adding peer, pokt address %s: %w", - peer.GetAddress(), - err, - ) - } - - pubKey, err := Libp2pPublicKeyFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer public key, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - libp2pPeer, err := Libp2pAddrInfoFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer info, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - - p2pNet.host.Peerstore().AddAddrs(libp2pPeer.ID, libp2pPeer.Addrs, defaultPeerTTL) - if err := p2pNet.host.Peerstore().AddPubKey(libp2pPeer.ID, pubKey); err != nil { - return fmt.Errorf( - "adding peer public key, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - return nil -} - -func (p2pNet *libp2pNetwork) RemovePeer(peer sharedP2P.Peer) error { - if err := p2pNet.pstore.RemovePeer(peer.GetAddress()); err != nil { - return fmt.Errorf( - "removing peer, pokt address %s: %w", - peer.GetAddress(), - err, - ) - } - - libp2pPeer, err := Libp2pAddrInfoFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer info, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - - p2pNet.host.Peerstore().RemovePeer(libp2pPeer.ID) - return nil -} - -func (p2pNet *libp2pNetwork) Close() error { - return p2pNet.host.Close() -} - -// setupPeerstoreProvider attempts to retrieve the peerstore provider from the -// bus, if one is registered, otherwise returns a new `persistencePeerstoreProvider`. -func (p2pNet *libp2pNetwork) setupPeerstoreProvider() providers.PeerstoreProvider { - pstoreProviderModule, err := p2pNet.GetBus().GetModulesRegistry().GetModule(peerstore_provider.ModuleName) - if err != nil { - pstoreProviderModule = persABP.NewPersistencePeerstoreProvider(p2pNet.GetBus()) - } - return pstoreProviderModule.(providers.PeerstoreProvider) -} - -// setupCurrentHeightProvider attempts to retrieve the current height provider -// from the bus registry, falls back to the consensus module if none is registered. -func (p2pNet *libp2pNetwork) setupCurrentHeightProvider() providers.CurrentHeightProvider { - currentHeightProviderModule, err := p2pNet.GetBus().GetModulesRegistry().GetModule(current_height_provider.ModuleName) - if err != nil { - currentHeightProviderModule = p2pNet.GetBus().GetConsensusModule() - } - return currentHeightProviderModule.(providers.CurrentHeightProvider) -} - -// setup initializes p2pNet.pstore using the PeerstoreProvider -// and CurrentHeightProvider registered on the bus, if preseent. -func (p2pNet *libp2pNetwork) setup() (err error) { - peerstoreProvider := p2pNet.setupPeerstoreProvider() - currentHeightProvider := p2pNet.setupCurrentHeightProvider() - - p2pNet.pstore, err = peerstoreProvider.GetStakedPeerstoreAtHeight(currentHeightProvider.CurrentHeight()) - if err != nil { - return fmt.Errorf("getting staked peerstore: %w", err) - } - - if err := p2pNet.setupHost(); err != nil { - return err - } - return nil -} diff --git a/p2p/module.go b/p2p/module.go index 09c61b736..f7f8f90d1 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -14,7 +14,6 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/pokt-network/pocket/libp2p/network" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/protocol" "github.com/pokt-network/pocket/p2p/providers" @@ -22,6 +21,7 @@ import ( "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" persABP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/persistence" "github.com/pokt-network/pocket/p2p/raintree" + "github.com/pokt-network/pocket/p2p/stdnetwork" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/runtime/configs" @@ -284,7 +284,11 @@ func (m *p2pModule) startNetwork() (err error) { m.currentHeightProvider, ) } else { - m.network, err = network.NewLibp2pNetwork(m.GetBus(), m.logger, m.host, m.topic) + m.network, err = stdnetwork.NewNetwork( + m.host, + m.pstoreProvider, + m.currentHeightProvider, + ) } return err } diff --git a/p2p/stdnetwork/network.go b/p2p/stdnetwork/network.go index cfa76677a..7bc79dfb9 100644 --- a/p2p/stdnetwork/network.go +++ b/p2p/stdnetwork/network.go @@ -4,13 +4,16 @@ package stdnetwork import ( "fmt" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" + + libp2pHost "github.com/libp2p/go-libp2p/core/host" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/providers" typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" + sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) var ( @@ -19,45 +22,49 @@ var ( ) type network struct { + host libp2pHost.Host pstore sharedP2P.Peerstore logger *modules.Logger } -func NewNetwork(bus modules.Bus, pstoreProvider providers.PeerstoreProvider, currentHeightProvider providers.CurrentHeightProvider) (n typesP2P.Network) { +func NewNetwork(host libp2pHost.Host, pstoreProvider providers.PeerstoreProvider, currentHeightProvider providers.CurrentHeightProvider) (typesP2P.Network, error) { networkLogger := logger.Global.CreateLoggerForModule("network") networkLogger.Info().Msg("Initializing stdnetwork") pstore, err := pstoreProvider.GetStakedPeerstoreAtHeight(currentHeightProvider.CurrentHeight()) if err != nil { - networkLogger.Fatal().Err(err).Msg("Error getting peerstore") + return nil, err } return &network{ + host: host, logger: networkLogger, pstore: pstore, - } + }, nil } -// TODO(olshansky): How do we avoid self-broadcasts given that `AddrBook` may contain self in the current p2p implementation? func (n *network) NetworkBroadcast(data []byte) error { for _, peer := range n.pstore.GetPeerList() { - if _, err := peer.GetStream().Write(data); err != nil { - n.logger.Error().Err(err).Msg("Error writing to one of the peers during broadcast") + if err := utils.Libp2pSendToPeer(n.host, data, peer); err != nil { + n.logger.Error(). + Err(err). + Bool("TODO", true). + Str("pokt address", peer.GetAddress().String()). + Msg("broadcasting to peer") continue } } return nil } -func (n *network) NetworkSend(data []byte, address cryptoPocket.Address) error { +func (n *network) NetworkSend(data []byte, address cryptoPocket.Address) (err error) { peer := n.pstore.GetPeer(address) if peer == nil { return fmt.Errorf("peer with address %s not in peerstore", address) } - if _, err := peer.GetStream().Write(data); err != nil { - n.logger.Error().Err(err).Msg("Error writing to peer during send") + if err := utils.Libp2pSendToPeer(n.host, data, peer); err != nil { return err } return nil diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 5b456b8dd..b851258de 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -1,11 +1,14 @@ package utils import ( + "context" "fmt" "time" libp2pHost "github.com/libp2p/go-libp2p/core/host" + "go.uber.org/multierr" + "github.com/pokt-network/pocket/p2p/protocol" "github.com/pokt-network/pocket/shared/p2p" ) @@ -50,3 +53,27 @@ func PopulateLibp2pHost(host libp2pHost.Host, pstore p2p.Peerstore) error { } return nil } + +func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) error { + // TECHDEBT(#595): add ctx to interface methods and propagate down. + ctx := context.Background() + + peerInfo, err := Libp2pAddrInfoFromPeer(peer) + if err != nil { + return err + } + + stream, err := host.NewStream(ctx, peerInfo.ID, protocol.PoktProtocolID) + if err != nil { + return fmt.Errorf("opening stream: %w", err) + } + + if _, err = stream.Write(data); err != nil { + return multierr.Append( + fmt.Errorf("writing to stream: %w", err), + stream.Reset(), + ) + } + + return stream.CloseWrite() +} From 6f159c0956b080eba83374dd51b51f7bf7b04da7 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 23 Mar 2023 15:43:40 +0100 Subject: [PATCH 06/82] refactor: consolidate P2P changelogs --- libp2p/docs/CHANGELOG.md | 8 -------- p2p/CHANGELOG.md | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libp2p/docs/CHANGELOG.md b/libp2p/docs/CHANGELOG.md index 126b23759..e1a4738bc 100644 --- a/libp2p/docs/CHANGELOG.md +++ b/libp2p/docs/CHANGELOG.md @@ -5,14 +5,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.7] - 2023-03-23 - -- Wrap IPv6 address in square brackets as per RFC3986 §3.2.2 - -## [0.0.0.6] - 2023-03-22 - -- Improve URL validation and error handling in Libp2pMultiaddrFromServiceURL function - ## [0.0.0.5] - 2023-03-21 - Refactored libp2p module to use new P2P interfaces diff --git a/p2p/CHANGELOG.md b/p2p/CHANGELOG.md index a86d6febf..1b6281d85 100644 --- a/p2p/CHANGELOG.md +++ b/p2p/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.37] - 2023-03-23 + +- Wrap IPv6 address in square brackets as per RFC3986 §3.2.2 + +## [0.0.0.36] - 2023-03-22 + +- Improve URL validation and error handling in Libp2pMultiaddrFromServiceURL function + ## [0.0.0.35] - 2023-03-21 - Add log for `StateMachineTransitionEvent` From e5c43f139090662363d6070dd948c46f5b26f108 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 23 Mar 2023 15:42:22 +0100 Subject: [PATCH 07/82] chore: delete remains of libp2p module --- libp2p/docs/CHANGELOG.md | 39 ------- libp2p/network/network_test.go | 140 ------------------------- libp2p/network/utils_test.go | 186 --------------------------------- libp2p/transport/transport.go | 42 -------- libp2p/types/mocks/mocks.go | 3 - 5 files changed, 410 deletions(-) delete mode 100644 libp2p/docs/CHANGELOG.md delete mode 100644 libp2p/network/network_test.go delete mode 100644 libp2p/network/utils_test.go delete mode 100644 libp2p/transport/transport.go delete mode 100644 libp2p/types/mocks/mocks.go diff --git a/libp2p/docs/CHANGELOG.md b/libp2p/docs/CHANGELOG.md deleted file mode 100644 index e1a4738bc..000000000 --- a/libp2p/docs/CHANGELOG.md +++ /dev/null @@ -1,39 +0,0 @@ -All notable changes to this module will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -## [0.0.0.5] - 2023-03-21 - -- Refactored libp2p module to use new P2P interfaces - -## [0.0.0.4] - 2023-03-18 - -- Corrected libp2p module tests -- Improve libp2p code quality in response to post-merge review feedback - -## [0.0.0.3] - 2023-03-15 - -- Added mockdns as a test dependency -- Mocked DNS resolution in url_conversion_test.go -- Added regression tests to url_conversion_test.go for single- and multi-record DNS responses - -## [0.0.0.2] - 2023-03-03 - -- Added a new `modules.P2PModule` implementation to the `libp2p` module directory - -## [0.0.0.1] - 2023-03-03 - -- Added a new `typesP2P.Network` implementation to the `libp2p` module directory -- Added `PoktProtocolID` for use within the libp2p module or by public API consumers - -## [0.0.0.0] - 2023-02-23 - -- prepare pocket repo new libp2p module -- add pocket / libp2p identity helpers -- add url <--> multiaddr conversion helpers for use with libp2p (see: https://github.com/multiformats/go-multiaddr) -- add `Multiaddr` field to `typesP2P.NetworkPeer` - - diff --git a/libp2p/network/network_test.go b/libp2p/network/network_test.go deleted file mode 100644 index 8306983d8..000000000 --- a/libp2p/network/network_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package network - -import ( - "context" - "testing" - - "github.com/libp2p/go-libp2p" - pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/stretchr/testify/require" - - "github.com/pokt-network/pocket/logger" - "github.com/pokt-network/pocket/p2p/types" - "github.com/pokt-network/pocket/shared/crypto" -) - -// https://www.rfc-editor.org/rfc/rfc3986#section-3.2.2 -const testIP6ServiceURL = "[2a00:1450:4005:802::2004]:8080" - -func TestLibp2pNetwork_AddPeer(t *testing.T) { - p2pNet := newTestLibp2pNetwork(t) - libp2pPStore := p2pNet.host.Peerstore() - - // NB: assert initial state - require.Equal(t, 1, p2pNet.pstore.Size()) - - existingPeer := p2pNet.pstore.GetPeerList()[0] - require.NotNil(t, existingPeer) - - existingPeerInfo, err := Libp2pAddrInfoFromPeer(existingPeer) - require.NoError(t, err) - - existingPeerstoreAddrs := libp2pPStore.Addrs(existingPeerInfo.ID) - require.Len(t, existingPeerstoreAddrs, 1) - - existingPeerMultiaddr, err := Libp2pMultiaddrFromServiceURL(existingPeer.GetServiceURL()) - require.NoError(t, err) - require.Equal(t, existingPeerstoreAddrs[0].String(), existingPeerMultiaddr.String()) - - newPublicKey, err := crypto.GeneratePublicKey() - newPoktAddr := newPublicKey.Address() - require.NoError(t, err) - - newPeer := &types.NetworkPeer{ - PublicKey: newPublicKey, - Address: newPoktAddr, - ServiceURL: testIP6ServiceURL, - } - newPeerInfo, err := Libp2pAddrInfoFromPeer(newPeer) - require.NoError(t, err) - newPeerMultiaddr := newPeerInfo.Addrs[0] - - // NB: add to address book - err = p2pNet.AddPeer(newPeer) - require.NoError(t, err) - - require.Len(t, p2pNet.pstore, 2) - require.Equal(t, p2pNet.pstore.GetPeer(existingPeer.GetAddress()), existingPeer) - require.Equal(t, p2pNet.pstore.GetPeer(newPeer.Address), newPeer) - - existingPeerstoreAddrs = libp2pPStore.Addrs(existingPeerInfo.ID) - newPeerstoreAddrs := libp2pPStore.Addrs(newPeerInfo.ID) - - require.Len(t, existingPeerstoreAddrs, 1) - require.Len(t, newPeerstoreAddrs, 1) - require.Equal(t, newPeerstoreAddrs[0].String(), newPeerMultiaddr.String()) -} - -func TestLibp2pNetwork_RemovePeer(t *testing.T) { - p2pNet := newTestLibp2pNetwork(t) - peerstore := p2pNet.host.Peerstore() - - // NB: assert initial state - require.Len(t, p2pNet.pstore, 1) - - existingPeer := p2pNet.pstore.GetPeerList()[0] - require.NotNil(t, existingPeer) - - existingPeerInfo, err := Libp2pAddrInfoFromPeer(existingPeer) - require.NoError(t, err) - - existingPeerstoreAddrs := peerstore.Addrs(existingPeerInfo.ID) - require.Len(t, existingPeerstoreAddrs, 1) - - existingPeerMultiaddr, err := Libp2pMultiaddrFromServiceURL(existingPeer.GetServiceURL()) - require.NoError(t, err) - require.Equal(t, existingPeerstoreAddrs[0].String(), existingPeerMultiaddr.String()) - - err = p2pNet.RemovePeer(existingPeer) - require.NoError(t, err) - - require.Len(t, p2pNet.pstore, 0) - - // NB: peerstore implementations seem to only remove peer keys and - // metadata but not the embedded AddrBook entry. - // (see: https://github.com/libp2p/go-libp2p/blob/v0.25.1/p2p/host/peerstore/pstoremem/peerstore.go#L108) - // (see: https://github.com/libp2p/go-libp2p/blob/v0.25.1/p2p/host/peerstore/pstoreds/peerstore.go#L187) - - existingPeerstoreAddrs = peerstore.Addrs(existingPeerInfo.ID) - require.Len(t, existingPeerstoreAddrs, 1) -} - -func newTestLibp2pNetwork(t *testing.T) *libp2pNetwork { - ctx := context.Background() - - // INCOMPLETE (SOON OBSOLETE): Only testing pocket address book <-> libp2p - // peerstore integration. No need to mock an entire network, just a - // starting pocket address book. - runtimeConfigs := createMockRuntimeMgrs(t, 1) - busMock := createMockBus(t, runtimeConfigs[0], 1) - consensusMock := prepareConsensusMock(t, busMock) - - prepareBusMock(busMock, consensusMock) - - networkLogger := logger.Global.CreateLoggerForModule("test_module") - - // NB: will bind to a random, available port on the loopback interface - // for the duration of this test. - host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) - require.NoError(t, err) - defer host.Close() - - floodSub, err := pubsub.NewFloodSub(ctx, host) - require.NoError(t, err) - - topic, err := floodSub.Join("test_protocol") - require.NoError(t, err) - - p2pNetwork, err := NewLibp2pNetwork( - busMock, - networkLogger, - host, - topic, - ) - require.NoError(t, err) - - libp2pNet, ok := p2pNetwork.(*libp2pNetwork) - require.Truef(t, ok, "unexpected p2pNetwork type: %T", p2pNetwork) - - return libp2pNet -} diff --git a/libp2p/network/utils_test.go b/libp2p/network/utils_test.go deleted file mode 100644 index 68676077e..000000000 --- a/libp2p/network/utils_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package network - -import ( - "fmt" - "sort" - "testing" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" - - "github.com/pokt-network/pocket/p2p/providers/current_height_provider" - "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" - typesP2P "github.com/pokt-network/pocket/p2p/types" - mock_typesP2P "github.com/pokt-network/pocket/p2p/types/mocks" - "github.com/pokt-network/pocket/runtime" - "github.com/pokt-network/pocket/runtime/configs" - configTypes "github.com/pokt-network/pocket/runtime/configs/types" - "github.com/pokt-network/pocket/runtime/genesis" - coreTypes "github.com/pokt-network/pocket/shared/core/types" - cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - "github.com/pokt-network/pocket/shared/modules" - "github.com/pokt-network/pocket/shared/modules/mocks" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" -) - -const ( - maxNumKeys = 42 - genesisConfigSeedStart = 42 -) - -var ( - keys []cryptoPocket.PrivateKey - serviceUrlFormat = "node%d.consensus:42069" - testServiceUrlFormat = "10.0.0.%d:42069" -) - -func init() { - keys = generateKeys(nil, maxNumKeys) -} - -func generateKeys(_ *testing.T, numValidators int) []cryptoPocket.PrivateKey { - keys := make([]cryptoPocket.PrivateKey, numValidators) - - for i := range keys { - seedInt := genesisConfigSeedStart + i - keys[i] = cryptoPocket.GetPrivKeySeed(seedInt) - } - sort.Slice(keys, func(i, j int) bool { - return keys[i].Address().String() < keys[j].Address().String() - }) - return keys -} - -// CLEANUP: This could (should?) be a codebase-wide shared test helper -func validatorId(i int) string { - return fmt.Sprintf(serviceUrlFormat, i) -} - -// createMockRuntimeMgrs creates `numValidators` instances of mocked `RuntimeMgr` that are essentially -// representing the runtime environments of the validators that we will use in our tests -func createMockRuntimeMgrs(t *testing.T, numValidators int) []modules.RuntimeMgr { - ctrl := gomock.NewController(t) - mockRuntimeMgrs := make([]modules.RuntimeMgr, numValidators) - valKeys := make([]cryptoPocket.PrivateKey, numValidators) - copy(valKeys, keys[:numValidators]) - mockGenesisState := createMockGenesisState(valKeys) - for i := range mockRuntimeMgrs { - cfg := &configs.Config{ - RootDirectory: "", - PrivateKey: valKeys[i].String(), - UseLibP2P: true, - P2P: &configs.P2PConfig{ - PrivateKey: valKeys[i].String(), - Port: 42069, - UseRainTree: false, - ConnectionType: configTypes.ConnectionType_EmptyConnection, - }, - } - - mockRuntimeMgr := mock_modules.NewMockRuntimeMgr(ctrl) - mockRuntimeMgr.EXPECT().GetConfig().Return(cfg).AnyTimes() - mockRuntimeMgr.EXPECT().GetGenesis().Return(mockGenesisState).AnyTimes() - mockRuntimeMgrs[i] = mockRuntimeMgr - } - return mockRuntimeMgrs -} - -func newTestPeerstoreProvider(t *testing.T, ctrl *gomock.Controller, numPeers int) peerstore_provider.PeerstoreProvider { - pstore := make(sharedP2P.PeerAddrMap) - // No expectations, transport is not used in current network test. - transport := mock_typesP2P.NewMockTransport(ctrl) - publicKey, err := cryptoPocket.GeneratePublicKey() - require.NoError(t, err) - - for i := 0; i < numPeers; i++ { - err = pstore.AddPeer(&typesP2P.NetworkPeer{ - Transport: transport, - PublicKey: publicKey, - Address: publicKey.Address(), - ServiceURL: fmt.Sprintf(testServiceUrlFormat, i), - }) - require.NoError(t, err) - } - - pstoreProviderMock := mock_typesP2P.NewMockPeerstoreProvider(ctrl) - pstoreProviderMock.EXPECT().GetStakedPeerstoreAtHeight(gomock.Any()).Return(pstore, nil) - return pstoreProviderMock -} - -func createMockBus(t *testing.T, runtimeMgr modules.RuntimeMgr, numPeers int) *mock_modules.MockBus { - ctrl := gomock.NewController(t) - mockBus := mock_modules.NewMockBus(ctrl) - mockBus.EXPECT().GetRuntimeMgr().Return(runtimeMgr).AnyTimes() - mockBus.EXPECT().GetPersistenceModule().Return(nil).AnyTimes() - mockBus.EXPECT().RegisterModule(gomock.Any()).DoAndReturn(func(m modules.Module) { - m.SetBus(mockBus) - }).AnyTimes() - - pstoreProviderMock := newTestPeerstoreProvider(t, ctrl, numPeers) - - mockModulesRegistry := mock_modules.NewMockModulesRegistry(ctrl) - mockModulesRegistry.EXPECT().GetModule(peerstore_provider.ModuleName).Return(pstoreProviderMock.(modules.Module), nil).AnyTimes() - mockModulesRegistry.EXPECT().GetModule(current_height_provider.ModuleName).Return(nil, runtime.ErrModuleNotRegistered(current_height_provider.ModuleName)).AnyTimes() - mockBus.EXPECT().GetModulesRegistry().Return(mockModulesRegistry).AnyTimes() - mockBus.EXPECT().PublishEventToBus(gomock.Any()).AnyTimes() - return mockBus -} - -// createMockGenesisState configures and returns a mocked GenesisState -func createMockGenesisState(valKeys []cryptoPocket.PrivateKey) *genesis.GenesisState { - genesisState := new(genesis.GenesisState) - validators := make([]*coreTypes.Actor, len(valKeys)) - for i, valKey := range valKeys { - addr := valKey.Address().String() - mockActor := &coreTypes.Actor{ - ActorType: coreTypes.ActorType_ACTOR_TYPE_VAL, - Address: addr, - PublicKey: valKey.PublicKey().String(), - ServiceUrl: validatorId(i + 1), - StakedAmount: "1000000000000000", - PausedHeight: int64(0), - UnstakingHeight: int64(0), - Output: addr, - } - validators[i] = mockActor - } - genesisState.Validators = validators - - return genesisState -} - -// Bus Mock - needed to return the appropriate modules when accessed -func prepareBusMock(busMock *mock_modules.MockBus, - consensusMock *mock_modules.MockConsensusModule, -) { - busMock.EXPECT().GetConsensusModule().Return(consensusMock).AnyTimes() -} - -// Consensus mock - only needed for validatorMap access -func prepareConsensusMock(t *testing.T, busMock *mock_modules.MockBus) *mock_modules.MockConsensusModule { - ctrl := gomock.NewController(t) - consensusMock := mock_modules.NewMockConsensusModule(ctrl) - consensusMock.EXPECT().CurrentHeight().Return(uint64(1)).AnyTimes() - - consensusMock.EXPECT().GetBus().Return(busMock).AnyTimes() - consensusMock.EXPECT().SetBus(busMock).AnyTimes() - consensusMock.EXPECT().GetModuleName().Return(modules.ConsensusModuleName).AnyTimes() - busMock.RegisterModule(consensusMock) - - return consensusMock -} - -func MockBus(ctrl *gomock.Controller) *mock_modules.MockBus { - consensusMock := mock_modules.NewMockConsensusModule(ctrl) - consensusMock.EXPECT().CurrentHeight().Return(uint64(0)).AnyTimes() - - runtimeMgrMock := mock_modules.NewMockRuntimeMgr(ctrl) - runtimeMgrMock.EXPECT().GetConfig().Return(configs.NewDefaultConfig()).AnyTimes() - - busMock := mock_modules.NewMockBus(ctrl) - busMock.EXPECT().GetPersistenceModule().Return(nil).AnyTimes() - busMock.EXPECT().GetConsensusModule().Return(consensusMock).AnyTimes() - busMock.EXPECT().GetRuntimeMgr().Return(runtimeMgrMock).AnyTimes() - - return busMock -} diff --git a/libp2p/transport/transport.go b/libp2p/transport/transport.go deleted file mode 100644 index 2d8fc4d28..000000000 --- a/libp2p/transport/transport.go +++ /dev/null @@ -1,42 +0,0 @@ -package transport - -import ( - "io" - - "github.com/libp2p/go-libp2p/core/network" - - "github.com/pokt-network/pocket/p2p/types" -) - -var _ types.Transport = &libP2PTransport{} - -type libP2PTransport struct { - stream network.Stream -} - -func NewLibP2PTransport(stream network.Stream) types.Transport { - return &libP2PTransport{ - stream: stream, - } -} - -func (transport *libP2PTransport) IsListener() bool { - // NB: libp2p streams are bi-directional - return true -} - -func (transport *libP2PTransport) ReadAll() ([]byte, error) { - return io.ReadAll(transport) -} - -func (transport *libP2PTransport) Read(buf []byte) (int, error) { - return transport.stream.Read(buf) -} - -func (transport *libP2PTransport) Write(data []byte) (int, error) { - return transport.stream.Write(data) -} - -func (transport *libP2PTransport) Close() error { - return transport.stream.Close() -} diff --git a/libp2p/types/mocks/mocks.go b/libp2p/types/mocks/mocks.go deleted file mode 100644 index 4fe74c6ec..000000000 --- a/libp2p/types/mocks/mocks.go +++ /dev/null @@ -1,3 +0,0 @@ -package mock_types - -// This file is in place to declare the package for dynamically generated mocks From 00c7f2bff6835d6b2f810b8cc90f2ea79b3b73fc Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 13:57:25 +0100 Subject: [PATCH 08/82] chore: remove libp2p module dir from mockgen make target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e79f26456..0a4c8ce4d 100644 --- a/Makefile +++ b/Makefile @@ -231,7 +231,7 @@ mockgen: clean_mocks ## Use `mockgen` to generate mocks used for testing purpose go generate ./${modules_dir} echo "Mocks generated in ${modules_dir}/mocks" - $(eval DIRS = p2p libp2p persistence) + $(eval DIRS = p2p persistence) for dir in $(DIRS); do \ echo "Processing $$dir mocks..."; \ find $$dir/types/mocks -type f ! -name "mocks.go" -exec rm {} \;; \ From 6d206810c40f5967e78950e66bf9dedb531e8edf Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 13:53:35 +0100 Subject: [PATCH 09/82] chore: cleanup debug CLI, post libp2p module deletion --- app/client/cli/debug.go | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index e1b68e4e6..257445c7b 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" "google.golang.org/protobuf/types/known/anypb" - "github.com/pokt-network/pocket/libp2p" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p" "github.com/pokt-network/pocket/p2p/providers/current_height_provider" @@ -103,13 +102,17 @@ func NewDebugCommand() *cobra.Command { setValueInCLIContext(cmd, busCLICtxKey, bus) - // TECHDEBT: simplify after P2P module consolidation. - var err error - p2pMod, err = getP2PModule(runtimeMgr) + mod, err := p2p.Create(bus) if err != nil { logger.Global.Fatal().Err(err).Msg("Failed to create p2p module") } + var ok bool + p2pMod, ok = mod.(modules.P2PModule) + if !ok { + logger.Global.Fatal().Msgf("unexpected P2P module type: %T", mod) + } + if err := p2pMod.Start(); err != nil { logger.Global.Fatal().Err(err).Msg("Failed to start p2p module") } @@ -285,19 +288,3 @@ func fetchPeerstore(cmd *cobra.Command) (sharedP2P.Peerstore, error) { } return pstore, err } - -func getP2PModule(runtimeMgr *runtime.Manager) (p2pModule modules.P2PModule, err error) { - bus := runtimeMgr.GetBus() - - var mod modules.Module - if runtimeMgr.GetConfig().UseLibP2P { - mod, err = libp2p.Create(bus) - } else { - mod, err = p2p.Create(bus) - } - if err != nil { - return nil, err - } - - return mod.(modules.P2PModule), nil -} From b16d2d04e425c27af39253e061f23cf1724a98fe Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 13:55:06 +0100 Subject: [PATCH 10/82] chore: cleanup `Node`, post libp2p module deletion --- shared/node.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/shared/node.go b/shared/node.go index 67b39171c..66c533aeb 100644 --- a/shared/node.go +++ b/shared/node.go @@ -2,7 +2,6 @@ package shared import ( "github.com/pokt-network/pocket/consensus" - "github.com/pokt-network/pocket/libp2p" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p" "github.com/pokt-network/pocket/persistence" @@ -34,13 +33,6 @@ func CreateNode(bus modules.Bus, options ...modules.ModuleOption) (modules.Modul } func (m *Node) Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { - // TECHDEBT: simplify after P2P module consolidation. - useLibP2P := bus.GetRuntimeMgr().GetConfig().UseLibP2P - p2pCreate := p2p.Create - if useLibP2P { - p2pCreate = libp2p.Create - } - for _, mod := range []func(modules.Bus, ...modules.ModuleOption) (modules.Module, error){ state_machine.Create, persistence.Create, @@ -49,7 +41,7 @@ func (m *Node) Create(bus modules.Bus, options ...modules.ModuleOption) (modules telemetry.Create, logger.Create, rpc.Create, - p2pCreate, + p2p.Create, } { if _, err := mod(bus); err != nil { return nil, err From c5c767e33ae4b9b45ba81ecc675458fe9380f67c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 20 Mar 2023 14:28:28 +0100 Subject: [PATCH 11/82] refactor: remove `UseLibp2p` config field --- runtime/configs/config.go | 2 -- runtime/defaults/defaults.go | 1 - runtime/manager_test.go | 1 - 3 files changed, 4 deletions(-) diff --git a/runtime/configs/config.go b/runtime/configs/config.go index 018a3ac67..802ce0b05 100644 --- a/runtime/configs/config.go +++ b/runtime/configs/config.go @@ -6,7 +6,6 @@ type Config struct { RootDirectory string `json:"root_directory"` PrivateKey string `json:"private_key"` // INVESTIGATE(#150): better architecture for key management (keybase, keyfiles, etc.) ClientDebugMode bool `json:"client_debug_mode"` - UseLibP2P bool `json:"use_lib_p2p"` // Determines if `root/libp2p` or `root/p2p` should be used as the p2p module Consensus *ConsensusConfig `json:"consensus"` Utility *UtilityConfig `json:"utility"` @@ -20,7 +19,6 @@ type Config struct { func NewDefaultConfig(options ...func(*Config)) *Config { cfg := &Config{ RootDirectory: "/go/src/github.com/pocket-network", - UseLibP2P: defaults.DefaultUseLibp2p, Consensus: &ConsensusConfig{ MaxMempoolBytes: defaults.DefaultConsensusMaxMempoolBytes, PacemakerConfig: &PacemakerConfig{ diff --git a/runtime/defaults/defaults.go b/runtime/defaults/defaults.go index 131d3b6eb..59d7e990d 100644 --- a/runtime/defaults/defaults.go +++ b/runtime/defaults/defaults.go @@ -18,7 +18,6 @@ const ( var ( DefaultRemoteCLIURL = fmt.Sprintf("http://%s:%s", DefaultRPCHost, DefaultRPCPort) - DefaultUseLibp2p = false // consensus DefaultConsensusMaxMempoolBytes = uint64(500000000) diff --git a/runtime/manager_test.go b/runtime/manager_test.go index b9f3873f5..a6fd09b0c 100644 --- a/runtime/manager_test.go +++ b/runtime/manager_test.go @@ -4180,7 +4180,6 @@ func TestNewManagerFromReaders(t *testing.T) { config: &configs.Config{ RootDirectory: "/go/src/github.com/pocket-network", PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", - UseLibP2P: false, Consensus: &configs.ConsensusConfig{ PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", MaxMempoolBytes: 500000000, From bd950afb78b02ec62c7df16e2e572fd34dac9ff4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 22:20:57 +0100 Subject: [PATCH 12/82] chore: add libp2p mock generator for `host.Host` --- p2p/types/libp2p_mocks.go | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 p2p/types/libp2p_mocks.go diff --git a/p2p/types/libp2p_mocks.go b/p2p/types/libp2p_mocks.go new file mode 100644 index 000000000..05cc3808c --- /dev/null +++ b/p2p/types/libp2p_mocks.go @@ -0,0 +1,3 @@ +package types + +//go:generate mockgen -package mock_types -destination=./mocks/host_mock.go github.com/libp2p/go-libp2p/core/host Host From 10aa7d9ad3c929bd711de34ff9b7d6dd37769d9c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 23 Mar 2023 16:17:30 +0100 Subject: [PATCH 13/82] refactor: move `shared/p2p` pkg into `p2p/types` --- app/client/cli/debug.go | 4 ++-- p2p/bootstrap.go | 4 ++-- p2p/peer_test.go | 5 ++--- .../peerstore_provider/peerstore_provider.go | 14 +++++++------- .../peerstore_provider/persistence/provider.go | 3 +-- p2p/providers/peerstore_provider/rpc/provider.go | 3 +-- p2p/raintree/network.go | 15 +++++++-------- p2p/raintree/network_test.go | 9 ++++----- p2p/raintree/peers_manager.go | 16 ++++++++-------- p2p/raintree/peers_manager_test.go | 10 +++++----- p2p/raintree/utils_test.go | 4 ++-- p2p/stdnetwork/network.go | 9 ++++----- p2p/types/network.go | 7 +++---- p2p/types/network_peer.go | 3 +-- {shared/p2p => p2p/types}/peer.go | 2 +- {shared/p2p => p2p/types}/peer_manager.go | 2 +- {shared/p2p => p2p/types}/peers_view.go | 2 +- {shared/p2p => p2p/types}/peers_view_test.go | 2 +- {shared/p2p => p2p/types}/peerstore.go | 2 +- p2p/utils/host.go | 4 ++-- p2p/utils/peer_conversion.go | 8 ++++---- 21 files changed, 60 insertions(+), 68 deletions(-) rename {shared/p2p => p2p/types}/peer.go (98%) rename {shared/p2p => p2p/types}/peer_manager.go (99%) rename {shared/p2p => p2p/types}/peers_view.go (99%) rename {shared/p2p => p2p/types}/peers_view_test.go (92%) rename {shared/p2p => p2p/types}/peerstore.go (99%) diff --git a/app/client/cli/debug.go b/app/client/cli/debug.go index 257445c7b..5217a9922 100644 --- a/app/client/cli/debug.go +++ b/app/client/cli/debug.go @@ -15,11 +15,11 @@ import ( rpcCHP "github.com/pokt-network/pocket/p2p/providers/current_height_provider/rpc" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" rpcABP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/rpc" + typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/defaults" "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) // TECHDEBT: Lowercase variables / constants that do not need to be exported. @@ -266,7 +266,7 @@ func sendDebugMessage(cmd *cobra.Command, debugMsg *messaging.DebugMessage) { } // fetchPeerstore retrieves the providers from the CLI context and uses them to retrieve the address book for the current height -func fetchPeerstore(cmd *cobra.Command) (sharedP2P.Peerstore, error) { +func fetchPeerstore(cmd *cobra.Command) (typesP2P.Peerstore, error) { bus, ok := getValueFromCLIContext[modules.Bus](cmd, busCLICtxKey) if !ok || bus == nil { return nil, errors.New("retrieving bus from CLI context") diff --git a/p2p/bootstrap.go b/p2p/bootstrap.go index 39a7ca1ce..bd49ea2fd 100644 --- a/p2p/bootstrap.go +++ b/p2p/bootstrap.go @@ -11,9 +11,9 @@ import ( rpcCHP "github.com/pokt-network/pocket/p2p/providers/current_height_provider/rpc" rpcABP "github.com/pokt-network/pocket/p2p/providers/peerstore_provider/rpc" + typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/rpc" "github.com/pokt-network/pocket/runtime/defaults" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) // configureBootstrapNodes parses the bootstrap nodes from the config and validates them @@ -43,7 +43,7 @@ func (m *p2pModule) configureBootstrapNodes() error { // bootstrap attempts to bootstrap from a bootstrap node func (m *p2pModule) bootstrap() error { - var pstore sharedP2P.Peerstore + var pstore typesP2P.Peerstore for _, bootstrapNode := range m.bootstrapNodes { m.logger.Info().Str("endpoint", bootstrapNode).Msg("Attempting to bootstrap from bootstrap node") diff --git a/p2p/peer_test.go b/p2p/peer_test.go index 4e6f630c1..17840c7b8 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -5,7 +5,6 @@ import ( typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/crypto" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" "github.com/stretchr/testify/require" ) @@ -59,12 +58,12 @@ func Test_getPeerListDelta(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - before := make(sharedP2P.PeerList, len(tt.args.before)) + before := make(typesP2P.PeerList, len(tt.args.before)) for i, peer := range tt.args.before { before[i] = peer } - after := make(sharedP2P.PeerList, len(tt.args.after)) + after := make(typesP2P.PeerList, len(tt.args.after)) for i, peer := range tt.args.after { after[i] = peer } diff --git a/p2p/providers/peerstore_provider/peerstore_provider.go b/p2p/providers/peerstore_provider/peerstore_provider.go index 210c94ee7..47059c096 100644 --- a/p2p/providers/peerstore_provider/peerstore_provider.go +++ b/p2p/providers/peerstore_provider/peerstore_provider.go @@ -4,13 +4,14 @@ package peerstore_provider import ( "fmt" + + "go.uber.org/multierr" + typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime/configs" coreTypes "github.com/pokt-network/pocket/shared/core/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" - "go.uber.org/multierr" ) const ModuleName = "peerstore_provider" @@ -19,14 +20,14 @@ const ModuleName = "peerstore_provider" type PeerstoreProvider interface { modules.Module - GetStakedPeerstoreAtHeight(height uint64) (sharedP2P.Peerstore, error) + GetStakedPeerstoreAtHeight(height uint64) (typesP2P.Peerstore, error) GetConnFactory() typesP2P.ConnectionFactory GetP2PConfig() *configs.P2PConfig SetConnectionFactory(typesP2P.ConnectionFactory) } -func ActorsToPeerstore(abp PeerstoreProvider, actors []*coreTypes.Actor) (pstore sharedP2P.Peerstore, errs error) { - pstore = make(sharedP2P.PeerAddrMap) +func ActorsToPeerstore(abp PeerstoreProvider, actors []*coreTypes.Actor) (pstore typesP2P.Peerstore, errs error) { + pstore = make(typesP2P.PeerAddrMap) for _, a := range actors { networkPeer, err := ActorToPeer(abp, a) if err != nil { @@ -41,7 +42,7 @@ func ActorsToPeerstore(abp PeerstoreProvider, actors []*coreTypes.Actor) (pstore return pstore, errs } -func ActorToPeer(abp PeerstoreProvider, actor *coreTypes.Actor) (sharedP2P.Peer, error) { +func ActorToPeer(abp PeerstoreProvider, actor *coreTypes.Actor) (typesP2P.Peer, error) { // TECHDEBT(#576): this should be the responsibility of some new `ConnManager` interface. // Peerstore (identity / address mapping) is a separate concern from managing // connections to/from peers. @@ -49,7 +50,6 @@ func ActorToPeer(abp PeerstoreProvider, actor *coreTypes.Actor) (sharedP2P.Peer, if err != nil { return nil, fmt.Errorf("error resolving addr: %v", err) } - pubKey, err := cryptoPocket.NewPublicKey(actor.GetPublicKey()) if err != nil { return nil, err diff --git a/p2p/providers/peerstore_provider/persistence/provider.go b/p2p/providers/peerstore_provider/persistence/provider.go index b5bb4feee..02e839687 100644 --- a/p2p/providers/peerstore_provider/persistence/provider.go +++ b/p2p/providers/peerstore_provider/persistence/provider.go @@ -7,7 +7,6 @@ import ( "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) var _ peerstore_provider.PeerstoreProvider = &persistencePeerstoreProvider{} @@ -44,7 +43,7 @@ func (*persistencePeerstoreProvider) GetModuleName() string { return peerstore_provider.ModuleName } -func (pabp *persistencePeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (sharedP2P.Peerstore, error) { +func (pabp *persistencePeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (typesP2P.Peerstore, error) { persistenceReadContext, err := pabp.GetBus().GetPersistenceModule().NewReadContext(int64(height)) if err != nil { return nil, err diff --git a/p2p/providers/peerstore_provider/rpc/provider.go b/p2p/providers/peerstore_provider/rpc/provider.go index c65486276..b8798e50e 100644 --- a/p2p/providers/peerstore_provider/rpc/provider.go +++ b/p2p/providers/peerstore_provider/rpc/provider.go @@ -17,7 +17,6 @@ import ( "github.com/pokt-network/pocket/shared/core/types" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) var ( @@ -68,7 +67,7 @@ func (*rpcPeerstoreProvider) GetModuleName() string { return peerstore_provider.ModuleName } -func (rabp *rpcPeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (sharedP2P.Peerstore, error) { +func (rabp *rpcPeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (typesP2P.Peerstore, error) { ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) defer cancel() diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 50587518d..5a45f9719 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -15,7 +15,6 @@ import ( "github.com/pokt-network/pocket/shared/messaging" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" telemetry "github.com/pokt-network/pocket/telemetry" "google.golang.org/protobuf/proto" ) @@ -216,24 +215,24 @@ func (n *rainTreeNetwork) HandleNetworkData(data []byte) ([]byte, error) { return rainTreeMsg.Data, nil } -func (n *rainTreeNetwork) GetPeerstore() sharedP2P.Peerstore { +func (n *rainTreeNetwork) GetPeerstore() typesP2P.Peerstore { return n.peersManager.GetPeerstore() } -func (n *rainTreeNetwork) AddPeer(peer sharedP2P.Peer) error { +func (n *rainTreeNetwork) AddPeer(peer typesP2P.Peer) error { n.peersManager.HandleEvent( - sharedP2P.PeerManagerEvent{ - EventType: sharedP2P.AddPeerEventType, + typesP2P.PeerManagerEvent{ + EventType: typesP2P.AddPeerEventType, Peer: peer, }, ) return nil } -func (n *rainTreeNetwork) RemovePeer(peer sharedP2P.Peer) error { +func (n *rainTreeNetwork) RemovePeer(peer typesP2P.Peer) error { n.peersManager.HandleEvent( - sharedP2P.PeerManagerEvent{ - EventType: sharedP2P.RemovePeerEventType, + typesP2P.PeerManagerEvent{ + EventType: typesP2P.RemovePeerEventType, Peer: peer, }, ) diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index 01bf4246a..152624a29 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -7,7 +7,6 @@ import ( typesP2P "github.com/pokt-network/pocket/p2p/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" "github.com/stretchr/testify/require" ) @@ -86,7 +85,7 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { require.Equal(t, expectedPStoreSize, len(peerAddrs)) require.Equal(t, expectedPStoreSize, len(peers)) - var peerToRemove sharedP2P.Peer + var peerToRemove typesP2P.Peer // Ensure we don't remove selfPeer. `Peerstore` interface isn't aware // of the concept of "self" so we have to find it. for _, peer := range pstore.GetPeerList() { @@ -105,7 +104,7 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { peerAddrs, peers = getPeersViewParts(network.peersManager) removedAddr := peerToRemove.GetAddress() - getPeer := func(addr cryptoPocket.Address) sharedP2P.Peer { + getPeer := func(addr cryptoPocket.Address) typesP2P.Peer { return network.GetPeerstore().GetPeer(addr) } @@ -118,9 +117,9 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { require.Nil(t, getPeer(removedAddr), "Peerstore contains removed peer") } -func getPeersViewParts(pm sharedP2P.PeerManager) ( +func getPeersViewParts(pm typesP2P.PeerManager) ( addrs []string, - peers sharedP2P.PeerList, + peers typesP2P.PeerList, ) { view := pm.GetPeersView() addrs = view.GetAddrs() diff --git a/p2p/raintree/peers_manager.go b/p2p/raintree/peers_manager.go index 7e0aed873..5fa1ba91c 100644 --- a/p2p/raintree/peers_manager.go +++ b/p2p/raintree/peers_manager.go @@ -6,15 +6,15 @@ import ( "sync" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" + typesP2P "github.com/pokt-network/pocket/p2p/types" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) -var _ sharedP2P.PeerManager = &rainTreePeersManager{} +var _ typesP2P.PeerManager = &rainTreePeersManager{} // rainTreePeersManager is in charge of handling operations on peers (like adding/removing them) within an Peerstore type rainTreePeersManager struct { - *sharedP2P.SortedPeerManager + *typesP2P.SortedPeerManager maxLevelsMutex sync.Mutex maxNumLevels uint32 @@ -33,8 +33,8 @@ func newPeersManagerWithPeerstoreProvider(selfAddr cryptoPocket.Address, pstoreP // it also takes care of keeping the Peerstore sorted and indexed for fast access // // If `isDynamic` is false, the rainTreePeersManager will not handle addressBook changes, it will only be used for querying the Peerstore -func newPeersManager(selfAddr cryptoPocket.Address, pstore sharedP2P.Peerstore, isDynamic bool) (*rainTreePeersManager, error) { - sortedPM, err := sharedP2P.NewSortedPeerManager(selfAddr, pstore, isDynamic) +func newPeersManager(selfAddr cryptoPocket.Address, pstore typesP2P.Peerstore, isDynamic bool) (*rainTreePeersManager, error) { + sortedPM, err := typesP2P.NewSortedPeerManager(selfAddr, pstore, isDynamic) if err != nil { return nil, err } @@ -50,12 +50,12 @@ func newPeersManager(selfAddr cryptoPocket.Address, pstore sharedP2P.Peerstore, return pm, nil } -func (pm *rainTreePeersManager) HandleEvent(evt sharedP2P.PeerManagerEvent) { +func (pm *rainTreePeersManager) HandleEvent(evt typesP2P.PeerManagerEvent) { pm.SortedPeerManager.HandleEvent(evt) pm.updateMaxNumLevels() } -func (pm *rainTreePeersManager) GetPeersView() sharedP2P.PeersView { +func (pm *rainTreePeersManager) GetPeersView() typesP2P.PeersView { return pm.SortedPeerManager.GetPeersView() } @@ -66,7 +66,7 @@ func (pm *rainTreePeersManager) GetMaxNumLevels() uint32 { return pm.maxNumLevels } -func (pm *rainTreePeersManager) getPeersViewWithLevels() (view sharedP2P.PeersView, level uint32) { +func (pm *rainTreePeersManager) getPeersViewWithLevels() (view typesP2P.PeersView, level uint32) { return pm.GetPeersView(), pm.GetMaxNumLevels() } diff --git a/p2p/raintree/peers_manager_test.go b/p2p/raintree/peers_manager_test.go index 78d009890..09b011370 100644 --- a/p2p/raintree/peers_manager_test.go +++ b/p2p/raintree/peers_manager_test.go @@ -7,11 +7,11 @@ import ( "testing" "github.com/golang/mock/gomock" + "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime/configs" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" "github.com/stretchr/testify/require" ) @@ -153,8 +153,8 @@ func BenchmarkPeerstoreUpdates(b *testing.B) { } // Generates an address book with a random set of `n` addresses -func getPeerstore(t *testing.T, n int) sharedP2P.Peerstore { - pstore := make(sharedP2P.PeerAddrMap) +func getPeerstore(t *testing.T, n int) typesP2P.Peerstore { + pstore := make(typesP2P.PeerAddrMap) for i := 0; i < n; i++ { addr, err := cryptoPocket.GenerateAddress() if t != nil { @@ -252,8 +252,8 @@ func testRainTreeMessageTargets(t *testing.T, expectedMsgProp *ExpectedRainTreeM } // Generates an address book with a constant set 27 addresses; ['A', ..., 'Z'] -func getAlphabetPeerstore(t *testing.T, n int) sharedP2P.Peerstore { - pstore := make(sharedP2P.PeerAddrMap) +func getAlphabetPeerstore(t *testing.T, n int) typesP2P.Peerstore { + pstore := make(typesP2P.PeerAddrMap) for i, ch := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ[" { if i >= n { return pstore diff --git a/p2p/raintree/utils_test.go b/p2p/raintree/utils_test.go index ec1738256..82e99438a 100644 --- a/p2p/raintree/utils_test.go +++ b/p2p/raintree/utils_test.go @@ -2,10 +2,10 @@ package raintree import ( "github.com/golang/mock/gomock" + typesP2P "github.com/pokt-network/pocket/p2p/types" mocksP2P "github.com/pokt-network/pocket/p2p/types/mocks" "github.com/pokt-network/pocket/runtime/configs" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) func mockBus(ctrl *gomock.Controller) *mockModules.MockBus { @@ -20,7 +20,7 @@ func mockBus(ctrl *gomock.Controller) *mockModules.MockBus { return busMock } -func mockPeerstoreProvider(ctrl *gomock.Controller, pstore sharedP2P.Peerstore) *mocksP2P.MockPeerstoreProvider { +func mockPeerstoreProvider(ctrl *gomock.Controller, pstore typesP2P.Peerstore) *mocksP2P.MockPeerstoreProvider { peerstoreProviderMock := mocksP2P.NewMockPeerstoreProvider(ctrl) peerstoreProviderMock.EXPECT().GetStakedPeerstoreAtHeight(gomock.Any()).Return(pstore, nil).AnyTimes() return peerstoreProviderMock diff --git a/p2p/stdnetwork/network.go b/p2p/stdnetwork/network.go index 7bc79dfb9..6d8ad3062 100644 --- a/p2p/stdnetwork/network.go +++ b/p2p/stdnetwork/network.go @@ -13,7 +13,6 @@ import ( "github.com/pokt-network/pocket/p2p/utils" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) var ( @@ -23,7 +22,7 @@ var ( type network struct { host libp2pHost.Host - pstore sharedP2P.Peerstore + pstore typesP2P.Peerstore logger *modules.Logger } @@ -74,15 +73,15 @@ func (n *network) HandleNetworkData(data []byte) ([]byte, error) { return data, nil // intentional passthrough } -func (n *network) GetPeerstore() sharedP2P.Peerstore { +func (n *network) GetPeerstore() typesP2P.Peerstore { return n.pstore } -func (n *network) AddPeer(peer sharedP2P.Peer) error { +func (n *network) AddPeer(peer typesP2P.Peer) error { return n.pstore.AddPeer(peer) } -func (n *network) RemovePeer(peer sharedP2P.Peer) error { +func (n *network) RemovePeer(peer typesP2P.Peer) error { return n.pstore.RemovePeer(peer.GetAddress()) } diff --git a/p2p/types/network.go b/p2p/types/network.go index 6652d5712..15c4fe7e4 100644 --- a/p2p/types/network.go +++ b/p2p/types/network.go @@ -5,7 +5,6 @@ package types import ( cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) // TECHDEBT(olshansky): When we delete `stdnetwork` and only go with `raintree`, this interface @@ -18,9 +17,9 @@ type Network interface { // Address book helpers // TECHDEBT: simplify - remove `GetPeerstore` - GetPeerstore() sharedP2P.Peerstore - AddPeer(peer sharedP2P.Peer) error - RemovePeer(peer sharedP2P.Peer) error + GetPeerstore() Peerstore + AddPeer(peer Peer) error + RemovePeer(peer Peer) error // This function was added to specifically support the RainTree implementation. // Handles the raw data received from the network and returns the data to be processed diff --git a/p2p/types/network_peer.go b/p2p/types/network_peer.go index 71a4f42fc..e6a2e06c4 100644 --- a/p2p/types/network_peer.go +++ b/p2p/types/network_peer.go @@ -6,10 +6,9 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/pokt-network/pocket/shared/crypto" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) -var _ sharedP2P.Peer = &NetworkPeer{} +var _ Peer = &NetworkPeer{} type NetworkPeer struct { Transport Transport diff --git a/shared/p2p/peer.go b/p2p/types/peer.go similarity index 98% rename from shared/p2p/peer.go rename to p2p/types/peer.go index d6f82de30..6b2cd84a0 100644 --- a/shared/p2p/peer.go +++ b/p2p/types/peer.go @@ -1,4 +1,4 @@ -package p2p +package types import ( "io" diff --git a/shared/p2p/peer_manager.go b/p2p/types/peer_manager.go similarity index 99% rename from shared/p2p/peer_manager.go rename to p2p/types/peer_manager.go index 39f6606bb..663c4fe2e 100644 --- a/shared/p2p/peer_manager.go +++ b/p2p/types/peer_manager.go @@ -1,4 +1,4 @@ -package p2p +package types import ( "sync" diff --git a/shared/p2p/peers_view.go b/p2p/types/peers_view.go similarity index 99% rename from shared/p2p/peers_view.go rename to p2p/types/peers_view.go index e97c3042f..d595940d6 100644 --- a/shared/p2p/peers_view.go +++ b/p2p/types/peers_view.go @@ -1,4 +1,4 @@ -package p2p +package types import ( "sort" diff --git a/shared/p2p/peers_view_test.go b/p2p/types/peers_view_test.go similarity index 92% rename from shared/p2p/peers_view_test.go rename to p2p/types/peers_view_test.go index 1ee82c5e3..8dd58ae0c 100644 --- a/shared/p2p/peers_view_test.go +++ b/p2p/types/peers_view_test.go @@ -1,4 +1,4 @@ -package p2p +package types import ( "testing" diff --git a/shared/p2p/peerstore.go b/p2p/types/peerstore.go similarity index 99% rename from shared/p2p/peerstore.go rename to p2p/types/peerstore.go index 3103e2128..3a80fafda 100644 --- a/shared/p2p/peerstore.go +++ b/p2p/types/peerstore.go @@ -1,4 +1,4 @@ -package p2p +package types import ( "fmt" diff --git a/p2p/utils/host.go b/p2p/utils/host.go index b851258de..988a20ad4 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -9,7 +9,7 @@ import ( "go.uber.org/multierr" "github.com/pokt-network/pocket/p2p/protocol" - "github.com/pokt-network/pocket/shared/p2p" + typesP2P "github.com/pokt-network/pocket/p2p/types" ) const ( @@ -23,7 +23,7 @@ const ( // info for use with libp2p and adding it to the underlying libp2p host's peerstore. // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/host#Host) // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/peerstore#Peerstore) -func PopulateLibp2pHost(host libp2pHost.Host, pstore p2p.Peerstore) error { +func PopulateLibp2pHost(host libp2pHost.Host, pstore typesP2P.Peerstore) error { for _, peer := range pstore.GetPeerList() { pubKey, err := Libp2pPublicKeyFromPeer(peer) if err != nil { diff --git a/p2p/utils/peer_conversion.go b/p2p/utils/peer_conversion.go index b0ba7b9ce..2268a8c4e 100644 --- a/p2p/utils/peer_conversion.go +++ b/p2p/utils/peer_conversion.go @@ -10,13 +10,13 @@ import ( "github.com/pokt-network/pocket/libp2p/transport" "github.com/pokt-network/pocket/p2p/types" + typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/crypto" - sharedP2P "github.com/pokt-network/pocket/shared/p2p" ) // PeerFromLibp2pStream builds a network peer using peer info available // from the given libp2p stream. -func PeerFromLibp2pStream(stream network.Stream) (sharedP2P.Peer, error) { +func PeerFromLibp2pStream(stream network.Stream) (typesP2P.Peer, error) { publicKeyBz, err := stream.Conn().RemotePublicKey().Raw() if err != nil { return nil, err @@ -42,7 +42,7 @@ func PeerFromLibp2pStream(stream network.Stream) (sharedP2P.Peer, error) { } // Libp2pPublicKeyFromPeer retrieves the libp2p compatible public key from a pocket peer. -func Libp2pPublicKeyFromPeer(peer sharedP2P.Peer) (libp2pCrypto.PubKey, error) { +func Libp2pPublicKeyFromPeer(peer typesP2P.Peer) (libp2pCrypto.PubKey, error) { publicKey, err := libp2pCrypto.UnmarshalEd25519PublicKey(peer.GetPublicKey().Bytes()) if err != nil { return nil, fmt.Errorf( @@ -55,7 +55,7 @@ func Libp2pPublicKeyFromPeer(peer sharedP2P.Peer) (libp2pCrypto.PubKey, error) { } // Libp2pAddrInfoFromPeer builds a libp2p AddrInfo which maps to the passed pocket peer. -func Libp2pAddrInfoFromPeer(peer sharedP2P.Peer) (libp2pPeer.AddrInfo, error) { +func Libp2pAddrInfoFromPeer(peer typesP2P.Peer) (libp2pPeer.AddrInfo, error) { publicKey, err := Libp2pPublicKeyFromPeer(peer) if err != nil { return libp2pPeer.AddrInfo{}, err From 304c4cdbfaa253fac3038ac4af64678bc8b82ba4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 22:55:05 +0100 Subject: [PATCH 14/82] chore: add `package` flag to `Network` mockgen call --- p2p/types/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/types/network.go b/p2p/types/network.go index 15c4fe7e4..e3f94fc04 100644 --- a/p2p/types/network.go +++ b/p2p/types/network.go @@ -1,6 +1,6 @@ package types -//go:generate mockgen -source=$GOFILE -destination=./mocks/network_mock.go github.com/pokt-network/pocket/p2p/types Network +//go:generate mockgen -source=$GOFILE -destination=./mocks/network_mock.go -package=mock_types github.com/pokt-network/pocket/p2p/types Network import ( cryptoPocket "github.com/pokt-network/pocket/shared/crypto" From 2d65acc4d8a22e711bc64def20eba1384d5484e9 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 22:55:46 +0100 Subject: [PATCH 15/82] chore: use `net.DefaultResolver` in `getPeerIP` --- p2p/utils/url_conversion.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/p2p/utils/url_conversion.go b/p2p/utils/url_conversion.go index 0a4e54fc4..172b4b943 100644 --- a/p2p/utils/url_conversion.go +++ b/p2p/utils/url_conversion.go @@ -1,6 +1,7 @@ package utils import ( + "context" "crypto/rand" "fmt" "math/big" @@ -130,6 +131,9 @@ func newResolvePeerIPErr(hostname string, err error) error { } func getPeerIP(hostname string) (net.IP, error) { + // TECHDEBT(#595): receive `ctx` from caller. + ctx := context.Background() + // Attempt to parse peer hostname as an IP address. // (see: https://pkg.go.dev/net#ParseIP) peerIP := net.ParseIP(hostname) @@ -140,7 +144,7 @@ func getPeerIP(hostname string) (net.IP, error) { // CONSIDERATION: using a `/dns<4 or 6>/` multiaddr instead of resolving here. // I attempted using `/dns4/.../tcp/...` and go this error: // > failed to listen on any addresses: [can only dial TCP over IPv4 or IPv6] - addrs, err := net.LookupHost(hostname) + addrs, err := net.DefaultResolver.LookupHost(ctx, hostname) if err != nil { return nil, newResolvePeerIPErr(hostname, err) } From d0c881e00df20d52d333b637d33625f0f4852875 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 22:24:50 +0100 Subject: [PATCH 16/82] refactor: P2P module tests --- p2p/module_raintree_test.go | 64 +++++++++++++--------- p2p/module_test.go | 46 +++++++++++++--- p2p/utils_test.go | 106 +++++++++++++++++++++++++++--------- 3 files changed, 156 insertions(+), 60 deletions(-) diff --git a/p2p/module_raintree_test.go b/p2p/module_raintree_test.go index 47515c281..d755aebaf 100644 --- a/p2p/module_raintree_test.go +++ b/p2p/module_raintree_test.go @@ -7,12 +7,16 @@ import ( "regexp" "sort" "strconv" + "strings" "sync" "testing" - typesP2P "github.com/pokt-network/pocket/p2p/types" + libp2pNetwork "github.com/libp2p/go-libp2p/core/network" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" + + "github.com/pokt-network/pocket/p2p/protocol" ) // TODO(#314): Add the tooling and instructions on how to generate unit tests in this file. @@ -228,14 +232,14 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te return iId < jId }) + prepareDNSResolverMock(t, valIds) + // Create connection and bus mocks along with a shared WaitGroup to track the number of expected // reads and writes throughout the mocked local network var wg sync.WaitGroup - connMocks := make(map[string]typesP2P.Transport) - count := 0 - for _, valId := range valIds { + for i, valId := range valIds { expectedCall := networkSimulationConfig[valId] - expectedReads := expectedCall.numNetworkReads + 1 + expectedReads := expectedCall.numNetworkReads expectedWrites := expectedCall.numNetworkWrites log.Printf("[valId: %s] expected reads: %d\n", valId, expectedReads) @@ -243,43 +247,51 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te wg.Add(expectedReads) wg.Add(expectedWrites) - connMocks[valId] = prepareConnMock(t, valId, &wg, expectedCall.numNetworkReads) - persistenceMock := preparePersistenceMock(t, busMocks[count], runtimeConfigs[0].GetGenesis()) - consensusMock := prepareConsensusMock(t, busMocks[count]) - telemetryMock := prepareTelemetryMock(t, busMocks[count], valId, &wg, expectedWrites) - - prepareBusMock(busMocks[count], persistenceMock, consensusMock, telemetryMock) + persistenceMock := preparePersistenceMock(t, busMocks[i], runtimeConfigs[0].GetGenesis()) + consensusMock := prepareConsensusMock(t, busMocks[i]) + telemetryMock := prepareTelemetryMock(t, busMocks[i], valId, &wg, expectedWrites) - count++ + prepareBusMock(busMocks[i], persistenceMock, consensusMock, telemetryMock) } + libp2pMockNet := mocknet.New() + defer func() { + err := libp2pMockNet.Close() + require.NoError(t, err) + }() + // Inject the connection and bus mocks into the P2P modules - p2pModules := createP2PModules(t, busMocks) - for validatorId, p2pMod := range p2pModules { - p2pMod.listener = connMocks[validatorId] + p2pModules := createP2PModules(t, busMocks, libp2pMockNet) + + for serviceURL, p2pMod := range p2pModules { err := p2pMod.Start() require.NoError(t, err) - for _, peer := range p2pMod.network.GetPeerstore().GetPeerList() { - netPeer, ok := peer.(*typesP2P.NetworkPeer) - require.True(t, ok, "unexpected `peer` type: %T", peer) - netPeer.Transport = connMocks[peer.GetServiceURL()] - } + sURL := strings.Clone(serviceURL) + mod := *p2pMod + p2pMod.host.SetStreamHandler(protocol.PoktProtocolID, func(stream libp2pNetwork.Stream) { + log.Printf("[valID: %s] Read\n", sURL) + (&mod).handleStream(stream) + wg.Done() + }) } // Wait for completion - defer waitForNetworkSimulationCompletion(t, &wg) + defer func() { + waitForNetworkSimulationCompletion(t, &wg) + + // Stop all p2p modules + for _, p2pMod := range p2pModules { + err := p2pMod.Stop() + require.NoError(t, err) + } + }() // Send the first message (by the originator) to trigger a RainTree broadcast p := &anypb.Any{} p2pMod := p2pModules[origNode] require.NoError(t, p2pMod.Broadcast(p)) - // Stop all p2p modules outside of loop - for _, p2pMod := range p2pModules { - err := p2pMod.Stop() - require.NoError(t, err) - } } func extractNumericId(valId string) int64 { diff --git a/p2p/module_test.go b/p2p/module_test.go index 7e79d5653..946ddfcbc 100644 --- a/p2p/module_test.go +++ b/p2p/module_test.go @@ -5,17 +5,22 @@ import ( "testing" "github.com/golang/mock/gomock" + libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" + libp2pHost "github.com/libp2p/go-libp2p/core/host" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" + + typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/defaults" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - "github.com/pokt-network/pocket/shared/modules" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" - "github.com/stretchr/testify/require" ) func Test_Create_configureBootstrapNodes(t *testing.T) { defaultBootstrapNodes := strings.Split(defaults.DefaultP2PBootstrapNodesCsv, ",") - key := cryptoPocket.GetPrivKeySeed(1) + privKey := cryptoPocket.GetPrivKeySeed(1) type args struct { initialBootstrapNodesCsv string @@ -103,19 +108,31 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { ctrl := gomock.NewController(t) mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) mockBus := createMockBus(t, mockRuntimeMgr) + + genesisStateMock := createMockGenesisState(keys[:1]) + persistenceMock := preparePersistenceMock(t, mockBus, genesisStateMock) + mockBus.EXPECT().GetPersistenceModule().Return(persistenceMock).AnyTimes() + mockConsensusModule := mockModules.NewMockConsensusModule(ctrl) + mockConsensusModule.EXPECT().CurrentHeight().Return(uint64(1)).AnyTimes() mockBus.EXPECT().GetConsensusModule().Return(mockConsensusModule).AnyTimes() mockRuntimeMgr.EXPECT().GetConfig().Return(&configs.Config{ - PrivateKey: key.String(), + PrivateKey: privKey.String(), P2P: &configs.P2PConfig{ BootstrapNodesCsv: tt.args.initialBootstrapNodesCsv, - PrivateKey: key.String(), + PrivateKey: privKey.String(), }, }).AnyTimes() mockBus.EXPECT().GetRuntimeMgr().Return(mockRuntimeMgr).AnyTimes() - var p2pMod modules.Module - p2pMod, err := Create(mockBus) + peer := &typesP2P.NetworkPeer{ + PublicKey: privKey.PublicKey(), + Address: privKey.Address(), + ServiceURL: "10.0.0.1:42069", + } + + host := newLibp2pMockNetHost(t, privKey, peer) + p2pMod, err := Create(mockBus, WithHostOption(host)) if (err != nil) != tt.wantErr { t.Errorf("p2pModule.Create() error = %v, wantErr %v", err, tt.wantErr) } @@ -127,3 +144,18 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { }) } } + +// TECHDEBT(#609): move & de-duplicate +func newLibp2pMockNetHost(t *testing.T, privKey cryptoPocket.PrivateKey, peer *typesP2P.NetworkPeer) libp2pHost.Host { + libp2pPrivKey, err := libp2pCrypto.UnmarshalEd25519PrivateKey(privKey.Bytes()) + require.NoError(t, err) + + libp2pMultiAddr, err := utils.Libp2pMultiaddrFromServiceURL(peer.ServiceURL) + require.NoError(t, err) + + libp2pMockNet := mocknet.New() + host, err := libp2pMockNet.AddPeer(libp2pPrivKey, libp2pMultiAddr) + require.NoError(t, err) + + return host +} diff --git a/p2p/utils_test.go b/p2p/utils_test.go index 9ff1c6955..90aa60b08 100644 --- a/p2p/utils_test.go +++ b/p2p/utils_test.go @@ -3,18 +3,24 @@ package p2p import ( "fmt" "log" + "net" + "net/url" "sort" "sync" "testing" "time" + "github.com/foxcpp/go-mockdns" "github.com/golang/mock/gomock" + libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" + libp2pPeer "github.com/libp2p/go-libp2p/core/peer" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" "github.com/pokt-network/pocket/p2p/providers/current_height_provider" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" - mocksP2P "github.com/pokt-network/pocket/p2p/types/mocks" + "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/configs/types" @@ -97,16 +103,85 @@ func waitForNetworkSimulationCompletion(t *testing.T, wg *sync.WaitGroup) { // ~~~~~~ RainTree Unit Test Mocks ~~~~~~ // createP2PModules returns a map of configured p2pModules keyed by an incremental naming convention (eg: `val_1`, `val_2`, etc.) -func createP2PModules(t *testing.T, busMocks []*mockModules.MockBus) (p2pModules map[string]*p2pModule) { +func createP2PModules(t *testing.T, busMocks []*mockModules.MockBus, netMock mocknet.Mocknet) (p2pModules map[string]*p2pModule) { + peerIDs := setupMockNetPeers(t, netMock, len(busMocks)) + p2pModules = make(map[string]*p2pModule, len(busMocks)) for i := range busMocks { - p2pMod, err := Create(busMocks[i]) + host := netMock.Host(peerIDs[i]) + p2pMod, err := Create(busMocks[i], WithHostOption(host)) require.NoError(t, err) p2pModules[validatorId(i+1)] = p2pMod.(*p2pModule) } return } +func setupMockNetPeers(t *testing.T, netMock mocknet.Mocknet, numPeers int) (peerIDs []libp2pPeer.ID) { + // Add a libp2p peers/hosts to the `MockNet` with the keypairs corresponding + // to the genesis validators' keypairs + for i, privKey := range keys[:numPeers] { + peerInfo, err := utils.Libp2pAddrInfoFromPeer(&typesP2P.NetworkPeer{ + PublicKey: privKey.PublicKey(), + Address: privKey.Address(), + ServiceURL: validatorId(i + 1), + }) + require.NoError(t, err) + + libp2pPrivKey, err := libp2pCrypto.UnmarshalEd25519PrivateKey(privKey.Bytes()) + require.NoError(t, err) + + _, err = netMock.AddPeer(libp2pPrivKey, peerInfo.Addrs[0]) + require.NoError(t, err) + + peerIDs = append(peerIDs, peerInfo.ID) + } + + // Link all peers such that any may dial/connect to any other. + err := netMock.LinkAll() + require.NoError(t, err) + + return peerIDs +} + +// TECHDEBT(#609): this is one of a few places where we could de-duplicate test +// code if we had a conventional place to store packages intended for import +// only into tests. +func prepareDNSResolverMock(t *testing.T, serviceURLs []string) (done func()) { + zones := make(map[string]mockdns.Zone) + for i, u := range serviceURLs { + // Perpend `scheme://` as serviceURLs are currently scheme-less. + // Required to for parsing to produce useful results. + // (see: https://pkg.go.dev/net/url@go1.20.2#URL) + serviceURL, err := url.Parse(fmt.Sprintf("scheme://%s", u)) + require.NoError(t, err) + + ipStr := fmt.Sprintf("10.0.0.%d", i+1) + + if i >= 254 { + panic(fmt.Sprintf("would generate invalid IPv4 address: %s", ipStr)) + } + + zones[fmt.Sprintf("%s.", serviceURL.Hostname())] = mockdns.Zone{ + A: []string{ipStr}, + } + } + + srv, _ := mockdns.NewServerWithLogger(zones, noopLogger{}, false) + srv.PatchNet(net.DefaultResolver) + return func() { + _ = srv.Close() + mockdns.UnpatchNet(net.DefaultResolver) + } +} + +// NB: default logging behavior is too noisy. +// noopLogger implements go-mockdns's `mockdns.Logger` interface. +type noopLogger struct{} + +func (nl noopLogger) Printf(format string, args ...interface{}) { + // noop +} + // createMockRuntimeMgrs creates `numValidators` instances of mocked `RuntimeMgr` that are essentially // representing the runtime environments of the validators that we will use in our tests func createMockRuntimeMgrs(t *testing.T, numValidators int) []modules.RuntimeMgr { @@ -120,6 +195,7 @@ func createMockRuntimeMgrs(t *testing.T, numValidators int) []modules.RuntimeMgr RootDirectory: "", PrivateKey: valKeys[i].String(), P2P: &configs.P2PConfig{ + Hostname: validatorId(i + 1), PrivateKey: valKeys[i].String(), Port: defaults.DefaultP2PPort, UseRainTree: true, @@ -269,27 +345,3 @@ func prepareEventMetricsAgentMock(t *testing.T, valId string, wg *sync.WaitGroup return eventMetricsAgentMock } - -// Network transport mock - used to simulate reading to/from the network via the main events channel -// as well as counting the number of expected reads -func prepareConnMock(t *testing.T, valId string, wg *sync.WaitGroup, expectedNumNetworkReads int) typesP2P.Transport { - eventsChannel := make(chan []byte, eventsChannelSize) - ctrl := gomock.NewController(t) - connMock := mocksP2P.NewMockTransport(ctrl) - - connMock.EXPECT().ReadAll().DoAndReturn(func() ([]byte, error) { - wg.Done() - log.Printf("[valId: %s] ReadAll\n", valId) - data := <-eventsChannel - return data, nil - }).Times(expectedNumNetworkReads + 1) // +1 is necessary because there is one extra read of empty data by every channel when it starts - - connMock.EXPECT().Write(gomock.Any()).DoAndReturn(func(data []byte) (int, error) { - eventsChannel <- data - return len(data), nil - }).Times(expectedNumNetworkReads) - - connMock.EXPECT().Close().Return(nil).Times(1) - - return connMock -} From 499f97e16dd5c6b9d8cbde7c51e6752440d30bd6 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 23:11:13 +0100 Subject: [PATCH 17/82] refactor: raintree network --- p2p/module.go | 3 +- p2p/raintree/network.go | 79 ++++++++++++++++++++++++--------- p2p/raintree/peerstore_utils.go | 8 ++-- 3 files changed, 63 insertions(+), 27 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index f7f8f90d1..c8abd6c2e 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -277,7 +277,8 @@ func (m *p2pModule) setupCurrentHeightProvider() { func (m *p2pModule) startNetwork() (err error) { if m.cfg.UseRainTree { - m.network = raintree.NewRainTreeNetwork( + m.network, err = raintree.NewRainTreeNetwork( + m.host, m.address, m.GetBus(), m.pstoreProvider, diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 5a45f9719..2afaa733f 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -4,10 +4,13 @@ import ( "fmt" "log" + libp2pHost "github.com/libp2p/go-libp2p/core/host" + "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/providers" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" "github.com/pokt-network/pocket/shared/codec" "github.com/pokt-network/pocket/shared/crypto" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" @@ -24,43 +27,51 @@ var _ typesP2P.Network = &rainTreeNetwork{} type rainTreeNetwork struct { base_modules.IntegratableModule - selfAddr cryptoPocket.Address - pstoreProvider peerstore_provider.PeerstoreProvider - - peersManager *rainTreePeersManager - nonceDeduper *mempool.GenericFIFOSet[uint64, uint64] - - currentHeightProvider providers.CurrentHeightProvider - logger *modules.Logger + // host represents a libp2p network node, it encapsulates a libp2p peerstore + // & connection manager. `libp2p.New` configures and starts listening + // according to options. + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) + host libp2pHost.Host + // selfAddr is the pocket address representing this host. + selfAddr cryptoPocket.Address + peersManager *rainTreePeersManager + pstoreProvider peerstore_provider.PeerstoreProvider + currentHeightProvider providers.CurrentHeightProvider + nonceDeduper *mempool.GenericFIFOSet[uint64, uint64] } -func NewRainTreeNetwork(addr cryptoPocket.Address, bus modules.Bus, pstoreProvider providers.PeerstoreProvider, currentHeightProvider providers.CurrentHeightProvider) typesP2P.Network { +// TECHDEBT: refactor signature to receive a config struct +// (i.e. `NewRainTreeNetwork(bus modules.Bus, cfg NetworkConfig). +func NewRainTreeNetwork(host libp2pHost.Host, addr cryptoPocket.Address, bus modules.Bus, pstoreProvider providers.PeerstoreProvider, currentHeightProvider providers.CurrentHeightProvider) (typesP2P.Network, error) { networkLogger := logger.Global.CreateLoggerForModule("network") networkLogger.Info().Msg("Initializing rainTreeNetwork") - pstore, err := pstoreProvider.GetStakedPeerstoreAtHeight(currentHeightProvider.CurrentHeight()) - if err != nil { - networkLogger.Fatal().Err(err).Msg("Error getting pstore") + if pstoreProvider == nil { + return nil, fmt.Errorf("peerstore provider required, got nil") } - pm, err := newPeersManager(addr, pstore, true) - if err != nil { - networkLogger.Fatal().Err(err).Msg("Error initializing rainTreeNetwork rainTreePeersManager") + if currentHeightProvider == nil { + return nil, fmt.Errorf("current height provider required, got nil") } p2pCfg := bus.GetRuntimeMgr().GetConfig().P2P n := &rainTreeNetwork{ + host: host, selfAddr: addr, - peersManager: pm, nonceDeduper: mempool.NewGenericFIFOSet[uint64, uint64](int(p2pCfg.MaxMempoolCount)), pstoreProvider: pstoreProvider, currentHeightProvider: currentHeightProvider, logger: networkLogger, } n.SetBus(bus) - return typesP2P.Network(n) + + if err := n.setupDependencies(); err != nil { + return nil, err + } + + return typesP2P.Network(n), nil } func (n *rainTreeNetwork) NetworkBroadcast(data []byte) error { @@ -129,13 +140,11 @@ func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket. peer := n.peersManager.GetPeerstore().GetPeer(address) if peer == nil { - n.logger.Error().Str("address", address.String()).Msg("address not found in peerstore") + return fmt.Errorf("no known peer with pokt address %s", address) } - // TECHDEBT: should not be `Peer`s responsibility - // to manage or expose its connections. - if _, err := peer.GetStream().Write(data); err != nil { - n.logger.Error().Err(err).Msg("Error writing to peer during send") + if err := utils.Libp2pSendToPeer(n.host, data, peer); err != nil { + logger.Global.Debug().Err(err).Msg("from libp2pSendInternal") return err } @@ -220,6 +229,11 @@ func (n *rainTreeNetwork) GetPeerstore() typesP2P.Peerstore { } func (n *rainTreeNetwork) AddPeer(peer typesP2P.Peer) error { + // Noop if peer with the same pokt address exists in the peerstore. + // TECHDEBT: add method(s) to update peers. + if p := n.peersManager.GetPeerstore().GetPeer(peer.GetAddress()); p != nil { + return nil + } n.peersManager.HandleEvent( typesP2P.PeerManagerEvent{ EventType: typesP2P.AddPeerEventType, @@ -246,3 +260,24 @@ func (n *rainTreeNetwork) Size() int { func shouldSendToTarget(target target) bool { return !target.isSelf } + +func (n *rainTreeNetwork) setupDependencies() error { + pstore, err := n.pstoreProvider.GetStakedPeerstoreAtHeight(n.currentHeightProvider.CurrentHeight()) + if err != nil { + return err + } + + if err := n.setupPeerManager(pstore); err != nil { + return err + } + + if err := utils.PopulateLibp2pHost(n.host, pstore); err != nil { + return err + } + return nil +} + +func (n *rainTreeNetwork) setupPeerManager(pstore typesP2P.Peerstore) (err error) { + n.peersManager, err = newPeersManager(n.selfAddr, pstore, true) + return err +} diff --git a/p2p/raintree/peerstore_utils.go b/p2p/raintree/peerstore_utils.go index 160d86161..ac5df0699 100644 --- a/p2p/raintree/peerstore_utils.go +++ b/p2p/raintree/peerstore_utils.go @@ -38,9 +38,11 @@ func (n *rainTreeNetwork) getTargetsAtLevel(level uint32) []target { n.logger.Debug().Fields( map[string]any{ - "firstTarget": firstTarget, - "secondTarget": secondTarget, + "firstTarget": firstTarget.serviceURL, + "secondTarget": secondTarget.serviceURL, "height": height, + "level": level, + "pstoreSize": pstoreSizeAtHeight, }, ).Msg("Targets at height") @@ -49,9 +51,7 @@ func (n *rainTreeNetwork) getTargetsAtLevel(level uint32) []target { func (n *rainTreeNetwork) getTarget(targetPercentage float64, pstoreSize int, level uint32) target { i := int(targetPercentage * float64(pstoreSize)) - peersView := n.peersManager.GetPeersView() - serviceURL := peersView.GetPeers()[i].GetServiceURL() target := target{ From 177f1c18477c0bde6b904efd9abe93c72db44e37 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 23 Mar 2023 16:05:05 +0100 Subject: [PATCH 18/82] refactor: raintree tests --- p2p/raintree/network_test.go | 80 ++++++++++++++---- p2p/raintree/peers_manager_test.go | 126 ++++++++++++++++++++++++----- 2 files changed, 168 insertions(+), 38 deletions(-) diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index 152624a29..2e507043e 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -4,8 +4,12 @@ import ( "testing" "github.com/golang/mock/gomock" + libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" + libp2pHost "github.com/libp2p/go-libp2p/core/host" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" typesP2P "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/p2p/utils" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/stretchr/testify/require" ) @@ -14,15 +18,27 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { ctrl := gomock.NewController(t) // Start with a peerstore containing self. - selfAddr, err := cryptoPocket.GenerateAddress() - require.NoError(t, err) - selfPeer := &typesP2P.NetworkPeer{Address: selfAddr} + selfPeer, host := newTestPeer(t) + selfAddr := selfPeer.GetAddress() expectedPStoreSize := 0 pstore := getPeerstore(nil, expectedPStoreSize) + peers := pstore.GetPeerList() + for _, peer := range peers { + libp2pPeerInfo, err := utils.Libp2pAddrInfoFromPeer(peer) + require.NoError(t, err) + + host.Peerstore().AddAddrs(libp2pPeerInfo.ID, libp2pPeerInfo.Addrs, utils.DefaultPeerTTL) + + libp2pPeerPubKey, err := utils.Libp2pPublicKeyFromPeer(peer) + require.NoError(t, err) + + err = host.Peerstore().AddPubKey(libp2pPeerInfo.ID, libp2pPeerPubKey) + require.NoError(t, err) + } // Add self to peerstore. - err = pstore.AddPeer(&typesP2P.NetworkPeer{Address: selfAddr}) + err := pstore.AddPeer(selfPeer) require.NoError(t, err) expectedPStoreSize++ @@ -30,18 +46,21 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { peerstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) - network := NewRainTreeNetwork(selfAddr, busMock, peerstoreProviderMock, currentHeightProviderMock).(*rainTreeNetwork) + network, err := NewRainTreeNetwork(host, selfAddr, busMock, peerstoreProviderMock, currentHeightProviderMock) + require.NoError(t, err) + + rainTreeNet := network.(*rainTreeNetwork) peerAddr, err := cryptoPocket.GenerateAddress() require.NoError(t, err) peerToAdd := &typesP2P.NetworkPeer{Address: peerAddr} // Add peerToAdd. - err = network.AddPeer(peerToAdd) + err = rainTreeNet.AddPeer(peerToAdd) require.NoError(t, err) expectedPStoreSize++ - peerAddrs, peers := getPeersViewParts(network.peersManager) + peerAddrs, peers := getPeersViewParts(rainTreeNet.peersManager) // Ensure size / lengths are consistent. require.Equal(t, expectedPStoreSize, network.GetPeerstore().Size()) @@ -63,13 +82,12 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { expectedPStoreSize := 3 pstore := getPeerstore(nil, expectedPStoreSize) - selfAddr, err := cryptoPocket.GenerateAddress() - require.NoError(t, err) - selfPeer := &typesP2P.NetworkPeer{Address: selfAddr} + selfPeer, host := newTestPeer(t) + selfAddr := selfPeer.GetAddress() // Add self to peerstore as a control to ensure existing peers persist after // removing the target peer. - err = pstore.AddPeer(&typesP2P.NetworkPeer{Address: selfAddr}) + err := pstore.AddPeer(selfPeer) require.NoError(t, err) expectedPStoreSize++ @@ -77,10 +95,12 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { peerstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) - network := NewRainTreeNetwork(selfAddr, busMock, peerstoreProviderMock, currentHeightProviderMock).(*rainTreeNetwork) + network, err := NewRainTreeNetwork(host, selfAddr, busMock, peerstoreProviderMock, currentHeightProviderMock) + require.NoError(t, err) + rainTree := network.(*rainTreeNetwork) // Ensure expected starting size / lengths are consistent. - peerAddrs, peers := getPeersViewParts(network.peersManager) + peerAddrs, peers := getPeersViewParts(rainTree.peersManager) require.Equal(t, expectedPStoreSize, pstore.Size()) require.Equal(t, expectedPStoreSize, len(peerAddrs)) require.Equal(t, expectedPStoreSize, len(peers)) @@ -98,14 +118,14 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { require.NotNil(t, peerToRemove, "did not find selfAddr in peerstore") // Remove peerToRemove - err = network.RemovePeer(peerToRemove) + err = rainTree.RemovePeer(peerToRemove) require.NoError(t, err) expectedPStoreSize-- - peerAddrs, peers = getPeersViewParts(network.peersManager) + peerAddrs, peers = getPeersViewParts(rainTree.peersManager) removedAddr := peerToRemove.GetAddress() getPeer := func(addr cryptoPocket.Address) typesP2P.Peer { - return network.GetPeerstore().GetPeer(addr) + return rainTree.GetPeerstore().GetPeer(addr) } // Ensure updated sizes are consistent. @@ -127,3 +147,31 @@ func getPeersViewParts(pm typesP2P.PeerManager) ( return addrs, peers } + +func newTestPeer(t *testing.T) (*typesP2P.NetworkPeer, libp2pHost.Host) { + selfPrivKey, err := cryptoPocket.GeneratePrivateKey() + require.NoError(t, err) + + selfAddr := selfPrivKey.Address() + selfPeer := &typesP2P.NetworkPeer{ + PublicKey: selfPrivKey.PublicKey(), + Address: selfAddr, + ServiceURL: "10.0.0.1:42069", + } + return selfPeer, newLibp2pMockNetHost(t, selfPrivKey, selfPeer) +} + +// TECHDEBT(#609): move & de-duplicate +func newLibp2pMockNetHost(t *testing.T, privKey cryptoPocket.PrivateKey, peer *typesP2P.NetworkPeer) libp2pHost.Host { + libp2pPrivKey, err := libp2pCrypto.UnmarshalEd25519PrivateKey(privKey.Bytes()) + require.NoError(t, err) + + libp2pMultiAddr, err := utils.Libp2pMultiaddrFromServiceURL(peer.ServiceURL) + require.NoError(t, err) + + libp2pMockNet := mocknet.New() + host, err := libp2pMockNet.AddPeer(libp2pPrivKey, libp2pMultiAddr) + require.NoError(t, err) + + return host +} diff --git a/p2p/raintree/peers_manager_test.go b/p2p/raintree/peers_manager_test.go index 09b011370..30e82ca87 100644 --- a/p2p/raintree/peers_manager_test.go +++ b/p2p/raintree/peers_manager_test.go @@ -3,20 +3,27 @@ package raintree import ( "encoding/hex" "fmt" + "net" + "net/url" "strings" "testing" + "github.com/foxcpp/go-mockdns" "github.com/golang/mock/gomock" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" - "github.com/pokt-network/pocket/p2p/types" + typesP2P "github.com/pokt-network/pocket/p2p/types" + mocksP2P "github.com/pokt-network/pocket/p2p/types/mocks" "github.com/pokt-network/pocket/runtime/configs" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" - "github.com/stretchr/testify/require" ) const ( - serviceURLFormat = "val_%d" + serviceURLFormat = "val_%d:42069" + addrAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ[" ) type ExpectedRainTreeNetworkConfig struct { @@ -38,7 +45,7 @@ type ExpectedRainTreeMessageProp struct { func TestRainTree_Peerstore_HandleUpdate(t *testing.T) { ctrl := gomock.NewController(t) - addr, err := cryptoPocket.GenerateAddress() + pubKey, err := cryptoPocket.GeneratePublicKey() require.NoError(t, err) testCases := []ExpectedRainTreeNetworkConfig{ @@ -74,16 +81,33 @@ func TestRainTree_Peerstore_HandleUpdate(t *testing.T) { n := testCase.numNodes t.Run(fmt.Sprintf("n=%d", n), func(t *testing.T) { pstore := getPeerstore(t, n-1) - err = pstore.AddPeer(&types.NetworkPeer{Address: addr}) + + err = pstore.AddPeer(&typesP2P.NetworkPeer{ + PublicKey: pubKey, + Address: pubKey.Address(), + ServiceURL: "10.0.0.1:42069", + }) require.NoError(t, err) mockBus := mockBus(ctrl) pstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) - network := NewRainTreeNetwork(addr, mockBus, pstoreProviderMock, currentHeightProviderMock).(*rainTreeNetwork) + libp2pMockNet, err := mocknet.WithNPeers(1) + require.NoError(t, err) + + network, err := NewRainTreeNetwork( + libp2pMockNet.Hosts()[0], + pubKey.Address(), + mockBus, + pstoreProviderMock, + currentHeightProviderMock, + ) + require.NoError(t, err) + + rainTree := network.(*rainTreeNetwork) - peersManagerStateView, actualMaxNumLevels := network.peersManager.getPeersViewWithLevels() + peersManagerStateView, actualMaxNumLevels := rainTree.peersManager.getPeersViewWithLevels() require.Equal(t, n, network.GetPeerstore().Size()) require.Len(t, peersManagerStateView.GetAddrs(), n) @@ -121,16 +145,21 @@ func BenchmarkPeerstoreUpdates(b *testing.B) { n := testCase.numNodes b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { pstore := getPeerstore(nil, n-1) - err := pstore.AddPeer(&types.NetworkPeer{Address: addr}) + err := pstore.AddPeer(&typesP2P.NetworkPeer{Address: addr}) require.NoError(b, err) mockBus := mockBus(ctrl) pstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) - network := NewRainTreeNetwork(addr, mockBus, pstoreProviderMock, currentHeightProviderMock).(*rainTreeNetwork) + hostMock := mocksP2P.NewMockHost(ctrl) - peersManagerStateView, actualMaxNumLevels := network.peersManager.getPeersViewWithLevels() + network, err := NewRainTreeNetwork(hostMock, addr, mockBus, pstoreProviderMock, currentHeightProviderMock) + require.NoError(b, err) + + rainTree := network.(*rainTreeNetwork) + + peersManagerStateView, actualMaxNumLevels := rainTree.peersManager.getPeersViewWithLevels() require.Equal(b, n, network.GetPeerstore().Size()) require.Equal(b, n, len(peersManagerStateView.GetAddrs())) @@ -140,11 +169,11 @@ func BenchmarkPeerstoreUpdates(b *testing.B) { for i := 0; i < numAddressesToBeAdded; i++ { newAddr, err := cryptoPocket.GenerateAddress() require.NoError(b, err) - err = network.AddPeer(&types.NetworkPeer{Address: newAddr}) + err = rainTree.AddPeer(&typesP2P.NetworkPeer{Address: newAddr}) require.NoError(b, err) } - peersManagerStateView = network.peersManager.GetPeersView() + peersManagerStateView = rainTree.peersManager.GetPeersView() require.Equal(b, n+numAddressesToBeAdded, network.GetPeerstore().Size()) require.Equal(b, n+numAddressesToBeAdded, len(peersManagerStateView.GetAddrs())) @@ -156,12 +185,15 @@ func BenchmarkPeerstoreUpdates(b *testing.B) { func getPeerstore(t *testing.T, n int) typesP2P.Peerstore { pstore := make(typesP2P.PeerAddrMap) for i := 0; i < n; i++ { - addr, err := cryptoPocket.GenerateAddress() + privKey, err := cryptoPocket.GeneratePrivateKey() if t != nil { require.NoError(t, err) } - err = pstore.AddPeer(&types.NetworkPeer{ - Address: addr, + + err = pstore.AddPeer(&typesP2P.NetworkPeer{ + PublicKey: privKey.PublicKey(), + Address: privKey.Address(), + ServiceURL: "10.0.0.1:42069", }) if t != nil { require.NoError(t, err) @@ -224,24 +256,39 @@ func testRainTreeMessageTargets(t *testing.T, expectedMsgProp *ExpectedRainTreeM busMock.EXPECT().GetRuntimeMgr().Return(runtimeMgrMock).AnyTimes() runtimeMgrMock.EXPECT().GetConfig().Return(configs.NewDefaultConfig()).AnyTimes() + mockAlphabetValidatorServiceURLsDNS(t) pstore := getAlphabetPeerstore(t, expectedMsgProp.numNodes) pstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 1) - network := NewRainTreeNetwork([]byte{expectedMsgProp.orig}, busMock, pstoreProviderMock, currentHeightProviderMock).(*rainTreeNetwork) + libp2pPStore, err := pstoremem.NewPeerstore() + require.NoError(t, err) + + hostMock := mocksP2P.NewMockHost(ctrl) + hostMock.EXPECT().Peerstore().Return(libp2pPStore).AnyTimes() + + network, err := NewRainTreeNetwork( + hostMock, + []byte{expectedMsgProp.orig}, + busMock, + pstoreProviderMock, + currentHeightProviderMock, + ) + require.NoError(t, err) + rainTree := network.(*rainTreeNetwork) - network.SetBus(busMock) + rainTree.SetBus(busMock) - peersManagerStateView := network.peersManager.GetPeersView() + peersManagerStateView := rainTree.peersManager.GetPeersView() require.Equal(t, strings.Join(peersManagerStateView.GetAddrs(), ""), strToAddrList(expectedMsgProp.addrList)) - i, found := network.peersManager.getSelfIndexInPeersView() + i, found := rainTree.peersManager.getSelfIndexInPeersView() require.True(t, found) require.Equal(t, i, 0) for _, target := range expectedMsgProp.targets { - actualTargets := network.getTargetsAtLevel(uint32(target.level)) + actualTargets := rainTree.getTargetsAtLevel(uint32(target.level)) require.True(t, shouldSendToTarget(actualTargets[0])) require.Equal(t, cryptoPocket.Address(target.left), actualTargets[0].address) @@ -254,11 +301,15 @@ func testRainTreeMessageTargets(t *testing.T, expectedMsgProp *ExpectedRainTreeM // Generates an address book with a constant set 27 addresses; ['A', ..., 'Z'] func getAlphabetPeerstore(t *testing.T, n int) typesP2P.Peerstore { pstore := make(typesP2P.PeerAddrMap) - for i, ch := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ[" { + for i, ch := range addrAlphabet { if i >= n { return pstore } - err := pstore.AddPeer(&types.NetworkPeer{ + pubKey, err := cryptoPocket.GeneratePublicKey() + require.NoError(t, err) + + err = pstore.AddPeer(&typesP2P.NetworkPeer{ + PublicKey: pubKey, ServiceURL: fmt.Sprintf(serviceURLFormat, i), Address: []byte{byte(ch)}, }) @@ -270,3 +321,34 @@ func getAlphabetPeerstore(t *testing.T, n int) typesP2P.Peerstore { func strToAddrList(s string) string { return hex.EncodeToString([]byte(s)) } + +func mockAlphabetValidatorServiceURLsDNS(t *testing.T) (done func()) { + zones := make(map[string]mockdns.Zone) + for i := range addrAlphabet { + serviceURL, err := url.Parse(fmt.Sprintf("scheme://"+serviceURLFormat, i)) + require.NoError(t, err) + + fqdn := fmt.Sprintf("%s.", serviceURL.Hostname()) + zones[fqdn] = mockdns.Zone{ + A: []string{fmt.Sprintf("10.0.0.%d", i+1)}, + } + } + return prepareDNSMock(zones) +} + +// TECHDEBT(#609): de-duplicate / refactor `prepaand reDNSMock` & `noopLogger`. +func prepareDNSMock(zones map[string]mockdns.Zone) (done func()) { + srv, _ := mockdns.NewServerWithLogger(zones, noopLogger{}, false) + srv.PatchNet(net.DefaultResolver) + return func() { + _ = srv.Close() + mockdns.UnpatchNet(net.DefaultResolver) + } +} + +// noopLogger implements go-mockdns's `mockdns.Logger` interface. +type noopLogger struct{} + +func (nl noopLogger) Printf(format string, args ...interface{}) { + // noop +} From 498229a5a524f3ace816745c9a0227f2326c8b84 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 22:52:10 +0100 Subject: [PATCH 19/82] chore: remove unused `Transport` interface and related --- .../peerstore_provider/peerstore_provider.go | 19 -- .../persistence/provider.go | 12 -- .../peerstore_provider/rpc/provider.go | 14 +- p2p/transport/transport.go | 172 ------------------ p2p/transport/transport_test.go | 59 ------ p2p/types/network_peer.go | 7 - p2p/types/peer.go | 5 - p2p/utils/peer_conversion.go | 3 - 8 files changed, 1 insertion(+), 290 deletions(-) delete mode 100644 p2p/transport/transport.go delete mode 100644 p2p/transport/transport_test.go diff --git a/p2p/providers/peerstore_provider/peerstore_provider.go b/p2p/providers/peerstore_provider/peerstore_provider.go index 47059c096..0e20a3b01 100644 --- a/p2p/providers/peerstore_provider/peerstore_provider.go +++ b/p2p/providers/peerstore_provider/peerstore_provider.go @@ -3,8 +3,6 @@ package peerstore_provider //go:generate mockgen -source=$GOFILE -destination=../../types/mocks/peerstore_provider_mock.go -package=mock_types github.com/pokt-network/pocket/p2p/types PeerstoreProvider import ( - "fmt" - "go.uber.org/multierr" typesP2P "github.com/pokt-network/pocket/p2p/types" @@ -21,9 +19,7 @@ type PeerstoreProvider interface { modules.Module GetStakedPeerstoreAtHeight(height uint64) (typesP2P.Peerstore, error) - GetConnFactory() typesP2P.ConnectionFactory GetP2PConfig() *configs.P2PConfig - SetConnectionFactory(typesP2P.ConnectionFactory) } func ActorsToPeerstore(abp PeerstoreProvider, actors []*coreTypes.Actor) (pstore typesP2P.Peerstore, errs error) { @@ -43,20 +39,12 @@ func ActorsToPeerstore(abp PeerstoreProvider, actors []*coreTypes.Actor) (pstore } func ActorToPeer(abp PeerstoreProvider, actor *coreTypes.Actor) (typesP2P.Peer, error) { - // TECHDEBT(#576): this should be the responsibility of some new `ConnManager` interface. - // Peerstore (identity / address mapping) is a separate concern from managing - // connections to/from peers. - conn, err := abp.GetConnFactory()(abp.GetP2PConfig(), actor.GetServiceUrl()) // generic param is service url - if err != nil { - return nil, fmt.Errorf("error resolving addr: %v", err) - } pubKey, err := cryptoPocket.NewPublicKey(actor.GetPublicKey()) if err != nil { return nil, err } peer := &typesP2P.NetworkPeer{ - Transport: conn, PublicKey: pubKey, Address: pubKey.Address(), ServiceURL: actor.GetServiceUrl(), // service url @@ -64,10 +52,3 @@ func ActorToPeer(abp PeerstoreProvider, actor *coreTypes.Actor) (typesP2P.Peer, return peer, nil } - -// WithConnectionFactory allows the user to specify a custom connection factory -func WithConnectionFactory(connFactory typesP2P.ConnectionFactory) func(PeerstoreProvider) { - return func(ap PeerstoreProvider) { - ap.SetConnectionFactory(connFactory) - } -} diff --git a/p2p/providers/peerstore_provider/persistence/provider.go b/p2p/providers/peerstore_provider/persistence/provider.go index 02e839687..c9c376064 100644 --- a/p2p/providers/peerstore_provider/persistence/provider.go +++ b/p2p/providers/peerstore_provider/persistence/provider.go @@ -2,7 +2,6 @@ package persistence import ( "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" - "github.com/pokt-network/pocket/p2p/transport" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/shared/modules" @@ -14,14 +13,11 @@ var _ peerstore_provider.PeerstoreProvider = &persistencePeerstoreProvider{} type persistencePeerstoreProvider struct { base_modules.IntegratableModule base_modules.InterruptableModule - - connFactory typesP2P.ConnectionFactory } func NewPersistencePeerstoreProvider(bus modules.Bus, options ...func(*persistencePeerstoreProvider)) *persistencePeerstoreProvider { pabp := &persistencePeerstoreProvider{ IntegratableModule: *base_modules.NewIntegratableModule(bus), - connFactory: transport.CreateDialer, // default connection factory, overridable with WithConnectionFactory() } for _, o := range options { @@ -57,14 +53,6 @@ func (pabp *persistencePeerstoreProvider) GetStakedPeerstoreAtHeight(height uint return peerstore_provider.ActorsToPeerstore(pabp, validators) } -func (pabp *persistencePeerstoreProvider) GetConnFactory() typesP2P.ConnectionFactory { - return pabp.connFactory -} - func (pabp *persistencePeerstoreProvider) GetP2PConfig() *configs.P2PConfig { return pabp.GetBus().GetRuntimeMgr().GetConfig().P2P } - -func (pabp *persistencePeerstoreProvider) SetConnectionFactory(connFactory typesP2P.ConnectionFactory) { - pabp.connFactory = connFactory -} diff --git a/p2p/providers/peerstore_provider/rpc/provider.go b/p2p/providers/peerstore_provider/rpc/provider.go index b8798e50e..bd6a70d58 100644 --- a/p2p/providers/peerstore_provider/rpc/provider.go +++ b/p2p/providers/peerstore_provider/rpc/provider.go @@ -8,7 +8,6 @@ import ( "time" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" - "github.com/pokt-network/pocket/p2p/transport" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/rpc" "github.com/pokt-network/pocket/runtime" @@ -36,14 +35,11 @@ type rpcPeerstoreProvider struct { rpcURL string p2pCfg *configs.P2PConfig rpcClient *rpc.ClientWithResponses - - connFactory typesP2P.ConnectionFactory } func NewRPCPeerstoreProvider(options ...modules.ModuleOption) *rpcPeerstoreProvider { rabp := &rpcPeerstoreProvider{ - rpcURL: fmt.Sprintf("http://%s:%s", rpcHost, defaults.DefaultRPCPort), // TODO: Make port configurable - connFactory: transport.CreateDialer, // default connection factory, overridable with WithConnectionFactory() + rpcURL: fmt.Sprintf("http://%s:%s", rpcHost, defaults.DefaultRPCPort), // TODO: Make port configurable } for _, o := range options { @@ -98,10 +94,6 @@ func (rabp *rpcPeerstoreProvider) GetStakedPeerstoreAtHeight(height uint64) (typ return peerstore_provider.ActorsToPeerstore(rabp, coreActors) } -func (rabp *rpcPeerstoreProvider) GetConnFactory() typesP2P.ConnectionFactory { - return rabp.connFactory -} - func (rabp *rpcPeerstoreProvider) GetP2PConfig() *configs.P2PConfig { if rabp.p2pCfg == nil { return rabp.GetBus().GetRuntimeMgr().GetConfig().P2P @@ -109,10 +101,6 @@ func (rabp *rpcPeerstoreProvider) GetP2PConfig() *configs.P2PConfig { return rabp.p2pCfg } -func (rabp *rpcPeerstoreProvider) SetConnectionFactory(connFactory typesP2P.ConnectionFactory) { - rabp.connFactory = connFactory -} - func (rabp *rpcPeerstoreProvider) initRPCClient() { rpcClient, err := rpc.NewClientWithResponses(rabp.rpcURL) if err != nil { diff --git a/p2p/transport/transport.go b/p2p/transport/transport.go deleted file mode 100644 index c49c4b05f..000000000 --- a/p2p/transport/transport.go +++ /dev/null @@ -1,172 +0,0 @@ -package transport - -import ( - "fmt" - "io" - "net" - "sync" - - typesP2P "github.com/pokt-network/pocket/p2p/types" - "github.com/pokt-network/pocket/runtime/configs" - typesConfigs "github.com/pokt-network/pocket/runtime/configs/types" -) - -const ( - TCPNetworkLayerProtocol = "tcp4" -) - -func CreateListener(cfg *configs.P2PConfig) (typesP2P.Transport, error) { - switch cfg.ConnectionType { - case typesConfigs.ConnectionType_EmptyConnection: - return createEmptyListener(cfg) - case typesConfigs.ConnectionType_TCPConnection: - return createTCPListener(cfg) - default: - return nil, fmt.Errorf("unsupported connection type for listener: %v", cfg.ConnectionType) - } -} - -func CreateDialer(cfg *configs.P2PConfig, url string) (typesP2P.Transport, error) { - switch cfg.ConnectionType { - case typesConfigs.ConnectionType_EmptyConnection: - return createEmptyDialer(cfg, url) - case typesConfigs.ConnectionType_TCPConnection: - return createTCPDialer(cfg, url) - default: - return nil, fmt.Errorf("unsupported connection type for dialer: %v", cfg.ConnectionType) - } -} - -var _ typesP2P.Transport = &tcpConn{} - -type tcpConn struct { - address *net.TCPAddr - listener *net.TCPListener - - muReadAll sync.Mutex - muRead sync.Mutex - conn net.Conn -} - -func createTCPListener(cfg *configs.P2PConfig) (*tcpConn, error) { - addr, err := net.ResolveTCPAddr(TCPNetworkLayerProtocol, fmt.Sprintf("%s:%d", cfg.Hostname, cfg.Port)) - if err != nil { - return nil, err - } - l, err := net.ListenTCP(TCPNetworkLayerProtocol, addr) - if err != nil { - return nil, err - } - return &tcpConn{ - address: addr, - listener: l, - }, nil -} - -func createTCPDialer(_ *configs.P2PConfig, url string) (*tcpConn, error) { - addr, err := net.ResolveTCPAddr(TCPNetworkLayerProtocol, url) - if err != nil { - return nil, err - } - return &tcpConn{ - address: addr, - }, nil -} - -func (c *tcpConn) IsListener() bool { - return c.listener != nil -} - -func (c *tcpConn) ReadAll() ([]byte, error) { - if !c.IsListener() { - return nil, fmt.Errorf("connection is not a listener") - } - - conn, err := c.listener.Accept() - if err != nil { - return nil, fmt.Errorf("error accepting connection: %v", err) - } - defer conn.Close() - - c.muReadAll.Lock() - defer c.muReadAll.Unlock() - - c.conn = conn - return io.ReadAll(c) -} - -// Read implements the respective member in the io.Reader interface. -// TECHDEBT (#554): Read in this implementation is not intended to be -// called directly and will return an error if `tcpConn.conn` is `nil`. -func (c *tcpConn) Read(buf []byte) (int, error) { - c.muRead.Lock() - defer c.muRead.Unlock() - - if c.conn == nil { - return 0, fmt.Errorf("no connection accepted on listener") - } - - numBz, err := c.conn.Read(buf) - if err != nil { - if err == io.EOF { - return numBz, io.EOF - } - return 0, fmt.Errorf("error reading from conn: %v", err) - } - - return numBz, nil -} - -func (c *tcpConn) Write(data []byte) (int, error) { - if c.IsListener() { - return 0, fmt.Errorf("connection is a listener") - } - - client, err := net.DialTCP(TCPNetworkLayerProtocol, nil, c.address) - if err != nil { - return 0, err - } - defer client.Close() - - return client.Write(data) -} - -func (c *tcpConn) Close() error { - if c.IsListener() { - return c.listener.Close() - } - return nil -} - -var _ typesP2P.Transport = &emptyConn{} - -type emptyConn struct { -} - -func createEmptyListener(_ *configs.P2PConfig) (typesP2P.Transport, error) { - return &emptyConn{}, nil -} - -func createEmptyDialer(_ *configs.P2PConfig, _ string) (typesP2P.Transport, error) { - return &emptyConn{}, nil -} - -func (c *emptyConn) IsListener() bool { - return false -} - -func (c *emptyConn) ReadAll() ([]byte, error) { - return nil, nil -} - -func (c *emptyConn) Read(data []byte) (int, error) { - return 0, nil -} - -func (c *emptyConn) Write(data []byte) (int, error) { - return 0, nil -} - -func (c *emptyConn) Close() error { - return nil -} diff --git a/p2p/transport/transport_test.go b/p2p/transport/transport_test.go deleted file mode 100644 index 5df0af758..000000000 --- a/p2p/transport/transport_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package transport - -import ( - "testing" - - "github.com/pokt-network/pocket/runtime/configs" - "github.com/pokt-network/pocket/runtime/configs/types" - "github.com/pokt-network/pocket/runtime/defaults" - "github.com/pokt-network/pocket/shared/crypto" - "github.com/stretchr/testify/require" -) - -// localhostName represents an IPv4 address on the loopback interface -const localhostName = "127.0.0.1" - -func TestTcpConn_ReadAll(t *testing.T) { - expectedData := []byte("testing 123") - receiver := newTestReceiver(t, localhostName, int(defaults.DefaultP2PPort)) - sender := newTestSender(t, receiver.address.String()) - - // Send via `Write` - numBzWritten, err := sender.Write(expectedData) - require.NoError(t, err) - require.Equal(t, len(expectedData), numBzWritten) - - // Receive via `ReadAll` - actualData, err := receiver.ReadAll() - require.NoError(t, err) - require.Equal(t, expectedData, actualData) -} - -func newTestReceiver(t *testing.T, hostname string, port int) *tcpConn { - pk, err := crypto.GeneratePrivateKey() - require.NoError(t, err) - - receiver, err := createTCPListener(&configs.P2PConfig{ - PrivateKey: pk.String(), - Hostname: hostname, - Port: uint32(port), - UseRainTree: false, - ConnectionType: types.ConnectionType_TCPConnection, - }) - require.NoError(t, err) - return receiver -} - -func newTestSender(t *testing.T, receiverURL string) *tcpConn { - pk, err := crypto.GeneratePrivateKey() - require.NoError(t, err) - - sender, err := createTCPDialer(&configs.P2PConfig{ - PrivateKey: pk.String(), - UseRainTree: false, - IsClientOnly: true, - ConnectionType: types.ConnectionType_TCPConnection, - }, receiverURL) - require.NoError(t, err) - return sender -} diff --git a/p2p/types/network_peer.go b/p2p/types/network_peer.go index e6a2e06c4..8b66ae7fb 100644 --- a/p2p/types/network_peer.go +++ b/p2p/types/network_peer.go @@ -1,8 +1,6 @@ package types import ( - "io" - "github.com/multiformats/go-multiaddr" "github.com/pokt-network/pocket/shared/crypto" @@ -11,7 +9,6 @@ import ( var _ Peer = &NetworkPeer{} type NetworkPeer struct { - Transport Transport PublicKey crypto.PublicKey Address crypto.Address Multiaddr multiaddr.Multiaddr @@ -24,10 +21,6 @@ func (peer *NetworkPeer) GetAddress() crypto.Address { return peer.Address } -func (peer *NetworkPeer) GetStream() io.ReadWriteCloser { - return peer.Transport -} - func (peer *NetworkPeer) GetPublicKey() crypto.PublicKey { return peer.PublicKey } diff --git a/p2p/types/peer.go b/p2p/types/peer.go index 6b2cd84a0..0385beef8 100644 --- a/p2p/types/peer.go +++ b/p2p/types/peer.go @@ -1,8 +1,6 @@ package types import ( - "io" - "github.com/pokt-network/pocket/shared/crypto" ) @@ -10,9 +8,6 @@ type Peer interface { GetAddress() crypto.Address GetPublicKey() crypto.PublicKey GetServiceURL() string - - // TECHDEBT(#576): move this to some new `ConnManager` interface. - GetStream() io.ReadWriteCloser } // PeerList is a convenience type for operating on a slice of `Peer`s. diff --git a/p2p/utils/peer_conversion.go b/p2p/utils/peer_conversion.go index 2268a8c4e..102d1221b 100644 --- a/p2p/utils/peer_conversion.go +++ b/p2p/utils/peer_conversion.go @@ -2,13 +2,11 @@ package utils import ( "fmt" - libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" libp2pPeer "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" - "github.com/pokt-network/pocket/libp2p/transport" "github.com/pokt-network/pocket/p2p/types" typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/crypto" @@ -33,7 +31,6 @@ func PeerFromLibp2pStream(stream network.Stream) (typesP2P.Peer, error) { } return &types.NetworkPeer{ - Transport: transport.NewLibP2PTransport(stream), PublicKey: publicKey, Address: publicKey.Address(), Multiaddr: peerMultiaddr, From b757af8860af28984d747de24aa906888b5c8e17 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Sat, 18 Mar 2023 23:45:06 +0100 Subject: [PATCH 20/82] chore: update changelog --- app/client/doc/CHANGELOG.md | 4 ++++ p2p/CHANGELOG.md | 13 +++++++++++++ runtime/docs/CHANGELOG.md | 4 ++++ shared/CHANGELOG.md | 4 ++++ 4 files changed, 25 insertions(+) diff --git a/app/client/doc/CHANGELOG.md b/app/client/doc/CHANGELOG.md index 7ee0c611e..ded7cf7f2 100644 --- a/app/client/doc/CHANGELOG.md +++ b/app/client/doc/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.24] - 2023-03-24 + +- Refactor debug CLI post P2P module re-consolidation + ## [0.0.0.23] - 2023-03-21 - Refactor debug CLI to use new P2P interfaces diff --git a/p2p/CHANGELOG.md b/p2p/CHANGELOG.md index 1b6281d85..63d75d09e 100644 --- a/p2p/CHANGELOG.md +++ b/p2p/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.38] - 2023-03-24 + +- Moved peer & url conversion utils to `p2p/utils` package +- Refactor `getPeerIP` to use `net.DefaultResolver` for easier testing +- Moved & refactor libp2p `host.Host` setup util to `p2p/utils` +- Consolidated Libp2p & P2P `modules.Module` implementations +- Consolidated Libp2p & P2P `stdnetwork` `typesP2P.Network` implementations +- Refactored raintree `typesP2P.Network` implementation to use libp2p +- Moved `shared/p2p` package into `p2p/types` packages +- Removed `Trnasport` interface and implementations +- Removed `ConnectionFactory` type and related members +- Added libp2p `host.Host` mock generator + ## [0.0.0.37] - 2023-03-23 - Wrap IPv6 address in square brackets as per RFC3986 §3.2.2 diff --git a/runtime/docs/CHANGELOG.md b/runtime/docs/CHANGELOG.md index af8b89759..6e3e99af8 100644 --- a/runtime/docs/CHANGELOG.md +++ b/runtime/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.26] - 2023-03-24 + +- Removed `runtime/configs.Config#UseLibp2p` field + ## [0.0.0.25] - 2023-03-01 - replace `consensus_port` with `port` in P2P config diff --git a/shared/CHANGELOG.md b/shared/CHANGELOG.md index 448eba04c..a625eacf6 100644 --- a/shared/CHANGELOG.md +++ b/shared/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.43] - 2023-03-24 + +- Removed *temporary* `shared/p2p` package; consolidated into `p2p` + ## [0.0.0.42] - 2023-03-21 - Add `TransitionEventToMap()` helper function for logging From 1e120cbb2ca56fbaa3b8fa8aa53c63875ce7f5cd Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:18:12 +0200 Subject: [PATCH 21/82] test: port libp2p network test to stdnetwork pkg --- p2p/stdnetwork/network_test.go | 159 +++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 p2p/stdnetwork/network_test.go diff --git a/p2p/stdnetwork/network_test.go b/p2p/stdnetwork/network_test.go new file mode 100644 index 000000000..f19c7334d --- /dev/null +++ b/p2p/stdnetwork/network_test.go @@ -0,0 +1,159 @@ +package stdnetwork + +import ( + "fmt" + "github.com/pokt-network/pocket/runtime/defaults" + "testing" + + "github.com/golang/mock/gomock" + libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" + libp2pHost "github.com/libp2p/go-libp2p/core/host" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" + + typesP2P "github.com/pokt-network/pocket/p2p/types" + mock_types "github.com/pokt-network/pocket/p2p/types/mocks" + "github.com/pokt-network/pocket/p2p/utils" + cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + mockModules "github.com/pokt-network/pocket/shared/modules/mocks" +) + +// https://www.rfc-editor.org/rfc/rfc3986#section-3.2.2 +const testIP6ServiceURL = "[2a00:1450:4005:802::2004]:8080" + +// TECHDEBT(#609): move & de-dup. +var testLocalServiceURL = fmt.Sprintf("127.0.0.1:%d", defaults.DefaultP2PPort) + +func TestLibp2pNetwork_AddPeer(t *testing.T) { + p2pNet := newTestLibp2pNetwork(t) + libp2pPStore := p2pNet.host.Peerstore() + + // NB: assert initial state + require.Equal(t, 1, p2pNet.pstore.Size()) + + existingPeer := p2pNet.pstore.GetPeerList()[0] + require.NotNil(t, existingPeer) + + existingPeerInfo, err := utils.Libp2pAddrInfoFromPeer(existingPeer) + require.NoError(t, err) + + existingPeerstoreAddrs := libp2pPStore.Addrs(existingPeerInfo.ID) + require.Len(t, existingPeerstoreAddrs, 1) + + existingPeerMultiaddr, err := utils.Libp2pMultiaddrFromServiceURL(existingPeer.GetServiceURL()) + require.NoError(t, err) + require.Equal(t, existingPeerstoreAddrs[0].String(), existingPeerMultiaddr.String()) + + newPublicKey, err := cryptoPocket.GeneratePublicKey() + newPoktAddr := newPublicKey.Address() + require.NoError(t, err) + + newPeer := &typesP2P.NetworkPeer{ + PublicKey: newPublicKey, + Address: newPoktAddr, + ServiceURL: testIP6ServiceURL, + } + newPeerInfo, err := utils.Libp2pAddrInfoFromPeer(newPeer) + require.NoError(t, err) + newPeerMultiaddr := newPeerInfo.Addrs[0] + + // NB: add to address book + err = p2pNet.AddPeer(newPeer) + require.NoError(t, err) + + require.Len(t, p2pNet.pstore, 2) + require.Equal(t, p2pNet.pstore.GetPeer(existingPeer.GetAddress()), existingPeer) + require.Equal(t, p2pNet.pstore.GetPeer(newPeer.Address), newPeer) + + existingPeerstoreAddrs = libp2pPStore.Addrs(existingPeerInfo.ID) + newPeerstoreAddrs := libp2pPStore.Addrs(newPeerInfo.ID) + require.Len(t, existingPeerstoreAddrs, 1) + require.Len(t, newPeerstoreAddrs, 1) + require.Equal(t, newPeerstoreAddrs[0].String(), newPeerMultiaddr.String()) +} + +func TestLibp2pNetwork_RemovePeer(t *testing.T) { + p2pNet := newTestLibp2pNetwork(t) + peerstore := p2pNet.host.Peerstore() + + // NB: assert initial state + require.Len(t, p2pNet.pstore, 1) + + existingPeer := p2pNet.pstore.GetPeerList()[0] + require.NotNil(t, existingPeer) + + existingPeerInfo, err := utils.Libp2pAddrInfoFromPeer(existingPeer) + require.NoError(t, err) + + existingPeerstoreAddrs := peerstore.Addrs(existingPeerInfo.ID) + require.Len(t, existingPeerstoreAddrs, 1) + + existingPeerMultiaddr, err := utils.Libp2pMultiaddrFromServiceURL(existingPeer.GetServiceURL()) + require.NoError(t, err) + require.Equal(t, existingPeerstoreAddrs[0].String(), existingPeerMultiaddr.String()) + + err = p2pNet.RemovePeer(existingPeer) + require.NoError(t, err) + + require.Len(t, p2pNet.pstore, 0) + + // NB: libp2p peerstore implementations only remove peer keys and metadata + // but continue to resolve multiaddrs until their respective TTLs expire. + // (see: https://github.com/libp2p/go-libp2p/blob/v0.25.1/p2p/host/peerstore/pstoremem/peerstore.go#L108) + // (see: https://github.com/libp2p/go-libp2p/blob/v0.25.1/p2p/host/peerstore/pstoreds/peerstore.go#L187) + + existingPeerstoreAddrs = peerstore.Addrs(existingPeerInfo.ID) + require.Len(t, existingPeerstoreAddrs, 1) +} + +// TECHDEBT(#609): move & de-duplicate +func newTestLibp2pNetwork(t *testing.T) *network { + ctrl := gomock.NewController(t) + consensusMock := mockModules.NewMockConsensusModule(ctrl) + consensusMock.EXPECT().CurrentHeight().Return(uint64(1)).AnyTimes() + + pstore := make(typesP2P.PeerAddrMap) + pstoreProviderMock := mock_types.NewMockPeerstoreProvider(ctrl) + pstoreProviderMock.EXPECT().GetStakedPeerstoreAtHeight(gomock.Any()).Return(pstore, nil).AnyTimes() + + privKey, err := cryptoPocket.GeneratePrivateKey() + require.NoError(t, err) + + selfPeer := &typesP2P.NetworkPeer{ + PublicKey: privKey.PublicKey(), + Address: privKey.Address(), + ServiceURL: testLocalServiceURL, + } + err = pstore.AddPeer(selfPeer) + require.NoError(t, err) + + host := newLibp2pMockNetHost(t, privKey, selfPeer) + defer host.Close() + + p2pNetwork, err := NewNetwork( + host, + pstoreProviderMock, + consensusMock, + ) + require.NoError(t, err) + + libp2pNet, ok := p2pNetwork.(*network) + require.Truef(t, ok, "unexpected p2pNetwork type: %T", p2pNetwork) + + return libp2pNet +} + +// TECHDEBT(#609): move & de-duplicate +func newLibp2pMockNetHost(t *testing.T, privKey cryptoPocket.PrivateKey, peer *typesP2P.NetworkPeer) libp2pHost.Host { + libp2pPrivKey, err := libp2pCrypto.UnmarshalEd25519PrivateKey(privKey.Bytes()) + require.NoError(t, err) + + libp2pMultiAddr, err := utils.Libp2pMultiaddrFromServiceURL(peer.ServiceURL) + require.NoError(t, err) + + libp2pMockNet := mocknet.New() + host, err := libp2pMockNet.AddPeer(libp2pPrivKey, libp2pMultiAddr) + require.NoError(t, err) + + return host +} From 8bf12e7c53b343e668a93b98e97a9925eaa8dc91 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:20:21 +0200 Subject: [PATCH 22/82] chore: un-name error return --- p2p/stdnetwork/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/stdnetwork/network.go b/p2p/stdnetwork/network.go index 6d8ad3062..65524d287 100644 --- a/p2p/stdnetwork/network.go +++ b/p2p/stdnetwork/network.go @@ -57,7 +57,7 @@ func (n *network) NetworkBroadcast(data []byte) error { return nil } -func (n *network) NetworkSend(data []byte, address cryptoPocket.Address) (err error) { +func (n *network) NetworkSend(data []byte, address cryptoPocket.Address) error { peer := n.pstore.GetPeer(address) if peer == nil { return fmt.Errorf("peer with address %s not in peerstore", address) From a981bf025effb10a086531be04f4e6cb59839a01 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:26:19 +0200 Subject: [PATCH 23/82] fix: stdnetwork libp2p peerstore sync --- p2p/stdnetwork/network.go | 14 ++++++++ p2p/utils/host.go | 71 ++++++++++++++++++++++++++------------- 2 files changed, 61 insertions(+), 24 deletions(-) diff --git a/p2p/stdnetwork/network.go b/p2p/stdnetwork/network.go index 65524d287..5d4cd072a 100644 --- a/p2p/stdnetwork/network.go +++ b/p2p/stdnetwork/network.go @@ -78,10 +78,24 @@ func (n *network) GetPeerstore() typesP2P.Peerstore { } func (n *network) AddPeer(peer typesP2P.Peer) error { + // Noop if peer with the pokt address already exists in the peerstore. + // TECHDEBT: add method(s) to update peers. + if p := n.pstore.GetPeer(peer.GetAddress()); p != nil { + return nil + } + + if err := utils.AddPeerToLibp2pHost(n.host, peer); err != nil { + return err + } + return n.pstore.AddPeer(peer) } func (n *network) RemovePeer(peer typesP2P.Peer) error { + if err := utils.RemovePeerFromLibp2pHost(n.host, peer); err != nil { + return err + } + return n.pstore.RemovePeer(peer.GetAddress()) } diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 988a20ad4..018ed39fd 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -23,37 +23,60 @@ const ( // info for use with libp2p and adding it to the underlying libp2p host's peerstore. // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/host#Host) // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.2/core/peerstore#Peerstore) -func PopulateLibp2pHost(host libp2pHost.Host, pstore typesP2P.Peerstore) error { +func PopulateLibp2pHost(host libp2pHost.Host, pstore typesP2P.Peerstore) (err error) { for _, peer := range pstore.GetPeerList() { - pubKey, err := Libp2pPublicKeyFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer public key, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } - libp2pPeer, err := Libp2pAddrInfoFromPeer(peer) - if err != nil { - return fmt.Errorf( - "converting peer info, pokt address: %s: %w", - peer.GetAddress(), - err, - ) + if addErr := AddPeerToLibp2pHost(host, peer); addErr != nil { + err = multierr.Append(err, addErr) } + } + return err +} - host.Peerstore().AddAddrs(libp2pPeer.ID, libp2pPeer.Addrs, DefaultPeerTTL) - if err := host.Peerstore().AddPubKey(libp2pPeer.ID, pubKey); err != nil { - return fmt.Errorf( - "adding peer public key, pokt address: %s: %w", - peer.GetAddress(), - err, - ) - } +// AddPeerToLibp2pHost covnerts the given pocket peer for use with libp2p and adds +// it to the given libp2p host's underlying peerstore. +func AddPeerToLibp2pHost(host libp2pHost.Host, peer typesP2P.Peer) error { + pubKey, err := Libp2pPublicKeyFromPeer(peer) + if err != nil { + return fmt.Errorf( + "converting peer public key, pokt address: %s: %w", + peer.GetAddress(), + err, + ) + } + + libp2pPeer, err := Libp2pAddrInfoFromPeer(peer) + if err != nil { + return fmt.Errorf( + "converting peer info, pokt address: %s: %w", + peer.GetAddress(), + err, + ) + } + + host.Peerstore().AddAddrs(libp2pPeer.ID, libp2pPeer.Addrs, defaultPeerTTL) + if err := host.Peerstore().AddPubKey(libp2pPeer.ID, pubKey); err != nil { + return fmt.Errorf( + "adding peer public key, pokt address: %s: %w", + peer.GetAddress(), + err, + ) } return nil } +// RemovePeerFromLibp2pHost removes the given peer's libp2p public keys and +// protocols from the libp2p host's underlying peerstore. +func RemovePeerFromLibp2pHost(host libp2pHost.Host, peer typesP2P.Peer) error { + peerInfo, err := Libp2pAddrInfoFromPeer(peer) + if err != nil { + return err + } + + host.Peerstore().RemovePeer(peerInfo.ID) + return host.Peerstore().RemoveProtocols(peerInfo.ID) +} + +// Libp2pSendToPeer sends data to the given pocket peer from the given libp2p host. func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) error { // TECHDEBT(#595): add ctx to interface methods and propagate down. ctx := context.Background() From 311d7fc78342ae6eec2a5e524aa7c5f7e075797f Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:28:36 +0200 Subject: [PATCH 24/82] chore: unexport `utils.defaultPeerTTL` --- p2p/raintree/network_test.go | 3 ++- p2p/utils/host.go | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index 2e507043e..33dc75dea 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -2,6 +2,7 @@ package raintree import ( "testing" + "time" "github.com/golang/mock/gomock" libp2pCrypto "github.com/libp2p/go-libp2p/core/crypto" @@ -28,7 +29,7 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { libp2pPeerInfo, err := utils.Libp2pAddrInfoFromPeer(peer) require.NoError(t, err) - host.Peerstore().AddAddrs(libp2pPeerInfo.ID, libp2pPeerInfo.Addrs, utils.DefaultPeerTTL) + host.Peerstore().AddAddrs(libp2pPeerInfo.ID, libp2pPeerInfo.Addrs, time.Hour) libp2pPeerPubKey, err := utils.Libp2pPublicKeyFromPeer(peer) require.NoError(t, err) diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 018ed39fd..5828a1727 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -15,8 +15,7 @@ import ( const ( week = time.Hour * 24 * 7 // TECHDEBT: consider more carefully and parameterize. - // TECHDEBT: unexport after consolidation of P2P modules - DefaultPeerTTL = 2 * week + defaultPeerTTL = 2 * week ) // PopulateLibp2pHost iterates through peers in given `pstore`, converting peer From a97dbb150d325d77cefc095c6e128e1130533388 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:33:14 +0200 Subject: [PATCH 25/82] chore: remove unused pubsub pieces --- p2p/module.go | 75 +++------------------------------------------------ 1 file changed, 3 insertions(+), 72 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index c8abd6c2e..222cdb636 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -1,13 +1,11 @@ package p2p import ( - "context" "fmt" "io" "time" "github.com/libp2p/go-libp2p" - pubsub "github.com/libp2p/go-libp2p-pubsub" libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pNetwork "github.com/libp2p/go-libp2p/core/network" "github.com/multiformats/go-multiaddr" @@ -56,16 +54,8 @@ type p2pModule struct { // & connection manager. `libp2p.New` configures and starts listening // according to options. // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) - host libp2pHost.Host - // pubsub is used for broadcast communication - // (i.e. multiple, unidentified receivers) - pubsub *pubsub.PubSub - // topic similar to pubsub but received messages are filtered by a "topic" string. - // Published messages are also given the respective topic before broadcast. - topic *pubsub.Topic - // subscription provides an interface to continuously read messages from. - subscription *pubsub.Subscription - network typesP2P.Network + host libp2pHost.Host + network typesP2P.Network } func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { @@ -142,9 +132,6 @@ func (m *p2pModule) GetModuleName() string { } func (m *p2pModule) Start() (err error) { - // TECHDEBT(#595): receive context in interface methods. - ctx := context.Background() - m.GetBus(). GetTelemetryModule(). GetTimeSeriesAgent(). @@ -157,39 +144,9 @@ func (m *p2pModule) Start() (err error) { return fmt.Errorf("starting libp2pHost: %w", err) } - listenAddrLogEvent := m.logger.Info() - for i, addr := range libp2pHost.InfoFromHost(m.host).Addrs { - listenAddrLogEvent.Str(fmt.Sprintf("listen_addr_%d", i), addr.String()) - } - listenAddrLogEvent.Msg("Listening for incoming connections...") - - // TECHDEBT: use RandomSub or GossipSub once we're on more stable ground. - // IMPROVE: consider supporting multiple router types via config. - m.pubsub, err = pubsub.NewFloodSub(ctx, m.host) - if err != nil { - return fmt.Errorf("unable to create pubsub: %w", err) - } - - // Topic is used to `#Publish` messages. - m.topic, err = m.pubsub.Join(protocol.DefaultTopicStr) - if err != nil { - return fmt.Errorf("unable to join pubsub topic: %w", err) - } - - // Subscription is notified when a new message is received on the topic. - m.subscription, err = m.topic.Subscribe() - if err != nil { - return fmt.Errorf("subscribing to pubsub topic: %w", err) - } - - if err := m.startNetwork(); err != nil { - return fmt.Errorf("creating network: %w", err) - } - - // Don't handle streams or read from the subscription in client debug mode. + // Don't handle incoming streams in client debug mode. if !m.isClientDebugMode() { m.host.SetStreamHandler(protocol.PoktProtocolID, m.handleStream) - go m.readFromSubscription(ctx) } m.GetBus(). @@ -380,32 +337,6 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { } } -// readFromSubscription is intended to be called in a goroutine. It continuously -// reads from the subscribed topic in preparation for handling at the network level. -// Used for handling "broadcast" messages (i.e. no specific target node). -func (m *p2pModule) readFromSubscription(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - default: - msg, err := m.subscription.Next(ctx) - if err != nil { - m.logger.Error().Err(err). - Bool("TODO", true). - Msg("reading from subscription") - } - - // Ignore messages from self - if msg.ReceivedFrom == m.host.ID() { - continue - } - - m.handleNetworkData(msg.Data) - } - } -} - func (m *p2pModule) handleNetworkData(data []byte) { appMsgData, err := m.network.HandleNetworkData(data) if err != nil { From e060ba7057055f013e587f6a5bb312d429e4e644 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:34:00 +0200 Subject: [PATCH 26/82] chore: update deps, remove go-libp2p-pubsub --- go.mod | 2 -- go.sum | 4 ---- 2 files changed, 6 deletions(-) diff --git a/go.mod b/go.mod index c7c814829..9ad287d8d 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( github.com/korovkin/limiter v0.0.0-20230307205149-3d4b2b34c99d github.com/labstack/echo/v4 v4.9.1 github.com/libp2p/go-libp2p v0.25.1 - github.com/libp2p/go-libp2p-pubsub v0.9.2 github.com/looplab/fsm v1.0.1 github.com/manifoldco/promptui v0.9.0 github.com/mitchellh/mapstructure v1.5.0 @@ -97,7 +96,6 @@ require ( github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect - github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/go-cid v0.3.2 // indirect diff --git a/go.sum b/go.sum index 40868dddd..dee258460 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,6 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= @@ -473,8 +471,6 @@ github.com/libp2p/go-libp2p v0.25.1 h1:YK+YDCHpYyTvitKWVxa5PfElgIpOONU01X5UcLEwJ github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= -github.com/libp2p/go-libp2p-pubsub v0.9.2 h1:CoWrvqtIbk+8iTLk1yCN8zODMgBSCqRgyVCvHaGJx8Y= -github.com/libp2p/go-libp2p-pubsub v0.9.2/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= From 957ebbb246e1cb1cb9022c2728748f9b67612a73 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:38:12 +0200 Subject: [PATCH 27/82] chore: rename `readStreamTimeout` --- p2p/module.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 222cdb636..f27d98718 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -31,12 +31,12 @@ import ( "github.com/pokt-network/pocket/telemetry" ) -const readStreamTimeoutDuration = time.Second * 10 - // TECHDEBT: configure timeouts. Consider security exposure vs. real-world conditions). // TECHDEBT: parameterize and expose via config. // readStreamTimeout is the duration to wait for a read operation on a // stream to complete, after which the stream is closed ("timed out"). +const readStreamTimeout = time.Second * 10 + var _ modules.P2PModule = &p2pModule{} type p2pModule struct { @@ -378,5 +378,5 @@ func (m *p2pModule) getMultiaddr() (multiaddr.Multiaddr, error) { // newReadStreamDeadline returns a future deadline // based on the read stream timeout duration. func newReadStreamDeadline() time.Time { - return time.Now().Add(readStreamTimeoutDuration) + return time.Now().Add(readStreamTimeout) } From b89fc4857394d554acb63866816b4e2f27885c00 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:41:10 +0200 Subject: [PATCH 28/82] chore: add issue numbers to TECHDEBT comments --- p2p/module.go | 4 ++-- p2p/utils/host.go | 2 +- p2p/utils/url_conversion.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index f27d98718..34b1bed8e 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -31,8 +31,8 @@ import ( "github.com/pokt-network/pocket/telemetry" ) -// TECHDEBT: configure timeouts. Consider security exposure vs. real-world conditions). -// TECHDEBT: parameterize and expose via config. +// TECHDEBT(#629): configure timeouts. Consider security exposure vs. real-world conditions. +// TECHDEBT(#629): parameterize and expose via config. // readStreamTimeout is the duration to wait for a read operation on a // stream to complete, after which the stream is closed ("timed out"). const readStreamTimeout = time.Second * 10 diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 5828a1727..ee94b740b 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -14,7 +14,7 @@ import ( const ( week = time.Hour * 24 * 7 - // TECHDEBT: consider more carefully and parameterize. + // TECHDEBT(#629): consider more carefully and parameterize. defaultPeerTTL = 2 * week ) diff --git a/p2p/utils/url_conversion.go b/p2p/utils/url_conversion.go index 172b4b943..c9b9af534 100644 --- a/p2p/utils/url_conversion.go +++ b/p2p/utils/url_conversion.go @@ -193,6 +193,7 @@ func getPeerIP(hostname string) (net.IP, error) { // stringLogArrayMarshaler implements the `zerolog.LogArrayMarshaler` interface // to marshal an array of strings for use with zerolog. +// TECHDEBT(#609): move to test common utilities. type stringLogArrayMarshaler struct { strs []string } From a18813886d2a3effcdcba16b10ddc888196f98b6 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:42:22 +0200 Subject: [PATCH 29/82] chore: call setup after considering options --- p2p/module.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 34b1bed8e..fe79f1054 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -83,11 +83,6 @@ func (m *p2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (mo // MUST call before referencing m.bus to ensure != nil. bus.RegisterModule(m) - m.setupDependencies() - - for _, option := range options { - option(m) - } if err := m.configureBootstrapNodes(); err != nil { return nil, err @@ -107,6 +102,14 @@ func (m *p2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (mo } m.identity = libp2p.Identity(libp2pPrivKey) + for _, option := range options { + option(m) + } + + if err := m.setupDependencies(); err != nil { + return nil, err + } + switch m.cfg.ConnectionType { case types.ConnectionType_TCPConnection: addr, err := m.getMultiaddr() From c42412e921cc368d735c9907d72fe5d899e903d8 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:47:21 +0200 Subject: [PATCH 30/82] refactor: `p2pModule#startHost()` not to reuse closed host --- p2p/module.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index fe79f1054..a78a1e59a 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -143,8 +143,11 @@ func (m *p2pModule) Start() (err error) { telemetry.P2P_NODE_STARTED_TIMESERIES_METRIC_DESCRIPTION, ) - if err = m.startHost(); err != nil { - return fmt.Errorf("starting libp2pHost: %w", err) + // Return early if host has already been started (e.g. via `WithHostOption`) + if m.host == nil { + if err = m.startHost(); err != nil { + return fmt.Errorf("starting libp2pHost: %w", err) + } } // Don't handle incoming streams in client debug mode. @@ -160,7 +163,11 @@ func (m *p2pModule) Start() (err error) { } func (m *p2pModule) Stop() error { - return m.host.Close() + err := m.host.Close() + + // Don't reuse closed host, `#Start()` will re-create. + m.host = nil + return err } func (m *p2pModule) Broadcast(msg *anypb.Any) error { @@ -254,11 +261,9 @@ func (m *p2pModule) startNetwork() (err error) { return err } +// startHost creates a new libp2p host and assignes it to `m.host`, if one does +// not already exist. Libp2p host starts listening upon instantiation. func (m *p2pModule) startHost() (err error) { - // Return early if host has already been started (e.g. via `WithHostOption`) - if m.host != nil { - return nil - } opts := []libp2p.Option{ // Explicitly specify supported transport security options (noise, TLS) // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.3#DefaultSecurity) From 343c7cffb2cf3d805f67156eef4461f4e417eb02 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:49:41 +0200 Subject: [PATCH 31/82] refactor: `p2pModule#setupDependencies`: - refactor: and rename `p2pModule#setupNetwork()` - refactor: `p2pModule#setupPeerstoreProvider()` to return an error - refactor: `p2pModule#setupCurrentHeightProvider()` to return an error --- p2p/module.go | 56 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index a78a1e59a..81c29919f 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -1,6 +1,7 @@ package p2p import ( + "errors" "fmt" "io" "time" @@ -9,6 +10,7 @@ import ( libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pNetwork "github.com/libp2p/go-libp2p/core/network" "github.com/multiformats/go-multiaddr" + "go.uber.org/multierr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -202,47 +204,70 @@ func (m *p2pModule) GetAddress() (cryptoPocket.Address, error) { return m.address, nil } -func (m *p2pModule) setupDependencies() { - m.setupCurrentHeightProvider() - m.setupPeerstoreProvider() +// setupDependencies sets up the module's current height and peerstore +// providers, and the network. +func (m *p2pModule) setupDependencies() error { + if err := m.setupCurrentHeightProvider(); err != nil { + return err + } + + if err := m.setupPeerstoreProvider(); err != nil { + return err + } + + if err := m.setupNetwork(); err != nil { + return err + } + return nil } // setupPeerstoreProvider attempts to retrieve the peerstore provider from the // bus, if one is registered, otherwise returns a new `persistencePeerstoreProvider`. -func (m *p2pModule) setupPeerstoreProvider() { +func (m *p2pModule) setupPeerstoreProvider() error { m.logger.Debug().Msg("setupPeerstoreProvider") pstoreProviderModule, err := m.GetBus().GetModulesRegistry().GetModule(peerstore_provider.ModuleName) - if pstoreProviderModule != nil { - m.logger.Debug().Msg("loaded persistence peerstore...") - } if err != nil { - m.logger.Debug().Msg("NewPersistencePeerstore...") + m.logger.Debug().Msg("creating new persistence peerstore...") pstoreProviderModule = persABP.NewPersistencePeerstoreProvider(m.GetBus()) + } else if pstoreProviderModule != nil { + m.logger.Debug().Msg("loaded persistence peerstore...") } var ok bool m.pstoreProvider, ok = pstoreProviderModule.(providers.PeerstoreProvider) if !ok { - m.logger.Fatal().Msgf("unknown peerstore provider type: %T", pstoreProviderModule) + typeErr := fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) + return multierr.Append(err, typeErr) } + return nil } // setupCurrentHeightProvider attempts to retrieve the current height provider // from the bus registry, falls back to the consensus module if none is registered. -func (m *p2pModule) setupCurrentHeightProvider() { +func (m *p2pModule) setupCurrentHeightProvider() error { + m.logger.Debug().Msg("setupCurrentHeightProvider") currentHeightProviderModule, err := m.GetBus().GetModulesRegistry().GetModule(current_height_provider.ModuleName) if err != nil { currentHeightProviderModule = m.GetBus().GetConsensusModule() } + if currentHeightProviderModule == nil { + return errors.New("no current height provider or consensus module registered") + } + + m.logger.Debug().Msg("loaded current height provider") + var ok bool m.currentHeightProvider, ok = currentHeightProviderModule.(providers.CurrentHeightProvider) if !ok { - m.logger.Fatal().Msgf("unexpected current height provider type: %T", currentHeightProviderModule) + typeErr := fmt.Errorf("unexpected current height provider type: %T", currentHeightProviderModule) + return multierr.Append(err, typeErr) } + return nil } -func (m *p2pModule) startNetwork() (err error) { +// setupNetwork instantiates the configured network implementation. +func (m *p2pModule) setupNetwork() (err error) { if m.cfg.UseRainTree { m.network, err = raintree.NewRainTreeNetwork( m.host, @@ -288,6 +313,13 @@ func (m *p2pModule) startHost() (err error) { if err != nil { return fmt.Errorf("unable to create libp2p host: %w", err) } + + // TECHDEBT(#609): use `StringArrayLogMarshaler` post test-utilities refactor. + addrStrs := make(map[int]string) + for i, addr := range libp2pHost.InfoFromHost(m.host).Addrs { + addrStrs[i] = addr.String() + } + m.logger.Info().Fields(addrStrs).Msg("Listening for incoming connections...") return nil } From 8cd52c7219986fc299194e7ad78f33e40bbcec7a Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:54:35 +0200 Subject: [PATCH 32/82] chore: add missing godoc comments --- p2p/module.go | 3 +++ p2p/raintree/network.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/p2p/module.go b/p2p/module.go index 81c29919f..df233ac8d 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -323,6 +323,7 @@ func (m *p2pModule) startHost() (err error) { return nil } +// isClientDebugMode returns the value of `ClientDebugMode` in the base config func (m *p2pModule) isClientDebugMode() bool { return m.GetBus().GetRuntimeMgr().GetConfig().ClientDebugMode } @@ -377,6 +378,8 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { } } +// handleNetworkData passes a network message to the configured +// `Network`implementation for routing. func (m *p2pModule) handleNetworkData(data []byte) { appMsgData, err := m.network.HandleNetworkData(data) if err != nil { diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 2afaa733f..892a6023a 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -74,10 +74,14 @@ func NewRainTreeNetwork(host libp2pHost.Host, addr cryptoPocket.Address, bus mod return typesP2P.Network(n), nil } +// NetworkBroadcast implements the respective member of `typesP2P.Network`. func (n *rainTreeNetwork) NetworkBroadcast(data []byte) error { return n.networkBroadcastAtLevel(data, n.peersManager.GetMaxNumLevels(), crypto.GetNonce()) } +// networkBroadcastAtLevel recursively sends to both left and right target peers +// from the starting level, demoting until level == 0. +// (see: https://github.com/pokt-network/pocket-network-protocol/tree/main/p2p) func (n *rainTreeNetwork) networkBroadcastAtLevel(data []byte, level uint32, nonce uint64) error { // This is handled either by the cleanup layer or redundancy layer if level == 0 { @@ -108,6 +112,8 @@ func (n *rainTreeNetwork) networkBroadcastAtLevel(data []byte, level uint32, non return nil } +// demote broadcasts to the decremented level's targets. +// (see: https://github.com/pokt-network/pocket-network-protocol/tree/main/p2p) func (n *rainTreeNetwork) demote(rainTreeMsg *typesP2P.RainTreeMessage) error { if rainTreeMsg.Level > 0 { if err := n.networkBroadcastAtLevel(rainTreeMsg.Data, rainTreeMsg.Level-1, rainTreeMsg.Nonce); err != nil { @@ -117,6 +123,7 @@ func (n *rainTreeNetwork) demote(rainTreeMsg *typesP2P.RainTreeMessage) error { return nil } +// NetworkSend implements the respective member of `typesP2P.Network`. func (n *rainTreeNetwork) NetworkSend(data []byte, address cryptoPocket.Address) error { msg := &typesP2P.RainTreeMessage{ Level: 0, // Direct send that does not need to be propagated @@ -132,6 +139,7 @@ func (n *rainTreeNetwork) NetworkSend(data []byte, address cryptoPocket.Address) return n.networkSendInternal(bz, address) } +// networkSendInternal sends `data` to the peer at pokt `address` if not self. func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket.Address) error { // NOOP: Trying to send a message to self if n.selfAddr.Equals(address) { @@ -166,6 +174,7 @@ func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket. return nil } +// HandleNetworkData implements the respective member of `typesP2P.Network`. func (n *rainTreeNetwork) HandleNetworkData(data []byte) ([]byte, error) { blockHeightInt := n.GetBus().GetConsensusModule().CurrentHeight() blockHeight := fmt.Sprintf("%d", blockHeightInt) @@ -224,10 +233,12 @@ func (n *rainTreeNetwork) HandleNetworkData(data []byte) ([]byte, error) { return rainTreeMsg.Data, nil } +// GetPeerstore implements the respective member of `typesP2P.Network`. func (n *rainTreeNetwork) GetPeerstore() typesP2P.Peerstore { return n.peersManager.GetPeerstore() } +// AddPeer implements the respective member of `typesP2P.Network`. func (n *rainTreeNetwork) AddPeer(peer typesP2P.Peer) error { // Noop if peer with the same pokt address exists in the peerstore. // TECHDEBT: add method(s) to update peers. @@ -253,10 +264,13 @@ func (n *rainTreeNetwork) RemovePeer(peer typesP2P.Peer) error { return nil } +// Size returns the number of peers the network is aware of and would attempt to +// broadcast to. func (n *rainTreeNetwork) Size() int { return n.peersManager.GetPeerstore().Size() } +// shouldSendToTarget returns false if target is self. func shouldSendToTarget(target target) bool { return !target.isSelf } From b7c690dde8583fb8ee815274133426a22b31857d Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:55:25 +0200 Subject: [PATCH 33/82] refactor: `p2pModule#HandleNetworkData` to return an error --- p2p/module.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index df233ac8d..0e43470dc 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -335,11 +335,12 @@ func (m *p2pModule) handleStream(stream libp2pNetwork.Stream) { if err != nil { m.logger.Error().Err(err). Str("address", peer.GetAddress().String()). - Msg("parsing remote peer public key") + Msg("parsing remote peer identity") if err = stream.Reset(); err != nil { m.logger.Error().Err(err).Msg("resetting stream") } + return } if err := m.network.AddPeer(peer); err != nil { @@ -371,7 +372,9 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { return } - m.handleNetworkData(data) + if err := m.handleNetworkData(data); err != nil { + m.logger.Error().Err(err).Msg("handling network data") + } if err := stream.CloseRead(); err != nil { m.logger.Debug().Err(err).Msg("closing read stream") @@ -380,25 +383,21 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { // handleNetworkData passes a network message to the configured // `Network`implementation for routing. -func (m *p2pModule) handleNetworkData(data []byte) { +func (m *p2pModule) handleNetworkData(data []byte) error { appMsgData, err := m.network.HandleNetworkData(data) if err != nil { - m.logger.Error().Err(err).Msg("handling network data") - return + return err } // There was no error, but we don't need to forward this to the app-specific bus. // For example, the message has already been handled by the application. if appMsgData == nil { - return + return nil } networkMessage := messaging.PocketEnvelope{} if err := proto.Unmarshal(appMsgData, &networkMessage); err != nil { - m.logger.Error().Err(err). - Bool("TODO", true). - Msg("Error decoding network message") - return + return fmt.Errorf("decoding network message: %w", err) } event := messaging.PocketEnvelope{ @@ -406,6 +405,7 @@ func (m *p2pModule) handleNetworkData(data []byte) { } m.GetBus().PublishEventToBus(&event) + return nil } // getMultiaddr returns a multiaddr constructed from the `hostname` and `port` From f78bd397d1a8751ffdf72b6ebd77c8f019e8bb3f Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:56:25 +0200 Subject: [PATCH 34/82] test: assign `genesisMock` to a var --- p2p/module_raintree_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/module_raintree_test.go b/p2p/module_raintree_test.go index d755aebaf..d7ba2ee2d 100644 --- a/p2p/module_raintree_test.go +++ b/p2p/module_raintree_test.go @@ -219,6 +219,7 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te // Configure & prepare test module numValidators := len(networkSimulationConfig) runtimeConfigs := createMockRuntimeMgrs(t, numValidators) + genesisMock := runtimeConfigs[0].GetGenesis() busMocks := createMockBuses(t, runtimeConfigs) valIds := make([]string, 0, numValidators) @@ -247,7 +248,7 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te wg.Add(expectedReads) wg.Add(expectedWrites) - persistenceMock := preparePersistenceMock(t, busMocks[i], runtimeConfigs[0].GetGenesis()) + persistenceMock := preparePersistenceMock(t, busMocks[i], genesisMock) consensusMock := prepareConsensusMock(t, busMocks[i]) telemetryMock := prepareTelemetryMock(t, busMocks[i], valId, &wg, expectedWrites) From 0e2a1d5f3369ef0804685160a091ea008e917a1b Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:57:01 +0200 Subject: [PATCH 35/82] test: improve test lifecycle --- p2p/module_raintree_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/p2p/module_raintree_test.go b/p2p/module_raintree_test.go index d7ba2ee2d..640c76116 100644 --- a/p2p/module_raintree_test.go +++ b/p2p/module_raintree_test.go @@ -278,15 +278,14 @@ func testRainTreeCalls(t *testing.T, origNode string, networkSimulationConfig Te } // Wait for completion - defer func() { - waitForNetworkSimulationCompletion(t, &wg) - + defer waitForNetworkSimulationCompletion(t, &wg) + t.Cleanup(func() { // Stop all p2p modules for _, p2pMod := range p2pModules { err := p2pMod.Stop() require.NoError(t, err) } - }() + }) // Send the first message (by the originator) to trigger a RainTree broadcast p := &anypb.Any{} From 7ca40de25269e49d68ebfa836eaef45a4fd7fb9a Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:57:48 +0200 Subject: [PATCH 36/82] test: fix genesis mock, use all test validators --- p2p/module_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/module_test.go b/p2p/module_test.go index 946ddfcbc..0d6f0b558 100644 --- a/p2p/module_test.go +++ b/p2p/module_test.go @@ -109,7 +109,7 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) mockBus := createMockBus(t, mockRuntimeMgr) - genesisStateMock := createMockGenesisState(keys[:1]) + genesisStateMock := createMockGenesisState(keys) persistenceMock := preparePersistenceMock(t, mockBus, genesisStateMock) mockBus.EXPECT().GetPersistenceModule().Return(persistenceMock).AnyTimes() From 3b6253e384c2f4eb408e277b4d8399b7d2aab883 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 18:59:36 +0200 Subject: [PATCH 37/82] test: factor out test service URL --- p2p/module_test.go | 6 +++++- p2p/raintree/network_test.go | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/p2p/module_test.go b/p2p/module_test.go index 0d6f0b558..36bfb224e 100644 --- a/p2p/module_test.go +++ b/p2p/module_test.go @@ -1,6 +1,7 @@ package p2p import ( + "fmt" "strings" "testing" @@ -18,6 +19,9 @@ import ( mockModules "github.com/pokt-network/pocket/shared/modules/mocks" ) +// TECHDEBT(#609): move & de-dup. +var testLocalServiceURL = fmt.Sprintf("127.0.0.1:%d", defaults.DefaultP2PPort) + func Test_Create_configureBootstrapNodes(t *testing.T) { defaultBootstrapNodes := strings.Split(defaults.DefaultP2PBootstrapNodesCsv, ",") privKey := cryptoPocket.GetPrivKeySeed(1) @@ -128,7 +132,7 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { peer := &typesP2P.NetworkPeer{ PublicKey: privKey.PublicKey(), Address: privKey.Address(), - ServiceURL: "10.0.0.1:42069", + ServiceURL: testLocalServiceURL, } host := newLibp2pMockNetHost(t, privKey, peer) diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index 33dc75dea..b647099f6 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -1,6 +1,7 @@ package raintree import ( + "fmt" "testing" "time" @@ -11,10 +12,14 @@ import ( typesP2P "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/p2p/utils" + "github.com/pokt-network/pocket/runtime/defaults" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/stretchr/testify/require" ) +// TECHDEBT(#609): move & de-dup. +var testLocalServiceURL = fmt.Sprintf("127.0.0.1:%d", defaults.DefaultP2PPort) + func TestRainTreeNetwork_AddPeer(t *testing.T) { ctrl := gomock.NewController(t) @@ -157,7 +162,7 @@ func newTestPeer(t *testing.T) (*typesP2P.NetworkPeer, libp2pHost.Host) { selfPeer := &typesP2P.NetworkPeer{ PublicKey: selfPrivKey.PublicKey(), Address: selfAddr, - ServiceURL: "10.0.0.1:42069", + ServiceURL: testLocalServiceURL, } return selfPeer, newLibp2pMockNetHost(t, selfPrivKey, selfPeer) } From 111ae0a9528f90e6471d9bb0dba810fd186272fa Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 19:00:59 +0200 Subject: [PATCH 38/82] chore: improve debug & error logging --- p2p/raintree/network.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 892a6023a..ea8c89a1d 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -100,13 +100,13 @@ func (n *rainTreeNetwork) networkBroadcastAtLevel(data []byte, level uint32, non for _, target := range n.getTargetsAtLevel(level) { if shouldSendToTarget(target) { if err = n.networkSendInternal(msgBz, target.address); err != nil { - n.logger.Error().Err(err).Msg("Error sending to peer during broadcast") + n.logger.Error().Err(err).Msg("sending to peer during broadcast") } } } if err = n.demote(msg); err != nil { - n.logger.Error().Err(err).Msg("Error demoting self during RainTree message propagation") + n.logger.Error().Err(err).Msg("demoting self during RainTree message propagation") } return nil @@ -142,7 +142,9 @@ func (n *rainTreeNetwork) NetworkSend(data []byte, address cryptoPocket.Address) // networkSendInternal sends `data` to the peer at pokt `address` if not self. func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket.Address) error { // NOOP: Trying to send a message to self + n.logger.Debug().Str("self", n.selfAddr.String()).Str("target", address.String()).Msg("HEER!!!!") if n.selfAddr.Equals(address) { + n.logger.Debug().Str("pokt_addr", address.String()).Msg("attempted to send to self") return nil } @@ -152,7 +154,7 @@ func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket. } if err := utils.Libp2pSendToPeer(n.host, data, peer); err != nil { - logger.Global.Debug().Err(err).Msg("from libp2pSendInternal") + n.logger.Debug().Err(err).Msg("from libp2pSendInternal") return err } From 4b3f1c0fcdbf44edb936d10ec5d4e47349f884c7 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 19:01:40 +0200 Subject: [PATCH 39/82] test: improve raintree network test --- p2p/raintree/network.go | 5 +++++ p2p/raintree/network_test.go | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index ea8c89a1d..115f460cb 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -247,6 +247,11 @@ func (n *rainTreeNetwork) AddPeer(peer typesP2P.Peer) error { if p := n.peersManager.GetPeerstore().GetPeer(peer.GetAddress()); p != nil { return nil } + + if err := utils.AddPeerToLibp2pHost(n.host, peer); err != nil { + return err + } + n.peersManager.HandleEvent( typesP2P.PeerManagerEvent{ EventType: typesP2P.AddPeerEventType, diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index b647099f6..dee2c1f7d 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -57,9 +57,13 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { rainTreeNet := network.(*rainTreeNetwork) - peerAddr, err := cryptoPocket.GenerateAddress() + privKey, err := cryptoPocket.GeneratePrivateKey() require.NoError(t, err) - peerToAdd := &typesP2P.NetworkPeer{Address: peerAddr} + peerToAdd := &typesP2P.NetworkPeer{ + PublicKey: privKey.PublicKey(), + Address: privKey.Address(), + ServiceURL: testLocalServiceURL, + } // Add peerToAdd. err = rainTreeNet.AddPeer(peerToAdd) @@ -73,11 +77,14 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { require.Equal(t, expectedPStoreSize, len(peerAddrs)) require.Equal(t, expectedPStoreSize, len(peers)) - require.ElementsMatch(t, []string{selfAddr.String(), peerAddr.String()}, peerAddrs, "addresses do not match") + libp2pPStore := host.Peerstore() + require.Len(t, libp2pPStore.Peers(), expectedPStoreSize) + + require.ElementsMatch(t, []string{selfAddr.String(), peerToAdd.GetAddress().String()}, peerAddrs, "addresses do not match") require.ElementsMatch(t, []*typesP2P.NetworkPeer{selfPeer, peerToAdd}, peers, "peers do not match") require.Equal(t, selfPeer, network.GetPeerstore().GetPeer(selfAddr), "Peerstore does not contain self") - require.Equal(t, peerToAdd, network.GetPeerstore().GetPeer(peerAddr), "Peerstore does not contain added peer") + require.Equal(t, peerToAdd, network.GetPeerstore().GetPeer(peerToAdd.GetAddress()), "Peerstore does not contain added peer") } func TestRainTreeNetwork_RemovePeer(t *testing.T) { @@ -111,6 +118,9 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { require.Equal(t, expectedPStoreSize, len(peerAddrs)) require.Equal(t, expectedPStoreSize, len(peers)) + libp2pPStore := host.Peerstore() + require.Len(t, libp2pPStore.Peers(), expectedPStoreSize) + var peerToRemove typesP2P.Peer // Ensure we don't remove selfPeer. `Peerstore` interface isn't aware // of the concept of "self" so we have to find it. From bd278b20550970bcf813e34724d7d0a3a6838a6f Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 19:02:20 +0200 Subject: [PATCH 40/82] chore: use `context.TODO()` --- p2p/utils/host.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/utils/host.go b/p2p/utils/host.go index ee94b740b..d41c723e5 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -78,7 +78,7 @@ func RemovePeerFromLibp2pHost(host libp2pHost.Host, peer typesP2P.Peer) error { // Libp2pSendToPeer sends data to the given pocket peer from the given libp2p host. func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) error { // TECHDEBT(#595): add ctx to interface methods and propagate down. - ctx := context.Background() + ctx := context.TODO() peerInfo, err := Libp2pAddrInfoFromPeer(peer) if err != nil { From a26e6414fe79ac2503a826497a79c1d50df2ea9e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 28 Mar 2023 19:06:05 +0200 Subject: [PATCH 41/82] chore: update changelog --- app/client/doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/doc/CHANGELOG.md b/app/client/doc/CHANGELOG.md index ded7cf7f2..d788bf535 100644 --- a/app/client/doc/CHANGELOG.md +++ b/app/client/doc/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.24] - 2023-03-24 +## [0.0.0.24] - 2023-03-28 - Refactor debug CLI post P2P module re-consolidation From dab3215e0d674f51d441cb806aed487f29a58ed2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 10:48:41 +0200 Subject: [PATCH 42/82] chore: add `RainTreeConfig` struct --- p2p/raintree/network.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 115f460cb..b854d1301 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -24,6 +24,13 @@ import ( var _ typesP2P.Network = &rainTreeNetwork{} +type RainTreeConfig struct { + Addr cryptoPocket.Address + PeerstoreProvider providers.PeerstoreProvider + CurrentHeightProvider providers.CurrentHeightProvider + Host libp2pHost.Host +} + type rainTreeNetwork struct { base_modules.IntegratableModule @@ -41,8 +48,6 @@ type rainTreeNetwork struct { nonceDeduper *mempool.GenericFIFOSet[uint64, uint64] } -// TECHDEBT: refactor signature to receive a config struct -// (i.e. `NewRainTreeNetwork(bus modules.Bus, cfg NetworkConfig). func NewRainTreeNetwork(host libp2pHost.Host, addr cryptoPocket.Address, bus modules.Bus, pstoreProvider providers.PeerstoreProvider, currentHeightProvider providers.CurrentHeightProvider) (typesP2P.Network, error) { networkLogger := logger.Global.CreateLoggerForModule("network") networkLogger.Info().Msg("Initializing rainTreeNetwork") @@ -302,3 +307,14 @@ func (n *rainTreeNetwork) setupPeerManager(pstore typesP2P.Peerstore) (err error n.peersManager, err = newPeersManager(n.selfAddr, pstore, true) return err } + +func (cfg RainTreeConfig) isValid() error { + if cfg.PeerstoreProvider == nil { + return fmt.Errorf("peerstore provider not configured") + } + + if cfg.CurrentHeightProvider == nil { + return fmt.Errorf("current height provider not configured") + } + return nil +} From b2f0cd0efa5957bfaeebf5e1ee6b21b3744f0641 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 10:51:14 +0200 Subject: [PATCH 43/82] chore: add `#Create()` & use new config struct --- p2p/module.go | 10 ++++++---- p2p/raintree/network.go | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 0e43470dc..c7d18197b 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -270,11 +270,13 @@ func (m *p2pModule) setupCurrentHeightProvider() error { func (m *p2pModule) setupNetwork() (err error) { if m.cfg.UseRainTree { m.network, err = raintree.NewRainTreeNetwork( - m.host, - m.address, m.GetBus(), - m.pstoreProvider, - m.currentHeightProvider, + raintree.RainTreeConfig{ + Host: m.host, + Addr: m.address, + PeerstoreProvider: m.pstoreProvider, + CurrentHeightProvider: m.currentHeightProvider, + }, ) } else { m.network, err = stdnetwork.NewNetwork( diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index b854d1301..fc39394c3 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -48,26 +48,26 @@ type rainTreeNetwork struct { nonceDeduper *mempool.GenericFIFOSet[uint64, uint64] } -func NewRainTreeNetwork(host libp2pHost.Host, addr cryptoPocket.Address, bus modules.Bus, pstoreProvider providers.PeerstoreProvider, currentHeightProvider providers.CurrentHeightProvider) (typesP2P.Network, error) { +func NewRainTreeNetwork(bus modules.Bus, cfg RainTreeConfig) (typesP2P.Network, error) { + return new(rainTreeNetwork).Create(bus, cfg) +} + +func (*rainTreeNetwork) Create(bus modules.Bus, netCfg RainTreeConfig) (typesP2P.Network, error) { networkLogger := logger.Global.CreateLoggerForModule("network") networkLogger.Info().Msg("Initializing rainTreeNetwork") - if pstoreProvider == nil { - return nil, fmt.Errorf("peerstore provider required, got nil") - } - - if currentHeightProvider == nil { - return nil, fmt.Errorf("current height provider required, got nil") + if err := netCfg.isValid(); err != nil { + return nil, err } p2pCfg := bus.GetRuntimeMgr().GetConfig().P2P n := &rainTreeNetwork{ - host: host, - selfAddr: addr, + host: netCfg.Host, + selfAddr: netCfg.Addr, nonceDeduper: mempool.NewGenericFIFOSet[uint64, uint64](int(p2pCfg.MaxMempoolCount)), - pstoreProvider: pstoreProvider, - currentHeightProvider: currentHeightProvider, + pstoreProvider: netCfg.PeerstoreProvider, + currentHeightProvider: netCfg.CurrentHeightProvider, logger: networkLogger, } n.SetBus(bus) From f0943478dabb4b3cfea801238cfa020a81934221 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 10:53:20 +0200 Subject: [PATCH 44/82] chore: refactor tests to use `RainTreeConfig` --- p2p/raintree/network_test.go | 17 +++++++++++-- p2p/raintree/peers_manager_test.go | 39 ++++++++++++++++++------------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index dee2c1f7d..0694f8e46 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -52,7 +52,14 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { peerstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) - network, err := NewRainTreeNetwork(host, selfAddr, busMock, peerstoreProviderMock, currentHeightProviderMock) + netCfg := RainTreeConfig{ + Host: host, + Addr: selfAddr, + PeerstoreProvider: peerstoreProviderMock, + CurrentHeightProvider: currentHeightProviderMock, + } + + network, err := NewRainTreeNetwork(busMock, netCfg) require.NoError(t, err) rainTreeNet := network.(*rainTreeNetwork) @@ -107,8 +114,14 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { busMock := mockBus(ctrl) peerstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) + netCfg := RainTreeConfig{ + Host: host, + Addr: selfAddr, + PeerstoreProvider: peerstoreProviderMock, + CurrentHeightProvider: currentHeightProviderMock, + } - network, err := NewRainTreeNetwork(host, selfAddr, busMock, peerstoreProviderMock, currentHeightProviderMock) + network, err := NewRainTreeNetwork(busMock, netCfg) require.NoError(t, err) rainTree := network.(*rainTreeNetwork) diff --git a/p2p/raintree/peers_manager_test.go b/p2p/raintree/peers_manager_test.go index 30e82ca87..8e2273c6a 100644 --- a/p2p/raintree/peers_manager_test.go +++ b/p2p/raintree/peers_manager_test.go @@ -96,13 +96,14 @@ func TestRainTree_Peerstore_HandleUpdate(t *testing.T) { libp2pMockNet, err := mocknet.WithNPeers(1) require.NoError(t, err) - network, err := NewRainTreeNetwork( - libp2pMockNet.Hosts()[0], - pubKey.Address(), - mockBus, - pstoreProviderMock, - currentHeightProviderMock, - ) + netCfg := RainTreeConfig{ + Host: libp2pMockNet.Hosts()[0], + Addr: pubKey.Address(), + PeerstoreProvider: pstoreProviderMock, + CurrentHeightProvider: currentHeightProviderMock, + } + + network, err := NewRainTreeNetwork(mockBus, netCfg) require.NoError(t, err) rainTree := network.(*rainTreeNetwork) @@ -151,10 +152,15 @@ func BenchmarkPeerstoreUpdates(b *testing.B) { mockBus := mockBus(ctrl) pstoreProviderMock := mockPeerstoreProvider(ctrl, pstore) currentHeightProviderMock := mockCurrentHeightProvider(ctrl, 0) - hostMock := mocksP2P.NewMockHost(ctrl) + netCfg := RainTreeConfig{ + Host: hostMock, + Addr: addr, + PeerstoreProvider: pstoreProviderMock, + CurrentHeightProvider: currentHeightProviderMock, + } - network, err := NewRainTreeNetwork(hostMock, addr, mockBus, pstoreProviderMock, currentHeightProviderMock) + network, err := NewRainTreeNetwork(mockBus, netCfg) require.NoError(b, err) rainTree := network.(*rainTreeNetwork) @@ -267,13 +273,14 @@ func testRainTreeMessageTargets(t *testing.T, expectedMsgProp *ExpectedRainTreeM hostMock := mocksP2P.NewMockHost(ctrl) hostMock.EXPECT().Peerstore().Return(libp2pPStore).AnyTimes() - network, err := NewRainTreeNetwork( - hostMock, - []byte{expectedMsgProp.orig}, - busMock, - pstoreProviderMock, - currentHeightProviderMock, - ) + netCfg := RainTreeConfig{ + Host: hostMock, + Addr: []byte{expectedMsgProp.orig}, + PeerstoreProvider: pstoreProviderMock, + CurrentHeightProvider: currentHeightProviderMock, + } + + network, err := NewRainTreeNetwork(busMock, netCfg) require.NoError(t, err) rainTree := network.(*rainTreeNetwork) From e2ec506d0fd0629c683d320739ca4922ad0ea582 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 10:54:10 +0200 Subject: [PATCH 45/82] chore: remove unnecessary debug log line --- p2p/raintree/network.go | 1 - 1 file changed, 1 deletion(-) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index fc39394c3..f5373d10c 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -147,7 +147,6 @@ func (n *rainTreeNetwork) NetworkSend(data []byte, address cryptoPocket.Address) // networkSendInternal sends `data` to the peer at pokt `address` if not self. func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket.Address) error { // NOOP: Trying to send a message to self - n.logger.Debug().Str("self", n.selfAddr.String()).Str("target", address.String()).Msg("HEER!!!!") if n.selfAddr.Equals(address) { n.logger.Debug().Str("pokt_addr", address.String()).Msg("attempted to send to self") return nil From 67ae3c76aefc5b547a9344aa31daf38b21512f12 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 10:54:33 +0200 Subject: [PATCH 46/82] fix: `context.TODO()` --- p2p/utils/url_conversion.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/p2p/utils/url_conversion.go b/p2p/utils/url_conversion.go index c9b9af534..af0a67f2e 100644 --- a/p2p/utils/url_conversion.go +++ b/p2p/utils/url_conversion.go @@ -131,9 +131,6 @@ func newResolvePeerIPErr(hostname string, err error) error { } func getPeerIP(hostname string) (net.IP, error) { - // TECHDEBT(#595): receive `ctx` from caller. - ctx := context.Background() - // Attempt to parse peer hostname as an IP address. // (see: https://pkg.go.dev/net#ParseIP) peerIP := net.ParseIP(hostname) @@ -144,7 +141,8 @@ func getPeerIP(hostname string) (net.IP, error) { // CONSIDERATION: using a `/dns<4 or 6>/` multiaddr instead of resolving here. // I attempted using `/dns4/.../tcp/...` and go this error: // > failed to listen on any addresses: [can only dial TCP over IPv4 or IPv6] - addrs, err := net.DefaultResolver.LookupHost(ctx, hostname) + // TECHDEBT(#595): receive `ctx` from caller. + addrs, err := net.DefaultResolver.LookupHost(context.TODO(), hostname) if err != nil { return nil, newResolvePeerIPErr(hostname, err) } From 5d625d880b17075aa4f8c36268449de132f2e980 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 11:10:02 +0200 Subject: [PATCH 47/82] chore: improve `rainTreeNetwork` struct comments --- p2p/module.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index c7d18197b..ba868591c 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -44,20 +44,22 @@ var _ modules.P2PModule = &p2pModule{} type p2pModule struct { base_modules.IntegratableModule - address cryptoPocket.Address - logger *modules.Logger - cfg *configs.P2PConfig - bootstrapNodes []string - currentHeightProvider providers.CurrentHeightProvider - pstoreProvider providers.PeerstoreProvider - identity libp2p.Option - listenAddrs libp2p.Option + address cryptoPocket.Address + logger *modules.Logger + cfg *configs.P2PConfig + bootstrapNodes []string + identity libp2p.Option + listenAddrs libp2p.Option // host represents a libp2p network node, it encapsulates a libp2p peerstore // & connection manager. `libp2p.New` configures and starts listening - // according to options. + // according to options. Assigned via `#Start()` (starts on instantiation). // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) - host libp2pHost.Host - network typesP2P.Network + host libp2pHost.Host + + // Assigned during creation via `#setupDependencies()`. + network typesP2P.Network + currentHeightProvider providers.CurrentHeightProvider + pstoreProvider providers.PeerstoreProvider } func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { From f87722c434f433b3fe95985f6fa08682040ba715 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 11:19:17 +0200 Subject: [PATCH 48/82] chore: update changelog --- p2p/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/CHANGELOG.md b/p2p/CHANGELOG.md index cde2b7ed2..20cf3202c 100644 --- a/p2p/CHANGELOG.md +++ b/p2p/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.39] - 2023-03-28 +## [0.0.0.39] - 2023-03-30 - Moved peer & url conversion utils to `p2p/utils` package - Refactor `getPeerIP` to use `net.DefaultResolver` for easier testing @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removed `Trnasport` interface and implementations - Removed `ConnectionFactory` type and related members - Added libp2p `host.Host` mock generator +- Refactor raintree constructor function signature to use new `RainTreeConfig` struct ## [0.0.0.38] - 2023-03-23 From a4f435700744da9545e84c714408135f7f05c3e4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 30 Mar 2023 11:21:11 +0200 Subject: [PATCH 49/82] chore: improve TECHDEBT comment --- p2p/utils/url_conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/utils/url_conversion.go b/p2p/utils/url_conversion.go index af0a67f2e..c2f0552bb 100644 --- a/p2p/utils/url_conversion.go +++ b/p2p/utils/url_conversion.go @@ -191,7 +191,7 @@ func getPeerIP(hostname string) (net.IP, error) { // stringLogArrayMarshaler implements the `zerolog.LogArrayMarshaler` interface // to marshal an array of strings for use with zerolog. -// TECHDEBT(#609): move to test common utilities. +// TECHDEBT(#609): move & de-duplicate type stringLogArrayMarshaler struct { strs []string } From 348a69815603f703745999e326b189fb17e5a958 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 31 Mar 2023 12:20:31 +0200 Subject: [PATCH 50/82] refactor: rename `#startHost` to `#setupHost` for consistency --- p2p/module.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 24e8a8deb..4c5835bf8 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -149,8 +149,8 @@ func (m *p2pModule) Start() (err error) { // Return early if host has already been started (e.g. via `WithHostOption`) if m.host == nil { - if err = m.startHost(); err != nil { - return fmt.Errorf("starting libp2pHost: %w", err) + if err = m.setupHost(); err != nil { + return fmt.Errorf("setting up libp2pHost: %w", err) } } @@ -290,9 +290,9 @@ func (m *p2pModule) setupNetwork() (err error) { return err } -// startHost creates a new libp2p host and assignes it to `m.host`, if one does +// setupHost creates a new libp2p host and assignes it to `m.host`, if one does // not already exist. Libp2p host starts listening upon instantiation. -func (m *p2pModule) startHost() (err error) { +func (m *p2pModule) setupHost() (err error) { opts := []libp2p.Option{ // Explicitly specify supported transport security options (noise, TLS) // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.3#DefaultSecurity) From 8249db99f655321f59f2b13fe06cbb32238bd2d4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 31 Mar 2023 12:22:02 +0200 Subject: [PATCH 51/82] fix: `RainTreeConfig#isValid()` checks `host` != nil --- p2p/raintree/network.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 61ee2f025..186ea1a27 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -308,6 +308,10 @@ func (n *rainTreeNetwork) setupPeerManager(pstore typesP2P.Peerstore) (err error } func (cfg RainTreeConfig) isValid() error { + if cfg.Host == nil { + return fmt.Errorf("host not configured") + } + if cfg.PeerstoreProvider == nil { return fmt.Errorf("peerstore provider not configured") } From e08f0247fb40ef9eb9ec261fbe7867ca072af5bf Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 31 Mar 2023 12:22:45 +0200 Subject: [PATCH 52/82] fix: move `#setupNetwork()` call into `#Start()` `#setupNetwork()` calls `NewRainTreeNetwork()`, which requires a non-nil libp2p host in its config argument. --- p2p/module.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 4c5835bf8..f60124492 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -138,6 +138,8 @@ func (m *p2pModule) GetModuleName() string { return modules.P2PModuleName } +// Start instantiates and assigns `m.host`, unless one already exists, and +// `m.network` (which depends on `m.host` as a required config field). func (m *p2pModule) Start() (err error) { m.GetBus(). GetTelemetryModule(). @@ -154,6 +156,10 @@ func (m *p2pModule) Start() (err error) { } } + if err := m.setupNetwork(); err != nil { + return fmt.Errorf("setting up network: %w", err) + } + // Don't handle incoming streams in client debug mode. if !m.isClientDebugMode() { m.host.SetStreamHandler(protocol.PoktProtocolID, m.handleStream) @@ -217,9 +223,6 @@ func (m *p2pModule) setupDependencies() error { return err } - if err := m.setupNetwork(); err != nil { - return err - } return nil } From 42077446dd5cf5a38d6fe8b988f5aebe7237437b Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 31 Mar 2023 12:38:10 +0200 Subject: [PATCH 53/82] chore: update `p2pModule` struct field order/comments --- p2p/module.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index f60124492..b58447f2e 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -50,16 +50,19 @@ type p2pModule struct { bootstrapNodes []string identity libp2p.Option listenAddrs libp2p.Option + + // Assigned during creation via `#setupDependencies()`. + currentHeightProvider providers.CurrentHeightProvider + pstoreProvider providers.PeerstoreProvider + + // Assigned during `#Start()`. TLDR; `host` listens on instantiation. + // and `network` depends on `host`. + network typesP2P.Network // host represents a libp2p network node, it encapsulates a libp2p peerstore // & connection manager. `libp2p.New` configures and starts listening // according to options. Assigned via `#Start()` (starts on instantiation). // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) host libp2pHost.Host - - // Assigned during creation via `#setupDependencies()`. - network typesP2P.Network - currentHeightProvider providers.CurrentHeightProvider - pstoreProvider providers.PeerstoreProvider } func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, error) { From c761647b568129606cff64db0d867dbda6bd6ec3 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 31 Mar 2023 12:40:46 +0200 Subject: [PATCH 54/82] chore: enforce `RainTreeNetwork` implements `modules.IntegratableModule` --- p2p/raintree/network.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 186ea1a27..c2587ddb2 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -22,7 +22,10 @@ import ( "google.golang.org/protobuf/proto" ) -var _ typesP2P.Network = &rainTreeNetwork{} +var ( + _ typesP2P.Network = &rainTreeNetwork{} + _ modules.IntegratableModule = &rainTreeNetwork{} +) type RainTreeConfig struct { Addr cryptoPocket.Address From 2585196951cd6b4b8b4ace561deacb62bbee816f Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Tue, 11 Apr 2023 15:01:48 -0700 Subject: [PATCH 55/82] Temp reference commit by Olshansky Bryan - please do not override this commit as its for reference purposes. But do clean it up Attempts: - Change max_conns from 50 to 5 - Added LIBP2P_DEBUG flag - Improved some logging - Added debug OLSH logging - Added a connection manager Solution - Replae `stream.CloseRead` with `stream.Reset` --- build/config/config1.json | 2 +- build/config/config2.json | 2 +- build/config/config3.json | 2 +- build/config/config4.json | 2 +- build/deployments/docker-compose.yaml | 1 + build/localnet/manifests/configs.yaml | 2 +- consensus/debugging.go | 11 +++++------ consensus/helpers.go | 24 +++++++----------------- consensus/module_consensus_pacemaker.go | 2 +- p2p/module.go | 15 ++++++++++++++- p2p/raintree/network.go | 1 + p2p/utils/host.go | 4 +++- shared/node.go | 3 +++ 13 files changed, 40 insertions(+), 31 deletions(-) diff --git a/build/config/config1.json b/build/config/config1.json index 06fc027b3..7d33ff802 100644 --- a/build/config/config1.json +++ b/build/config/config1.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 50, + "max_conns_count": 5, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/config/config2.json b/build/config/config2.json index 58aa180a6..88974fde9 100644 --- a/build/config/config2.json +++ b/build/config/config2.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 50, + "max_conns_count": 5, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/config/config3.json b/build/config/config3.json index 3862126b7..9ce2a792a 100644 --- a/build/config/config3.json +++ b/build/config/config3.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 50, + "max_conns_count": 5, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/config/config4.json b/build/config/config4.json index 275b5523b..c5daa2342 100644 --- a/build/config/config4.json +++ b/build/config/config4.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 50, + "max_conns_count": 5, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/deployments/docker-compose.yaml b/build/deployments/docker-compose.yaml index 5861b7f39..3031db52d 100755 --- a/build/deployments/docker-compose.yaml +++ b/build/deployments/docker-compose.yaml @@ -53,6 +53,7 @@ services: - "seccomp:unconfined" environment: - POCKET_RPC_USE_CORS=true + - LIBP2P_DEBUG=info # Uncomment to enable the pprof server # - PPROF_ENABLED=true # Uncomment to enable DLV debugging diff --git a/build/localnet/manifests/configs.yaml b/build/localnet/manifests/configs.yaml index c3a7d23be..6d9d4b3e8 100644 --- a/build/localnet/manifests/configs.yaml +++ b/build/localnet/manifests/configs.yaml @@ -28,7 +28,7 @@ data: "block_store_path": "/validator-storage/blockstore", "tx_indexer_path": "", "trees_store_dir": "/validator-storage/trees", - "max_conns_count": 50, + "max_conns_count": 5, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/consensus/debugging.go b/consensus/debugging.go index 0db59bbf6..27be89ed8 100644 --- a/consensus/debugging.go +++ b/consensus/debugging.go @@ -49,12 +49,11 @@ func (m *consensusModule) resetToGenesis(_ *messaging.DebugMessage) error { func (m *consensusModule) printNodeState(_ *messaging.DebugMessage) { state := m.GetNodeState() - m.logger.Debug(). - Fields(map[string]any{ - "step": state.Step, - "height": state.Height, - "round": state.Round, - }).Msg("Node state") + m.logger.Debug().Fields(map[string]any{ + "step": typesCons.StepToString[typesCons.HotstuffStep(state.Step)], + "height": state.Height, + "round": state.Round, + }).Msg("Node state") } func (m *consensusModule) triggerNextView(_ *messaging.DebugMessage) { diff --git a/consensus/helpers.go b/consensus/helpers.go index c5eb08da9..681052879 100644 --- a/consensus/helpers.go +++ b/consensus/helpers.go @@ -146,15 +146,11 @@ func protoHash(m proto.Message) string { func (m *consensusModule) sendToLeader(msg *typesCons.HotstuffMessage) { leaderId := m.leaderId - m.logger.Debug().Fields( - map[string]any{ - "src": m.nodeId, - "dst": leaderId, - "height": msg.GetHeight(), - "step": msg.GetStep(), - "round": msg.GetRound(), - }, - ).Msg("✉️ About to try sending hotstuff message ✉️") + + loggingFields := msgToLoggingFields(msg) + loggingFields["src"] = m.nodeId + loggingFields["dst"] = leaderId + m.logger.Debug().Fields(loggingFields).Msg("✉️ About to try sending hotstuff message ✉️") // TODO: This can happen due to a race condition with the pacemaker. if leaderId == nil { @@ -184,13 +180,7 @@ func (m *consensusModule) sendToLeader(msg *typesCons.HotstuffMessage) { // Star-like (O(n)) broadcast - send to all nodes directly // INVESTIGATE: Re-evaluate if we should be using our structured broadcast (RainTree O(log3(n))) algorithm instead func (m *consensusModule) broadcastToValidators(msg *typesCons.HotstuffMessage) { - m.logger.Info().Fields( - map[string]any{ - "height": m.CurrentHeight(), - "step": m.step, - "round": m.round, - }, - ).Msg("📣 Broadcasting message 📣") + m.logger.Info().Fields(msgToLoggingFields(msg)).Msg("📣 Broadcasting message 📣") anyConsensusMessage, err := codec.GetCodec().ToAny(msg) if err != nil { @@ -285,6 +275,6 @@ func msgToLoggingFields(msg *typesCons.HotstuffMessage) map[string]any { return map[string]any{ "height": msg.GetHeight(), "round": msg.GetRound(), - "step": msg.GetStep(), + "step": typesCons.StepToString[msg.GetStep()], } } diff --git a/consensus/module_consensus_pacemaker.go b/consensus/module_consensus_pacemaker.go index 7c2c38b00..fcc2ff4f4 100644 --- a/consensus/module_consensus_pacemaker.go +++ b/consensus/module_consensus_pacemaker.go @@ -28,7 +28,7 @@ func (m *consensusModule) ResetRound(isNewHeight bool) { func (m *consensusModule) ReleaseUtilityUnitOfWork() error { utilityUnitOfWork := m.utilityUnitOfWork if utilityUnitOfWork == nil { - m.logger.Debug().Msg("Try to release utilityUnitOfWork is nil...") + m.logger.Debug().Msg("Try to release a nil utilityUnitOfWork... Ideally this should not happen") return nil } if err := utilityUnitOfWork.Release(); err != nil { diff --git a/p2p/module.go b/p2p/module.go index b58447f2e..2a7fb0f6d 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -9,6 +9,7 @@ import ( "github.com/libp2p/go-libp2p" libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pNetwork "github.com/libp2p/go-libp2p/core/network" + connmgr "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/multiformats/go-multiaddr" "go.uber.org/multierr" "google.golang.org/protobuf/proto" @@ -319,11 +320,19 @@ func (m *p2pModule) setupHost() (err error) { opts = append(opts, m.listenAddrs) } + cm, err := connmgr.NewConnManager(0, 1) + + // cm, _ := connmgr.NewConnManager(2, 10, opts) + opts = append(opts, libp2p.ConnectionManager(cm)) m.host, err = libp2p.New(opts...) if err != nil { return fmt.Errorf("unable to create libp2p host: %w", err) } + // libp2p.ConnectionManager( + // libp2p.ConnectionManagerWithMaxPeers(maxOutboundConns), + // ) + // TECHDEBT(#609): use `StringArrayLogMarshaler` post test-utilities refactor. addrStrs := make(map[int]string) for i, addr := range libp2pHost.InfoFromHost(m.host).Addrs { @@ -341,6 +350,7 @@ func (m *p2pModule) isClientDebugMode() bool { // handleStream is called each time a peer establishes a new stream with this // module's libp2p `host.Host`. func (m *p2pModule) handleStream(stream libp2pNetwork.Stream) { + fmt.Println("OLSH handleStream") peer, err := utils.PeerFromLibp2pStream(stream) if err != nil { m.logger.Error().Err(err). @@ -386,7 +396,10 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { m.logger.Error().Err(err).Msg("handling network data") } - if err := stream.CloseRead(); err != nil { + // if err := stream.CloseRead(); err != nil { + // m.logger.Debug().Err(err).Msg("closing read stream") + // } + if err := stream.Reset(); err != nil { m.logger.Debug().Err(err).Msg("closing read stream") } } diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index c2587ddb2..692a91c55 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -171,6 +171,7 @@ func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket. return nil } + fmt.Println("OLSH networkSendInternal") bus. GetTelemetryModule(). GetEventMetricsAgent(). diff --git a/p2p/utils/host.go b/p2p/utils/host.go index d41c723e5..1d8a8aa56 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -90,11 +90,13 @@ func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) err return fmt.Errorf("opening stream: %w", err) } - if _, err = stream.Write(data); err != nil { + if n, err := stream.Write(data); err != nil { return multierr.Append( fmt.Errorf("writing to stream: %w", err), stream.Reset(), ) + } else { + fmt.Println("OLSH Libp2pSendToPeer Bytes written", n) } return stream.CloseWrite() diff --git a/shared/node.go b/shared/node.go index 0270b0f9a..f9c9a53c9 100644 --- a/shared/node.go +++ b/shared/node.go @@ -2,6 +2,7 @@ package shared import ( "context" + "fmt" "time" "github.com/pokt-network/pocket/consensus" @@ -155,7 +156,9 @@ func (m *Node) GetBus() modules.Bus { // TECHDEBT: The `shared` package has dependencies on types in the individual modules. // TODO: Move all message types this is dependant on to the `messaging` package func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { + contentType := message.GetContentType() + fmt.Println("OLSH handleEvent", message, contentType) switch contentType { case messaging.NodeStartedEventType: logger.Global.Info().Msg("Received NodeStartedEvent") From 2185906a4572c3719dccd17a1b8bd541db2d0c21 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 12 Apr 2023 13:24:46 +0200 Subject: [PATCH 56/82] chore: improve logging --- p2p/module.go | 26 ++++++++++++++++++-- p2p/raintree/network.go | 11 ++++++--- p2p/utils/host.go | 12 ++++++++- p2p/utils/logging.go | 54 +++++++++++++++++++++++++++++++++++++++++ shared/node.go | 8 +++--- 5 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 p2p/utils/logging.go diff --git a/p2p/module.go b/p2p/module.go index 2a7fb0f6d..84ec0a036 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -11,6 +11,7 @@ import ( libp2pNetwork "github.com/libp2p/go-libp2p/core/network" connmgr "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/multiformats/go-multiaddr" + "github.com/rs/zerolog" "go.uber.org/multierr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -350,7 +351,7 @@ func (m *p2pModule) isClientDebugMode() bool { // handleStream is called each time a peer establishes a new stream with this // module's libp2p `host.Host`. func (m *p2pModule) handleStream(stream libp2pNetwork.Stream) { - fmt.Println("OLSH handleStream") + m.logger.Debug().Msg("handling incoming stream") peer, err := utils.PeerFromLibp2pStream(stream) if err != nil { m.logger.Error().Err(err). @@ -383,15 +384,36 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { m.logger.Debug().Err(err).Msg("setting stream read deadline") } + // debug logging: stream scope stats + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#StreamScope) + // TECHDEBT: `logger.Global` is not a `*module.Logger` + _logger := m.logger.Level(zerolog.DebugLevel) + if err := utils.LogScopeStatFactory( + &_logger, + "stream scope (read-side)", + )(stream.Scope()); err != nil { + m.logger.Debug().Err(err).Msg("logging stream scope stats") + } + // --- + data, err := io.ReadAll(stream) if err != nil { m.logger.Error().Err(err).Msg("reading from stream") if err := stream.Reset(); err != nil { - m.logger.Debug().Err(err).Msg("resetting stream") + m.logger.Debug().Err(err).Msg("resetting stream (read-side)") } return } + // debug logging + remotePeer, err := utils.PeerFromLibp2pStream(stream) + if err != nil { + m.logger.Debug().Err(err).Msg("getting remote remotePeer") + } else { + utils.LogIncomingMsg(m.logger, m.cfg.Hostname, remotePeer) + } + // --- + if err := m.handleNetworkData(data); err != nil { m.logger.Error().Err(err).Msg("handling network data") } diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index 692a91c55..b0c347679 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -5,6 +5,7 @@ import ( "log" libp2pHost "github.com/libp2p/go-libp2p/core/host" + "google.golang.org/protobuf/proto" "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/providers" @@ -19,7 +20,6 @@ import ( "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/pocket/shared/modules/base_modules" telemetry "github.com/pokt-network/pocket/telemetry" - "google.golang.org/protobuf/proto" ) var ( @@ -44,7 +44,9 @@ type rainTreeNetwork struct { // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p#section-readme) host libp2pHost.Host // selfAddr is the pocket address representing this host. - selfAddr cryptoPocket.Address + selfAddr cryptoPocket.Address + // hostname is the network hostname from the config + hostname string peersManager *rainTreePeersManager pstoreProvider peerstore_provider.PeerstoreProvider currentHeightProvider providers.CurrentHeightProvider @@ -68,6 +70,7 @@ func (*rainTreeNetwork) Create(bus modules.Bus, netCfg RainTreeConfig) (typesP2P n := &rainTreeNetwork{ host: netCfg.Host, selfAddr: netCfg.Addr, + hostname: p2pCfg.Hostname, nonceDeduper: mempool.NewGenericFIFOSet[uint64, uint64](int(p2pCfg.MaxMempoolCount)), pstoreProvider: netCfg.PeerstoreProvider, currentHeightProvider: netCfg.CurrentHeightProvider, @@ -160,6 +163,9 @@ func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket. return fmt.Errorf("no known peer with pokt address %s", address) } + // debug logging + utils.LogOutgoingMsg(n.logger, n.hostname, peer) + if err := utils.Libp2pSendToPeer(n.host, data, peer); err != nil { n.logger.Debug().Err(err).Msg("from libp2pSendInternal") return err @@ -171,7 +177,6 @@ func (n *rainTreeNetwork) networkSendInternal(data []byte, address cryptoPocket. return nil } - fmt.Println("OLSH networkSendInternal") bus. GetTelemetryModule(). GetEventMetricsAgent(). diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 1d8a8aa56..aeda22164 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -8,6 +8,7 @@ import ( libp2pHost "github.com/libp2p/go-libp2p/core/host" "go.uber.org/multierr" + "github.com/pokt-network/pocket/logger" "github.com/pokt-network/pocket/p2p/protocol" typesP2P "github.com/pokt-network/pocket/p2p/types" ) @@ -85,6 +86,15 @@ func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) err return err } + // debug logging: network resource scope stats + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#ResourceManager) + // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#ResourceScopeViewer) + logScope := LogScopeStatFactory(&logger.Global.Logger, "host transient resource scope") + err = host.Network().ResourceManager().ViewTransient(logScope) + if err != nil { + logger.Global.Debug().Err(err).Msg("logging resource scope stats") + } + stream, err := host.NewStream(ctx, peerInfo.ID, protocol.PoktProtocolID) if err != nil { return fmt.Errorf("opening stream: %w", err) @@ -96,7 +106,7 @@ func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) err stream.Reset(), ) } else { - fmt.Println("OLSH Libp2pSendToPeer Bytes written", n) + logger.Global.Debug().Int("bytes", n).Msg("written to peer stream") } return stream.CloseWrite() diff --git a/p2p/utils/logging.go b/p2p/utils/logging.go new file mode 100644 index 000000000..ffaba15af --- /dev/null +++ b/p2p/utils/logging.go @@ -0,0 +1,54 @@ +package utils + +import ( + "github.com/libp2p/go-libp2p/core/network" + "github.com/pokt-network/pocket/p2p/types" + "github.com/pokt-network/pocket/shared/modules" + "github.com/rs/zerolog" + "net" +) + +type scopeCallback func(scope network.ResourceScope) error + +// LogScopeStatFactory returns a function which prints the given scope stat fields +// to the debug level of the provided logger. +// (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#ScopeStat) +// (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#ResourceScope) +// TECHDEBT: would be nice to receive either a pocket logger object instead. +func LogScopeStatFactory(logger *zerolog.Logger, msg string) scopeCallback { + return func(scope network.ResourceScope) error { + stat := scope.Stat() + logger.Debug().Fields(map[string]any{ + "InboundConns": stat.NumConnsInbound, + "OutboundConns": stat.NumConnsOutbound, + "InboundStreams": stat.NumStreamsInbound, + "OutboundStreams": stat.NumStreamsOutbound, + }).Msg(msg) + return nil + } +} + +func LogOutgoingMsg(logger *modules.Logger, hostname string, peer types.Peer) { + msg := "OUTGOING MSG" + logMessage(logger, msg, hostname, peer) +} + +func LogIncomingMsg(logger *modules.Logger, hostname string, peer types.Peer) { + msg := "INCOMING MSG" + logMessage(logger, msg, hostname, peer) +} + +func logMessage(logger *modules.Logger, msg, hostname string, peer types.Peer) { + remoteHostname, _, err := net.SplitHostPort(peer.GetServiceURL()) + if err != nil { + logger.Debug().Err(err). + Str("serviceURL", peer.GetServiceURL()). + Msg("parsing remote service URL") + return + } + + logger.Debug().Fields(map[string]any{ + "local_hostname": hostname, + "remote_hostname": remoteHostname, + }).Msg(msg) +} diff --git a/shared/node.go b/shared/node.go index fc23ae938..aa4292bfb 100644 --- a/shared/node.go +++ b/shared/node.go @@ -2,7 +2,6 @@ package shared import ( "context" - "fmt" "time" "github.com/pokt-network/pocket/consensus" @@ -157,9 +156,12 @@ func (m *Node) GetBus() modules.Bus { // TECHDEBT: The `shared` package has dependencies on types in the individual modules. // TODO: Move all message types this is dependant on to the `messaging` package func (node *Node) handleEvent(message *messaging.PocketEnvelope) error { - contentType := message.GetContentType() - fmt.Println("OLSH handleEvent", message, contentType) + logger.Global.Debug().Fields(map[string]any{ + "message": message, + "contentType": contentType, + }).Msg("node handling event") + switch contentType { case messaging.NodeStartedEventType: logger.Global.Info().Msg("Received NodeStartedEvent") From 55c8a666a408b4010e4067c9219fda3fa9e96d94 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 12 Apr 2023 13:24:55 +0200 Subject: [PATCH 57/82] fix: stream / resource limit bug (cherry picked from commit fd3246a6571504bcf6e0ca0f3b8e52ea6523d29f) --- p2p/module.go | 20 ++++---------------- p2p/utils/host.go | 5 ++++- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 84ec0a036..e43b372bd 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -9,7 +9,6 @@ import ( "github.com/libp2p/go-libp2p" libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pNetwork "github.com/libp2p/go-libp2p/core/network" - connmgr "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/multiformats/go-multiaddr" "github.com/rs/zerolog" "go.uber.org/multierr" @@ -321,19 +320,11 @@ func (m *p2pModule) setupHost() (err error) { opts = append(opts, m.listenAddrs) } - cm, err := connmgr.NewConnManager(0, 1) - - // cm, _ := connmgr.NewConnManager(2, 10, opts) - opts = append(opts, libp2p.ConnectionManager(cm)) m.host, err = libp2p.New(opts...) if err != nil { return fmt.Errorf("unable to create libp2p host: %w", err) } - // libp2p.ConnectionManager( - // libp2p.ConnectionManagerWithMaxPeers(maxOutboundConns), - // ) - // TECHDEBT(#609): use `StringArrayLogMarshaler` post test-utilities refactor. addrStrs := make(map[int]string) for i, addr := range libp2pHost.InfoFromHost(m.host).Addrs { @@ -405,6 +396,10 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { return } + if err := stream.Reset(); err != nil { + m.logger.Debug().Err(err).Msg("resetting stream (read-side)") + } + // debug logging remotePeer, err := utils.PeerFromLibp2pStream(stream) if err != nil { @@ -417,13 +412,6 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { if err := m.handleNetworkData(data); err != nil { m.logger.Error().Err(err).Msg("handling network data") } - - // if err := stream.CloseRead(); err != nil { - // m.logger.Debug().Err(err).Msg("closing read stream") - // } - if err := stream.Reset(); err != nil { - m.logger.Debug().Err(err).Msg("closing read stream") - } } // handleNetworkData passes a network message to the configured diff --git a/p2p/utils/host.go b/p2p/utils/host.go index aeda22164..4d610a1da 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -109,5 +109,8 @@ func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) err logger.Global.Debug().Int("bytes", n).Msg("written to peer stream") } - return stream.CloseWrite() + // MUST USE `streamClose` NOT `stream.CloswWrite`; otherwise, outbound streams + // will accumulate until resource limits are hit (e.g.): + // > "opening stream: stream-3478: transient: cannot reserve outbound stream: resource limit exceeded" + return stream.Close() } From 9d68e1d801f1f8085803e39f8fa8f29ba0c553bb Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 12 Apr 2023 14:37:00 +0200 Subject: [PATCH 58/82] revert: max conn count config --- build/config/config1.json | 2 +- build/config/config2.json | 2 +- build/config/config3.json | 2 +- build/config/config4.json | 2 +- build/localnet/manifests/configs.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/config/config1.json b/build/config/config1.json index 7d33ff802..06fc027b3 100644 --- a/build/config/config1.json +++ b/build/config/config1.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 5, + "max_conns_count": 50, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/config/config2.json b/build/config/config2.json index 88974fde9..58aa180a6 100644 --- a/build/config/config2.json +++ b/build/config/config2.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 5, + "max_conns_count": 50, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/config/config3.json b/build/config/config3.json index 9ce2a792a..3862126b7 100644 --- a/build/config/config3.json +++ b/build/config/config3.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 5, + "max_conns_count": 50, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/config/config4.json b/build/config/config4.json index c5daa2342..275b5523b 100644 --- a/build/config/config4.json +++ b/build/config/config4.json @@ -22,7 +22,7 @@ "block_store_path": "/var/blockstore", "tx_indexer_path": "", "trees_store_dir": "/var/trees", - "max_conns_count": 5, + "max_conns_count": 50, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", diff --git a/build/localnet/manifests/configs.yaml b/build/localnet/manifests/configs.yaml index f2b772477..418e2399d 100644 --- a/build/localnet/manifests/configs.yaml +++ b/build/localnet/manifests/configs.yaml @@ -28,7 +28,7 @@ data: "block_store_path": "/validator-storage/blockstore", "tx_indexer_path": "", "trees_store_dir": "/validator-storage/trees", - "max_conns_count": 5, + "max_conns_count": 50, "min_conns_count": 1, "max_conn_lifetime": "5m", "max_conn_idle_time": "1m", From 41ddffd39bbe505922bda4da3d3f7be860611c5c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 12 Apr 2023 15:29:07 +0200 Subject: [PATCH 59/82] wip: debugging k8s/tilt localnet > 1:28PM level=FATAL Failed to start pocket node error="setting up libp2pHost: unable to create libp2p host: failed to listen on any addresses: [listen tcp4 10.43.214.215:42069: bind: cannot assign requested address]" --- build/localnet/manifests/cli-client.yaml | 2 ++ build/localnet/manifests/configs.yaml | 1 + build/localnet/templates/v1-validator-template.yaml.tpl | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build/localnet/manifests/cli-client.yaml b/build/localnet/manifests/cli-client.yaml index d2c101ca2..eb037897b 100644 --- a/build/localnet/manifests/cli-client.yaml +++ b/build/localnet/manifests/cli-client.yaml @@ -76,6 +76,8 @@ spec: # Any host that is visible and connected to the cluster can be arbitrarily selected as the RPC host - name: RPC_HOST value: v1-validator001 + - name: POCKET_P2P_HOSTNAME + value: "127.0.0.1" volumeMounts: # IMPROVE: should probably go in /etc/pocket and have Viper read from there as a default path - mountPath: /var/pocket/config diff --git a/build/localnet/manifests/configs.yaml b/build/localnet/manifests/configs.yaml index 418e2399d..8952f31ae 100644 --- a/build/localnet/manifests/configs.yaml +++ b/build/localnet/manifests/configs.yaml @@ -36,6 +36,7 @@ data: }, "p2p": { "port": 42069, + "hostname": "", "use_rain_tree": true, "is_empty_connection_type": false, "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", diff --git a/build/localnet/templates/v1-validator-template.yaml.tpl b/build/localnet/templates/v1-validator-template.yaml.tpl index bbcb1b480..11eafd72c 100644 --- a/build/localnet/templates/v1-validator-template.yaml.tpl +++ b/build/localnet/templates/v1-validator-template.yaml.tpl @@ -64,7 +64,7 @@ spec: - name: POCKET_PERSISTENCE_NODE_SCHEMA value: validator${VALIDATOR_NUMBER} - name: POCKET_P2P_HOSTNAME - value: "v1-validator${VALIDATOR_NUMBER}" + value: "0.0.0.0" volumeMounts: - name: config-volume mountPath: /configs From bb07e7e96e1c34425478b1a4d2d42ce6cc632061 Mon Sep 17 00:00:00 2001 From: Dmitry K Date: Wed, 12 Apr 2023 16:08:59 -0700 Subject: [PATCH 60/82] use pod IP for POCKET_P2P_HOSTNAME --- build/localnet/templates/v1-validator-template.yaml.tpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/localnet/templates/v1-validator-template.yaml.tpl b/build/localnet/templates/v1-validator-template.yaml.tpl index 11eafd72c..0599d2742 100644 --- a/build/localnet/templates/v1-validator-template.yaml.tpl +++ b/build/localnet/templates/v1-validator-template.yaml.tpl @@ -64,7 +64,9 @@ spec: - name: POCKET_PERSISTENCE_NODE_SCHEMA value: validator${VALIDATOR_NUMBER} - name: POCKET_P2P_HOSTNAME - value: "0.0.0.0" + valueFrom: + fieldRef: + fieldPath: status.podIP volumeMounts: - name: config-volume mountPath: /configs From c869310b8aa85293b973c2d85186349ad3dceda2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:38:50 +0200 Subject: [PATCH 61/82] chore: update/improve comments --- p2p/module.go | 9 ++++----- p2p/utils/host.go | 2 +- p2p/utils/logging.go | 4 +++- p2p/utils_test.go | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index e43b372bd..70fc16bbb 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -216,8 +216,7 @@ func (m *p2pModule) GetAddress() (cryptoPocket.Address, error) { return m.address, nil } -// setupDependencies sets up the module's current height and peerstore -// providers, and the network. +// setupDependencies sets up the module's current height and peerstore providers. func (m *p2pModule) setupDependencies() error { if err := m.setupCurrentHeightProvider(); err != nil { return err @@ -297,8 +296,8 @@ func (m *p2pModule) setupNetwork() (err error) { return err } -// setupHost creates a new libp2p host and assignes it to `m.host`, if one does -// not already exist. Libp2p host starts listening upon instantiation. +// setupHost creates a new libp2p host and assignes it to `m.host`. Libp2p host +// starts listening upon instantiation. func (m *p2pModule) setupHost() (err error) { opts := []libp2p.Option{ // Explicitly specify supported transport security options (noise, TLS) @@ -441,7 +440,7 @@ func (m *p2pModule) handleNetworkData(data []byte) error { } // getMultiaddr returns a multiaddr constructed from the `hostname` and `port` -// in the P2P config which pas provided upon creation. +// in the P2P config which was provided upon creation. func (m *p2pModule) getMultiaddr() (multiaddr.Multiaddr, error) { // TECHDEBT: as soon as we add support for multiple transports // (i.e. not just TCP), we'll need to do something else. diff --git a/p2p/utils/host.go b/p2p/utils/host.go index 4d610a1da..56e6e662a 100644 --- a/p2p/utils/host.go +++ b/p2p/utils/host.go @@ -110,7 +110,7 @@ func Libp2pSendToPeer(host libp2pHost.Host, data []byte, peer typesP2P.Peer) err } // MUST USE `streamClose` NOT `stream.CloswWrite`; otherwise, outbound streams - // will accumulate until resource limits are hit (e.g.): + // will accumulate until resource limits are hit; e.g.: // > "opening stream: stream-3478: transient: cannot reserve outbound stream: resource limit exceeded" return stream.Close() } diff --git a/p2p/utils/logging.go b/p2p/utils/logging.go index ffaba15af..60f745569 100644 --- a/p2p/utils/logging.go +++ b/p2p/utils/logging.go @@ -14,7 +14,9 @@ type scopeCallback func(scope network.ResourceScope) error // to the debug level of the provided logger. // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#ScopeStat) // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#ResourceScope) -// TECHDEBT: would be nice to receive either a pocket logger object instead. +// TECHDEBT: would prefer receive a pocket logger object instead. +// Typical calls would pass either `logger.Global` or a `*modules.Logger` which +// are disparate types. func LogScopeStatFactory(logger *zerolog.Logger, msg string) scopeCallback { return func(scope network.ResourceScope) error { stat := scope.Stat() diff --git a/p2p/utils_test.go b/p2p/utils_test.go index 109927e65..8370c5305 100644 --- a/p2p/utils_test.go +++ b/p2p/utils_test.go @@ -150,7 +150,7 @@ func prepareDNSResolverMock(t *testing.T, serviceURLs []string) (done func()) { zones := make(map[string]mockdns.Zone) for i, u := range serviceURLs { // Perpend `scheme://` as serviceURLs are currently scheme-less. - // Required to for parsing to produce useful results. + // Required for parsing to produce useful results. // (see: https://pkg.go.dev/net/url@go1.20.2#URL) serviceURL, err := url.Parse(fmt.Sprintf("scheme://%s", u)) require.NoError(t, err) From b74d28928120e280ef1422dc85d75c4e947a3203 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:40:25 +0200 Subject: [PATCH 62/82] chore: remove unnecessary multierror usage --- p2p/module.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 70fc16bbb..2a4f0ec9c 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -11,7 +11,6 @@ import ( libp2pNetwork "github.com/libp2p/go-libp2p/core/network" "github.com/multiformats/go-multiaddr" "github.com/rs/zerolog" - "go.uber.org/multierr" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -244,8 +243,7 @@ func (m *p2pModule) setupPeerstoreProvider() error { var ok bool m.pstoreProvider, ok = pstoreProviderModule.(providers.PeerstoreProvider) if !ok { - typeErr := fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) - return multierr.Append(err, typeErr) + return fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) } return nil } @@ -268,8 +266,7 @@ func (m *p2pModule) setupCurrentHeightProvider() error { var ok bool m.currentHeightProvider, ok = currentHeightProviderModule.(providers.CurrentHeightProvider) if !ok { - typeErr := fmt.Errorf("unexpected current height provider type: %T", currentHeightProviderModule) - return multierr.Append(err, typeErr) + return fmt.Errorf("unexpected current height provider type: %T", currentHeightProviderModule) } return nil } From a449d37836a059915edb0f7429983d54ff980c8e Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:42:30 +0200 Subject: [PATCH 63/82] chore: improve error log messaging --- p2p/module.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/module.go b/p2p/module.go index 2a4f0ec9c..4d6457157 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -354,7 +354,7 @@ func (m *p2pModule) handleStream(stream libp2pNetwork.Stream) { if err := m.network.AddPeer(peer); err != nil { m.logger.Error().Err(err). Str("address", peer.GetAddress().String()). - Msg("adding remote peer to address book") + Msg("adding remote peer to network") } go m.readStream(stream) From 6bb54ae961d4ee7527ed2ebb16b269d9931c02e4 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:43:38 +0200 Subject: [PATCH 64/82] test: pass `*testing.T` to `getPeerstore` helper --- p2p/raintree/network_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/raintree/network_test.go b/p2p/raintree/network_test.go index 0694f8e46..5fe834cc3 100644 --- a/p2p/raintree/network_test.go +++ b/p2p/raintree/network_test.go @@ -28,7 +28,7 @@ func TestRainTreeNetwork_AddPeer(t *testing.T) { selfAddr := selfPeer.GetAddress() expectedPStoreSize := 0 - pstore := getPeerstore(nil, expectedPStoreSize) + pstore := getPeerstore(t, expectedPStoreSize) peers := pstore.GetPeerList() for _, peer := range peers { libp2pPeerInfo, err := utils.Libp2pAddrInfoFromPeer(peer) @@ -100,7 +100,7 @@ func TestRainTreeNetwork_RemovePeer(t *testing.T) { // Start with a peerstore which contains self and some number of peers: the // initial value of `expectedPStoreSize`. expectedPStoreSize := 3 - pstore := getPeerstore(nil, expectedPStoreSize) + pstore := getPeerstore(t, expectedPStoreSize) selfPeer, host := newTestPeer(t) selfAddr := selfPeer.GetAddress() From 957373082c23db4edff1d1966135be9204758003 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:42:34 +0200 Subject: [PATCH 65/82] chore: remove `use_libp2p` from k8s config --- build/localnet/manifests/configs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/build/localnet/manifests/configs.yaml b/build/localnet/manifests/configs.yaml index 8952f31ae..fe396e438 100644 --- a/build/localnet/manifests/configs.yaml +++ b/build/localnet/manifests/configs.yaml @@ -8,7 +8,6 @@ data: { "root_directory": "/go/src/github.com/pocket-network", "private_key": "", - "use_libp2p": false, "consensus": { "max_mempool_bytes": 500000000, "pacemaker_config": { From 5be78e6981dd16c1e16d9b5ad1c60d5f86003d74 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:45:54 +0200 Subject: [PATCH 66/82] chore: `RainTreeConfig#isValid()` returns multierr --- p2p/raintree/network.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/p2p/raintree/network.go b/p2p/raintree/network.go index b0c347679..a94e52928 100644 --- a/p2p/raintree/network.go +++ b/p2p/raintree/network.go @@ -5,6 +5,7 @@ import ( "log" libp2pHost "github.com/libp2p/go-libp2p/core/host" + "go.uber.org/multierr" "google.golang.org/protobuf/proto" "github.com/pokt-network/pocket/logger" @@ -316,17 +317,17 @@ func (n *rainTreeNetwork) setupPeerManager(pstore typesP2P.Peerstore) (err error return err } -func (cfg RainTreeConfig) isValid() error { +func (cfg RainTreeConfig) isValid() (err error) { if cfg.Host == nil { - return fmt.Errorf("host not configured") + err = multierr.Append(err, fmt.Errorf("host not configured")) } if cfg.PeerstoreProvider == nil { - return fmt.Errorf("peerstore provider not configured") + err = multierr.Append(err, fmt.Errorf("peerstore provider not configured")) } if cfg.CurrentHeightProvider == nil { - return fmt.Errorf("current height provider not configured") + err = multierr.Append(err, fmt.Errorf("current height provider not configured")) } - return nil + return err } From e1f9c5038e7c743b7de59ef7b6d0fe99058f1f96 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 11:46:25 +0200 Subject: [PATCH 67/82] chore: group imports --- p2p/utils/logging.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/p2p/utils/logging.go b/p2p/utils/logging.go index 60f745569..932ae14c8 100644 --- a/p2p/utils/logging.go +++ b/p2p/utils/logging.go @@ -1,11 +1,13 @@ package utils import ( + "net" + "github.com/libp2p/go-libp2p/core/network" + "github.com/rs/zerolog" + "github.com/pokt-network/pocket/p2p/types" "github.com/pokt-network/pocket/shared/modules" - "github.com/rs/zerolog" - "net" ) type scopeCallback func(scope network.ResourceScope) error From 847409b1e90b2f7ea08a455ef202af4c34c56a71 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 12:25:42 +0200 Subject: [PATCH 68/82] fix: re-evaluate options on p2p module `#Start()` --- p2p/module.go | 13 ++++++- p2p/module_test.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/p2p/module.go b/p2p/module.go index 4d6457157..d31c55256 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -46,6 +46,7 @@ type p2pModule struct { address cryptoPocket.Address logger *modules.Logger + options []modules.ModuleOption cfg *configs.P2PConfig bootstrapNodes []string identity libp2p.Option @@ -109,7 +110,8 @@ func (m *p2pModule) Create(bus modules.Bus, options ...modules.ModuleOption) (mo } m.identity = libp2p.Identity(libp2pPrivKey) - for _, option := range options { + m.options = options + for _, option := range m.options { option(m) } @@ -152,6 +154,15 @@ func (m *p2pModule) Start() (err error) { telemetry.P2P_NODE_STARTED_TIMESERIES_METRIC_DESCRIPTION, ) + // TECHDEBT: reconsider if this is acceptable as more `modules.ModuleOption`s + // become supported. At time of writing, `WithHost()` is the only option + // and it is only used in tests. + // Re-evaluate options in case there is a `WithHost` option which would + // assign`m.host`. + for _, option := range m.options { + option(m) + } + // Return early if host has already been started (e.g. via `WithHostOption`) if m.host == nil { if err = m.setupHost(); err != nil { diff --git a/p2p/module_test.go b/p2p/module_test.go index 36bfb224e..592f479b1 100644 --- a/p2p/module_test.go +++ b/p2p/module_test.go @@ -16,6 +16,7 @@ import ( "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/defaults" cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/modules" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" ) @@ -149,6 +150,62 @@ func Test_Create_configureBootstrapNodes(t *testing.T) { } } +func TestP2pModule_WithHostOption_Restart(t *testing.T) { + ctrl := gomock.NewController(t) + + privKey := cryptoPocket.GetPrivKeySeed(1) + mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) + mockBus := createMockBus(t, mockRuntimeMgr) + + genesisStateMock := createMockGenesisState(keys) + persistenceMock := preparePersistenceMock(t, mockBus, genesisStateMock) + mockBus.EXPECT().GetPersistenceModule().Return(persistenceMock).AnyTimes() + + consensusModuleMock := mockModules.NewMockConsensusModule(ctrl) + consensusModuleMock.EXPECT().CurrentHeight().Return(uint64(1)).AnyTimes() + mockBus.EXPECT().GetConsensusModule().Return(consensusModuleMock).AnyTimes() + + telemetryModuleMock := baseTelemetryMock(t, nil) + mockBus.EXPECT().GetTelemetryModule().Return(telemetryModuleMock).AnyTimes() + + mockRuntimeMgr.EXPECT().GetConfig().Return(&configs.Config{ + PrivateKey: privKey.String(), + P2P: &configs.P2PConfig{ + PrivateKey: privKey.String(), + }, + }).AnyTimes() + mockBus.EXPECT().GetRuntimeMgr().Return(mockRuntimeMgr).AnyTimes() + + peer := &typesP2P.NetworkPeer{ + PublicKey: privKey.PublicKey(), + Address: privKey.Address(), + ServiceURL: testLocalServiceURL, + } + + mockNetHost := newLibp2pMockNetHost(t, privKey, peer) + p2pMod, err := Create(mockBus, WithHostOption(mockNetHost)) + require.NoError(t, err) + + mod, ok := p2pMod.(*p2pModule) + require.Truef(t, ok, "unknown module type: %T", mod) + + // start the module; should not create a new host + err = mod.Start() + require.NoError(t, err) + + // assert initial host matches the one provided via `WithHost` + require.Equal(t, mockNetHost, mod.host, "initial hosts don't match") + + // stop the module; destroys module's host + err = mod.Stop() + require.NoError(t, err) + + // assert host matches still after restart + err = mod.Start() + require.NoError(t, err) + require.Equal(t, mockNetHost, mod.host, "post-restart hosts don't match") +} + // TECHDEBT(#609): move & de-duplicate func newLibp2pMockNetHost(t *testing.T, privKey cryptoPocket.PrivateKey, peer *typesP2P.NetworkPeer) libp2pHost.Host { libp2pPrivKey, err := libp2pCrypto.UnmarshalEd25519PrivateKey(privKey.Bytes()) @@ -163,3 +220,36 @@ func newLibp2pMockNetHost(t *testing.T, privKey cryptoPocket.PrivateKey, peer *t return host } + +// TECHDEBT(#609): move & de-duplicate +func baseTelemetryMock(t *testing.T, _ modules.EventsChannel) *mockModules.MockTelemetryModule { + ctrl := gomock.NewController(t) + telemetryMock := mockModules.NewMockTelemetryModule(ctrl) + timeSeriesAgentMock := baseTelemetryTimeSeriesAgentMock(t) + eventMetricsAgentMock := baseTelemetryEventMetricsAgentMock(t) + + telemetryMock.EXPECT().Start().Return(nil).AnyTimes() + telemetryMock.EXPECT().SetBus(gomock.Any()).Return().AnyTimes() + telemetryMock.EXPECT().GetTimeSeriesAgent().Return(timeSeriesAgentMock).AnyTimes() + telemetryMock.EXPECT().GetEventMetricsAgent().Return(eventMetricsAgentMock).AnyTimes() + telemetryMock.EXPECT().GetModuleName().Return(modules.TelemetryModuleName).AnyTimes() + + return telemetryMock +} + +// TECHDEBT(#609): move & de-duplicate +func baseTelemetryTimeSeriesAgentMock(t *testing.T) *mockModules.MockTimeSeriesAgent { + ctrl := gomock.NewController(t) + timeSeriesAgentMock := mockModules.NewMockTimeSeriesAgent(ctrl) + timeSeriesAgentMock.EXPECT().CounterRegister(gomock.Any(), gomock.Any()).AnyTimes() + timeSeriesAgentMock.EXPECT().CounterIncrement(gomock.Any()).AnyTimes() + return timeSeriesAgentMock +} + +// TECHDEBT(#609): move & de-duplicate +func baseTelemetryEventMetricsAgentMock(t *testing.T) *mockModules.MockEventMetricsAgent { + ctrl := gomock.NewController(t) + eventMetricsAgentMock := mockModules.NewMockEventMetricsAgent(ctrl) + eventMetricsAgentMock.EXPECT().EmitEvent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + return eventMetricsAgentMock +} From 39d16cd7adaf03dfd329f06a8935399a9ec8b068 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Thu, 13 Apr 2023 13:16:27 +0200 Subject: [PATCH 69/82] chore: update changelog --- app/client/doc/CHANGELOG.md | 2 +- build/docs/CHANGELOG.md | 6 ++++++ consensus/doc/CHANGELOG.md | 4 ++++ p2p/CHANGELOG.md | 2 +- runtime/docs/CHANGELOG.md | 2 +- shared/CHANGELOG.md | 2 +- 6 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/client/doc/CHANGELOG.md b/app/client/doc/CHANGELOG.md index bddf38800..d6c64aef1 100644 --- a/app/client/doc/CHANGELOG.md +++ b/app/client/doc/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.28] - 2023-04-12 +## [0.0.0.28] - 2023-04-13 - Refactor debug CLI post P2P module re-consolidation diff --git a/build/docs/CHANGELOG.md b/build/docs/CHANGELOG.md index b117e6b47..d236fd7ff 100644 --- a/build/docs/CHANGELOG.md +++ b/build/docs/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.33] - 2023-04-13 + +- Removed runtime/configs.Config#UseLibp2p field +- Use pod IP for validator DNS resolution tilt localnet +- Add `LIBP2P_DEBUG` env var + ## [0.0.0.32] - 2023-04-10 - Adds e2e-tests button to Tiltfile diff --git a/consensus/doc/CHANGELOG.md b/consensus/doc/CHANGELOG.md index 101ea2fbb..040d3a955 100644 --- a/consensus/doc/CHANGELOG.md +++ b/consensus/doc/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.46] - 2023-04-13 + +- Debug logging improvements + ## [0.0.0.45] - 2023-04-12 - Added new helper functions for testing hotstuff consensus stages: `waitForNewRound()`, `waitForPrepareProposal()`, `waitForPrepareVoteswaitForPreCommit()`, `waitForCommit()`, and `waitForDecide()` , and a wrapper function `waitForNextBlock()`. diff --git a/p2p/CHANGELOG.md b/p2p/CHANGELOG.md index 45425f9c2..29ab1ef66 100644 --- a/p2p/CHANGELOG.md +++ b/p2p/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.41] - 2023-04-12 +## [0.0.0.41] - 2023-04-13 - Moved peer & url conversion utils to `p2p/utils` package - Refactor `getPeerIP` to use `net.DefaultResolver` for easier testing diff --git a/runtime/docs/CHANGELOG.md b/runtime/docs/CHANGELOG.md index c83e0c16e..d3d9f7a0b 100644 --- a/runtime/docs/CHANGELOG.md +++ b/runtime/docs/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.32] - 2023-04-12 +## [0.0.0.32] - 2023-04-13 - Removed `runtime/configs.Config#UseLibp2p` field diff --git a/shared/CHANGELOG.md b/shared/CHANGELOG.md index 67e335aa7..69b623268 100644 --- a/shared/CHANGELOG.md +++ b/shared/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.51] - 2023-04-12 +## [0.0.0.51] - 2023-04-13 - Removed *temporary* `shared/p2p` package; consolidated into `p2p` From 10f31173f31ef28fb9c83bd924574ae8cf0c67b0 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 09:42:34 +0200 Subject: [PATCH 70/82] fix: test helper & add TECHDEBT comment --- p2p/utils_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/p2p/utils_test.go b/p2p/utils_test.go index 8370c5305..8d6ab6fae 100644 --- a/p2p/utils_test.go +++ b/p2p/utils_test.go @@ -6,6 +6,7 @@ import ( "net" "net/url" "sort" + "strconv" "sync" "testing" "time" @@ -24,7 +25,6 @@ import ( "github.com/pokt-network/pocket/runtime" "github.com/pokt-network/pocket/runtime/configs" "github.com/pokt-network/pocket/runtime/configs/types" - "github.com/pokt-network/pocket/runtime/defaults" "github.com/pokt-network/pocket/runtime/genesis" "github.com/pokt-network/pocket/runtime/test_artifacts" coreTypes "github.com/pokt-network/pocket/shared/core/types" @@ -78,6 +78,7 @@ type TestNetworkSimulationConfig map[string]struct { } // CLEANUP: This could (should?) be a codebase-wide shared test helper +// TECHDEBT: rename `validatorId()` to `serviceURL()` func validatorId(i int) string { return fmt.Sprintf(serviceURLFormat, i) } @@ -191,13 +192,20 @@ func createMockRuntimeMgrs(t *testing.T, numValidators int) []modules.RuntimeMgr copy(valKeys, keys[:numValidators]) mockGenesisState := createMockGenesisState(valKeys) for i := range mockRuntimeMgrs { + serviceURL := validatorId(i + 1) + hostname, portStr, err := net.SplitHostPort(serviceURL) + require.NoError(t, err) + + port, err := strconv.Atoi(portStr) + require.NoError(t, err) + cfg := &configs.Config{ RootDirectory: "", PrivateKey: valKeys[i].String(), P2P: &configs.P2PConfig{ - Hostname: validatorId(i + 1), + Hostname: hostname, PrivateKey: valKeys[i].String(), - Port: defaults.DefaultP2PPort, + Port: uint32(port), UseRainTree: true, ConnectionType: types.ConnectionType_EmptyConnection, }, From 7033668a0cd29dae1e5a281acc7d89d6c23a05a1 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 09:47:47 +0200 Subject: [PATCH 71/82] chore: add TECHDEBT comment --- build/localnet/manifests/cli-client.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/localnet/manifests/cli-client.yaml b/build/localnet/manifests/cli-client.yaml index eb037897b..712d41592 100644 --- a/build/localnet/manifests/cli-client.yaml +++ b/build/localnet/manifests/cli-client.yaml @@ -76,6 +76,8 @@ spec: # Any host that is visible and connected to the cluster can be arbitrarily selected as the RPC host - name: RPC_HOST value: v1-validator001 + # TECHDEBT(#678): debug client requires hostname to participate + # in P2P networking. - name: POCKET_P2P_HOSTNAME value: "127.0.0.1" volumeMounts: From c7aaeb0ba204ae8dad2bc0bc410250d20a0050ce Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 10:16:03 +0200 Subject: [PATCH 72/82] chore: improve comments --- p2p/module.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/module.go b/p2p/module.go index d31c55256..82e3ba087 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -72,6 +72,7 @@ func Create(bus modules.Bus, options ...modules.ModuleOption) (modules.Module, e // WithHostOption associates an existing (i.e. "started") libp2p `host.Host` // with this module, instead of creating a new one on `#Start()`. +// Primarily intended for testing. func WithHostOption(host libp2pHost.Host) modules.ModuleOption { return func(m modules.InitializableModule) { mod, ok := m.(*p2pModule) From df787e450e388cbe962f3c25fbd35fb7a770b51c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 10:18:12 +0200 Subject: [PATCH 73/82] fix: module restart `WithHost` (again) --- p2p/module.go | 19 ++++++++++--------- p2p/module_test.go | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 82e3ba087..66cf324a4 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -155,17 +155,16 @@ func (m *p2pModule) Start() (err error) { telemetry.P2P_NODE_STARTED_TIMESERIES_METRIC_DESCRIPTION, ) - // TECHDEBT: reconsider if this is acceptable as more `modules.ModuleOption`s - // become supported. At time of writing, `WithHost()` is the only option - // and it is only used in tests. - // Re-evaluate options in case there is a `WithHost` option which would - // assign`m.host`. - for _, option := range m.options { - option(m) - } - // Return early if host has already been started (e.g. via `WithHostOption`) if m.host == nil { + // Libp2p host providea via `WithHost()` option are destroyed when + // `#Stop()`ing the module. Therefore, a new one must be created. + // The new host may be configured differently that which was provided + // originally in `WithHost()`. + if len(m.options) != 0 { + m.logger.Warn().Msg("creating new libp2p host") + } + if err = m.setupHost(); err != nil { return fmt.Errorf("setting up libp2pHost: %w", err) } @@ -308,6 +307,8 @@ func (m *p2pModule) setupNetwork() (err error) { // setupHost creates a new libp2p host and assignes it to `m.host`. Libp2p host // starts listening upon instantiation. func (m *p2pModule) setupHost() (err error) { + m.logger.Debug().Msg("creating new libp2p host") + opts := []libp2p.Option{ // Explicitly specify supported transport security options (noise, TLS) // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.26.3#DefaultSecurity) diff --git a/p2p/module_test.go b/p2p/module_test.go index 592f479b1..9a8cc1e0c 100644 --- a/p2p/module_test.go +++ b/p2p/module_test.go @@ -200,10 +200,10 @@ func TestP2pModule_WithHostOption_Restart(t *testing.T) { err = mod.Stop() require.NoError(t, err) - // assert host matches still after restart + // assert host does *not* match after restart err = mod.Start() require.NoError(t, err) - require.Equal(t, mockNetHost, mod.host, "post-restart hosts don't match") + require.NotEqual(t, mockNetHost, mod.host, "post-restart hosts don't match") } // TECHDEBT(#609): move & de-duplicate From fe0b979ee4ef12166e5dce74bfca9090b0234db0 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 10:18:36 +0200 Subject: [PATCH 74/82] chore: simplify log --- p2p/module.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 66cf324a4..8e79ca18f 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -10,7 +10,6 @@ import ( libp2pHost "github.com/libp2p/go-libp2p/core/host" libp2pNetwork "github.com/libp2p/go-libp2p/core/network" "github.com/multiformats/go-multiaddr" - "github.com/rs/zerolog" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -386,10 +385,8 @@ func (m *p2pModule) readStream(stream libp2pNetwork.Stream) { // debug logging: stream scope stats // (see: https://pkg.go.dev/github.com/libp2p/go-libp2p@v0.27.0/core/network#StreamScope) - // TECHDEBT: `logger.Global` is not a `*module.Logger` - _logger := m.logger.Level(zerolog.DebugLevel) if err := utils.LogScopeStatFactory( - &_logger, + &logger.Global.Logger, "stream scope (read-side)", )(stream.Scope()); err != nil { m.logger.Debug().Err(err).Msg("logging stream scope stats") From 517b940e3cb13bb72413af3c977d685f0720f0fa Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 10:18:53 +0200 Subject: [PATCH 75/82] chore: tidy type assertions --- p2p/module.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/p2p/module.go b/p2p/module.go index 8e79ca18f..7f97a418e 100644 --- a/p2p/module.go +++ b/p2p/module.go @@ -250,11 +250,12 @@ func (m *p2pModule) setupPeerstoreProvider() error { m.logger.Debug().Msg("loaded persistence peerstore...") } - var ok bool - m.pstoreProvider, ok = pstoreProviderModule.(providers.PeerstoreProvider) + pstoreProvider, ok := pstoreProviderModule.(providers.PeerstoreProvider) if !ok { return fmt.Errorf("unknown peerstore provider type: %T", pstoreProviderModule) } + m.pstoreProvider = pstoreProvider + return nil } @@ -273,11 +274,12 @@ func (m *p2pModule) setupCurrentHeightProvider() error { m.logger.Debug().Msg("loaded current height provider") - var ok bool - m.currentHeightProvider, ok = currentHeightProviderModule.(providers.CurrentHeightProvider) + currentHeightProvider, ok := currentHeightProviderModule.(providers.CurrentHeightProvider) if !ok { return fmt.Errorf("unexpected current height provider type: %T", currentHeightProviderModule) } + m.currentHeightProvider = currentHeightProvider + return nil } From 55123e2b18ae1d6acb9d3caf97877900685d2558 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 14 Apr 2023 10:50:49 +0200 Subject: [PATCH 76/82] chore: update changelog --- app/client/doc/CHANGELOG.md | 2 +- p2p/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/client/doc/CHANGELOG.md b/app/client/doc/CHANGELOG.md index d6c64aef1..c38bc48cf 100644 --- a/app/client/doc/CHANGELOG.md +++ b/app/client/doc/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.28] - 2023-04-13 +## [0.0.0.28] - 2023-04-14 - Refactor debug CLI post P2P module re-consolidation diff --git a/p2p/CHANGELOG.md b/p2p/CHANGELOG.md index 29ab1ef66..99b3020a9 100644 --- a/p2p/CHANGELOG.md +++ b/p2p/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.41] - 2023-04-13 +## [0.0.0.41] - 2023-04-14 - Moved peer & url conversion utils to `p2p/utils` package - Refactor `getPeerIP` to use `net.DefaultResolver` for easier testing From b09e98cb6c84742e20714f2ee2b8685783f0c34b Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 17 Apr 2023 09:12:11 +0200 Subject: [PATCH 77/82] chore: remove `use_libp2p` from validator chart --- charts/pocket-validator/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/pocket-validator/values.yaml b/charts/pocket-validator/values.yaml index 25cd3b699..fa9dcd6cb 100644 --- a/charts/pocket-validator/values.yaml +++ b/charts/pocket-validator/values.yaml @@ -70,7 +70,6 @@ privateKeySecretKeyRef: config: root_directory: "/go/src/github.com/pocket-network" private_key: "" # @ignored This value is needed but ignored - use privateKeySecretKeyRef instead - use_libp2p: false consensus: max_mempool_bytes: 500000000 pacemaker_config: From c3f92559e4a149750a437671e47b2cec30dd4d44 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 17 Apr 2023 09:13:21 +0200 Subject: [PATCH 78/82] chore: update validator chart P2P hostname config --- charts/pocket-validator/templates/statefulset.yaml | 4 ++++ charts/pocket-validator/values.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/charts/pocket-validator/templates/statefulset.yaml b/charts/pocket-validator/templates/statefulset.yaml index c58606f72..108975ef9 100644 --- a/charts/pocket-validator/templates/statefulset.yaml +++ b/charts/pocket-validator/templates/statefulset.yaml @@ -35,6 +35,10 @@ spec: "until nc -z $(POSTGRES_HOST) $(POSTGRES_PORT); do echo waiting for postgres...; sleep 2; done;", ] env: + - name: POCKET_P2P_HOSTNAME + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POSTGRES_HOST value: {{ include "pocket-validator.postgresqlHost" . }} - name: POSTGRES_PORT diff --git a/charts/pocket-validator/values.yaml b/charts/pocket-validator/values.yaml index fa9dcd6cb..7ae121492 100644 --- a/charts/pocket-validator/values.yaml +++ b/charts/pocket-validator/values.yaml @@ -92,6 +92,7 @@ config: max_conn_idle_time: 1m health_check_period: 30s p2p: + hostname: "" port: 42069 use_rain_tree: true is_empty_connection_type: false From 7e9343be66e02eab2fd69a80254a7f7b71999bd9 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 17 Apr 2023 07:33:19 +0000 Subject: [PATCH 79/82] add generated helm docs --- charts/pocket-validator/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/pocket-validator/README.md b/charts/pocket-validator/README.md index 40c48b8f8..db8b813d2 100644 --- a/charts/pocket-validator/README.md +++ b/charts/pocket-validator/README.md @@ -45,6 +45,7 @@ privateKeySecretKeyRef: | config.consensus.private_key | string | `""` | | | config.logger.format | string | `"json"` | | | config.logger.level | string | `"debug"` | | +| config.p2p.hostname | string | `""` | | | config.p2p.is_empty_connection_type | bool | `false` | | | config.p2p.max_mempool_count | int | `100000` | | | config.p2p.port | int | `42069` | | @@ -69,7 +70,6 @@ privateKeySecretKeyRef: | config.telemetry.address | string | `"0.0.0.0:9000"` | | | config.telemetry.enabled | bool | `true` | | | config.telemetry.endpoint | string | `"/metrics"` | | -| config.use_libp2p | bool | `false` | | | config.utility.max_mempool_transaction_bytes | int | `1073741824` | | | config.utility.max_mempool_transactions | int | `9000` | | | externalPostgresql.database | string | `""` | name of the external database | From b99e81c1e0a82c3ca020c118b1fbbf2de0b3896c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 17 Apr 2023 09:34:36 +0200 Subject: [PATCH 80/82] chore: update changelog --- app/client/doc/CHANGELOG.md | 2 +- build/docs/CHANGELOG.md | 2 +- consensus/doc/CHANGELOG.md | 2 +- p2p/CHANGELOG.md | 2 +- runtime/docs/CHANGELOG.md | 2 +- shared/CHANGELOG.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/client/doc/CHANGELOG.md b/app/client/doc/CHANGELOG.md index c38bc48cf..48d0ecdb2 100644 --- a/app/client/doc/CHANGELOG.md +++ b/app/client/doc/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.28] - 2023-04-14 +## [0.0.0.28] - 2023-04-17 - Refactor debug CLI post P2P module re-consolidation diff --git a/build/docs/CHANGELOG.md b/build/docs/CHANGELOG.md index 3aec504e9..c5c7cf393 100644 --- a/build/docs/CHANGELOG.md +++ b/build/docs/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.34] - 2023-04-14 +## [0.0.0.35] - 2023-04-17 - Removed runtime/configs.Config#UseLibp2p field - Use pod IP for validator DNS resolution tilt localnet diff --git a/consensus/doc/CHANGELOG.md b/consensus/doc/CHANGELOG.md index 56b4ceecb..b2e6c9651 100644 --- a/consensus/doc/CHANGELOG.md +++ b/consensus/doc/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.47] - 2023-04-14 +## [0.0.0.47] - 2023-04-17 - Debug logging improvements diff --git a/p2p/CHANGELOG.md b/p2p/CHANGELOG.md index 99b3020a9..a7232ef1e 100644 --- a/p2p/CHANGELOG.md +++ b/p2p/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.41] - 2023-04-14 +## [0.0.0.41] - 2023-04-17 - Moved peer & url conversion utils to `p2p/utils` package - Refactor `getPeerIP` to use `net.DefaultResolver` for easier testing diff --git a/runtime/docs/CHANGELOG.md b/runtime/docs/CHANGELOG.md index 8a3032eb9..662cd6c21 100644 --- a/runtime/docs/CHANGELOG.md +++ b/runtime/docs/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.33] - 2023-04-14 +## [0.0.0.33] - 2023-04-17 - Removed `runtime/configs.Config#UseLibp2p` field diff --git a/shared/CHANGELOG.md b/shared/CHANGELOG.md index 8d319c2d7..9be09b733 100644 --- a/shared/CHANGELOG.md +++ b/shared/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [0.0.0.52] - 2023-04-14 +## [0.0.0.52] - 2023-04-17 - Removed *temporary* `shared/p2p` package; consolidated into `p2p` From 09d173a4f78953902a6c3a197d62f3e1583e0dad Mon Sep 17 00:00:00 2001 From: Dmitry K Date: Mon, 17 Apr 2023 10:21:37 -0700 Subject: [PATCH 81/82] switch to new dns introduced with helm charts --- build/localnet/manifests/configs.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/localnet/manifests/configs.yaml b/build/localnet/manifests/configs.yaml index d029d12bf..f99bd29d1 100644 --- a/build/localnet/manifests/configs.yaml +++ b/build/localnet/manifests/configs.yaml @@ -4044,7 +4044,7 @@ data: "address": "00104055c00bed7c983a48aac7dc6335d7c607a7", "public_key": "dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e", "chains": null, - "service_url": "v1-validator001:42069", + "service_url": "validator-001-pocket-validator:42069", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -4055,7 +4055,7 @@ data: "address": "00204737d2a165ebe4be3a7d5b0af905b0ea91d8", "public_key": "eb2c78364525a210d994a83e02d18b4287ab81f6670cf4510ab6c9f51e296d91", "chains": null, - "service_url": "v1-validator002:42069", + "service_url": "validator-002-pocket-validator:42069", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -4066,7 +4066,7 @@ data: "address": "00304d0101847b37fd62e7bebfbdddecdbb7133e", "public_key": "1041a9c76539791fef9bee5b4fcd5bf4a1a489e0790c44cbdfa776b901e13b50", "chains": null, - "service_url": "v1-validator003:42069", + "service_url": "validator-003-pocket-validator:42069", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -4077,7 +4077,7 @@ data: "address": "00404a570febd061274f72b50d0a37f611dfe339", "public_key": "d6cea8706f6ee6672c1e013e667ec8c46231e0e7abcf97ba35d89fceb8edae45", "chains": null, - "service_url": "v1-validator004:42069", + "service_url": "validator-004-pocket-validator:42069", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -4102,7 +4102,7 @@ data: "address": "43d9ea9d9ad9c58bb96ec41340f83cb2cabb6496", "public_key": "16cd0a304c38d76271f74dd3c90325144425d904ef1b9a6fbab9b201d75a998b", "chains": ["0001"], - "service_url": "v1-validator001:42069", + "service_url": "validator-001-pocket-validator:42069", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, @@ -4115,7 +4115,7 @@ data: "address": "9ba047197ec043665ad3f81278ab1f5d3eaf6b8b", "public_key": "68efd26af01692fcd77dc135ca1de69ede464e8243e6832bd6c37f282db8c9cb", "chains": ["0001"], - "service_url": "v1-validator001:42069", + "service_url": "validator-001-pocket-validator:42069", "staked_amount": "1000000000000", "paused_height": -1, "unstaking_height": -1, From 90cf45ea628dfe2863186b5eb8c7130c4191e44f Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 17 Apr 2023 19:42:31 +0200 Subject: [PATCH 82/82] chore: update changelog --- charts/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charts/CHANGELOG.md b/charts/CHANGELOG.md index 0881dd85e..fc778139f 100644 --- a/charts/CHANGELOG.md +++ b/charts/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.2] - 2023-04-17 + +- Removed `runtime/configs.Config#UseLibp2p` field +- Set validator `POCKET_P2P_HOSTNAME` env var to the pod IP +- Set validator `p2p.hostname` config value to empty string so that the env var applies + ## [0.0.0.1] - 2023-04-14 - Introduced `pocket-validator` helm chart.