[Telemetry]: Add StdLogAgent and Unify Logging Across the Codebase

CONTRIBUTING.md

@@ -0,0 +1,17 @@
+## How to contribute
+
+Pocket is an open source project, and as such we welcome any contribution from anyone on the internet.
+
+Please fork, code and submit a Pull Request for the team to review and merge. We ask that you please follow the guidelines below in order to submit your contributions for review:
+
+### High impact or architectural changes
+
+Reach out to us on [Telegram](https://t.me/POKTnetwork) and start a discussion with the team regarding your change before you start working. Communication is key for open source projects and asynchronous contributions.
+
+For an active research forum, checkout and post on [our forum](https://research.pokt.network).
+
+### Coding style
+- Code must adhere to the official Go formatting guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt)).
+- (Optional) Use [Editor Config](https://editorconfig.org) to help your Text Editor keep the same formatting used throughout the project.
+- Code must be documented adhering to the official Go commentary guidelines.
+- Pull requests need to be based on and opened against the `main` branch.

Makefile

@@ -41,10 +41,15 @@ go_staticcheck:
 go_clean_deps:
 	go mod tidy && go mod vendor
 
+.PHONY: refresh
+## Removes vendor, installs deps, generates mocks and protobuf files. Perform after a new pull or a branch switch
+refresh: go_clean_deps
+	go mod tidy && go mod vendor && make protogen_clean && make protogen_local
+
 .PHONY: build_and_watch
 ## Continous build Pocket's main entrypoint as files change
 build_and_watch:
-	/bin/sh ${PWD}/scripts/watch_build.sh
+	/bin/sh ${PWD}/build/scripts/watch_build.sh
 
 .PHONY: client_start
 ## Run a client daemon which is only used for debugging purposes
@@ -59,9 +64,14 @@ client_connect:
 # TODO(olshansky): Need to think of a Pocket related name for `compose_and_watch`, maybe just `pocket_watch`?
 .PHONY: compose_and_watch
 ## Run a localnet composed of 4 consensus validators w/ hot reload & debugging
-compose_and_watch: db_start
+compose_and_watch: db_start monitoring_start
 	docker-compose -f build/deployments/docker-compose.yaml up --force-recreate node1.consensus node2.consensus node3.consensus node4.consensus
 
+.PHONY: rebuild_and_compose_and_watch
+## Rebuilds the container from scratch and launches compose_and_watch
+rebuild_and_compose_and_watch: db_start monitoring_start
+	docker-compose -f build/deployments/docker-compose.yaml up --build --force-recreate node1.consensus node2.consensus node3.consensus node4.consensus
+
 .PHONY: db_start
 ## Start a detached local postgres and admin instance (this is auto-triggered by compose_and_watch)
 db_start:
@@ -106,6 +116,16 @@ docker_wipe: prompt_user
 	docker images -q | xargs -r -I {} docker rmi {}
 	docker volume ls -q | xargs -r -I {} docker volume rm {}
 
+.PHONY: monitoring_start
+## Start grafana, metrics and logging system (this is auto-triggered by compose_and_watch)
+monitoring_start:
+	docker-compose -f build/deployments/docker-compose.yaml up --no-recreate -d grafana loki vm
+
+.PHONY: make docker_loki_install
+## Installs the loki docker driver
+docker_loki_install:
+	docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+
 .PHONY: mockgen
 ## Use `mockgen` to generate mocks used for testing purposes of all the modules.
 mockgen:
@@ -115,6 +135,7 @@ mockgen:
 	mockgen --source=${modules_dir}/utility_module.go -destination=${modules_dir}/mocks/utility_module_mock.go -aux_files=github.com/pokt-network/pocket/${modules_dir}=${modules_dir}/module.go
 	mockgen --source=${modules_dir}/consensus_module.go -destination=${modules_dir}/mocks/consensus_module_mock.go -aux_files=github.com/pokt-network/pocket/${modules_dir}=${modules_dir}/module.go
 	mockgen --source=${modules_dir}/bus_module.go -destination=${modules_dir}/mocks/bus_module_mock.go -aux_files=github.com/pokt-network/pocket/${modules_dir}=${modules_dir}/module.go
+	mockgen --source=${modules_dir}/telemetry_module.go -destination=${modules_dir}/mocks/telemetry_module_mock.go -aux_files=github.com/pokt-network/pocket/${modules_dir}=${modules_dir}/module.go
 	echo "Mocks generated in ${modules_dir}/mocks"
 
 	$(eval p2p_types_dir = "p2p/pre2p/types")

README.md

@@ -1,4 +1,4 @@
-<div align="center">
+****<div align="center">
   <a href="https://www.pokt.network">
     <img src="https://user-images.githubusercontent.com/2219004/151564884-212c0e40-3bfa-412e-a341-edb54b5f1498.jpeg" alt="Pocket Network logo" width="340"/>
   </a>

app/client/main.go

@@ -8,9 +8,11 @@ import (
 
 	"github.com/manifoldco/promptui"
 	"github.com/pokt-network/pocket/p2p/pre2p"
+	"github.com/pokt-network/pocket/shared"
 	"github.com/pokt-network/pocket/shared/config"
 	"github.com/pokt-network/pocket/shared/crypto"
 	"github.com/pokt-network/pocket/shared/modules"
+	"github.com/pokt-network/pocket/shared/telemetry"
 	"github.com/pokt-network/pocket/shared/types"
 	typesGenesis "github.com/pokt-network/pocket/shared/types/genesis"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -51,6 +53,7 @@ func main() {
 			UseRainTree:    true,
 			ConnectionType: config.TCPConnection,
 		},
+		EnableTelemetry: false,
 	}
 
 	// Initialize the state singleton
@@ -61,6 +64,18 @@ func main() {
 		log.Fatalf("[ERROR] Failed to create pre2p module: " + err.Error())
 	}
 
+	// This telemetry module instance will be NOOP
+	// because of the client config 'enable_telemetry' key that is set to false.
+	// Since this client mimics partial (networking only) functionality of a full node, some of the telemetry-related
+	// code paths are executed. To avoid those messages interfering with the data collected, a non-nil telemetry
+	// module that NOOPs (per the configs above) is injected.
+	telemetryMod, err := telemetry.Create(cfg)
+	if err != nil {
+		log.Fatalf("[ERROR] Failed to create NOOP telemetry module: " + err.Error())
+	}
+
+	_ = shared.CreateBusWithOptionalModules(nil, pre2pMod, nil, nil, telemetryMod)
+
 	for {
 		selection, err := promptGetInput()
 		if err == nil {

build/Dockerfile

@@ -1,4 +1,4 @@
-ARG GOLANG_IMAGE_VERSION=golang:1.17-rc-alpine3.14
+ARG GOLANG_IMAGE_VERSION=golang:1.18.3-alpine3.16
 
 FROM ${GOLANG_IMAGE_VERSION} AS builder
 

build/config/config1.json

@@ -2,10 +2,12 @@
   "root_dir": "/go/src/github.com/pocket-network",
   "genesis": "build/config/genesis.json",
   "private_key": "2e00000000000000000000000000000000000000000000000000000000000000264a0707979e0d6691f74b055429b5f318d39c2883bb509310b67424252e9ef2",
+  "enable_telemetry": true,
   "pre2p": {
     "consensus_port": 8080,
     "use_raintree": true,
-    "connection_type": "tcp"
+    "connection_type": "tcp",
+    "log_level": "debug"
   },
   "p2p": {
     "protocol": "tcp",
@@ -16,7 +18,8 @@
       "172.18.0.1:8082",
       "172.18.0.1:8083",
       "172.18.0.1:8084"
-    ]
+    ],
+    "log_level": "debug"
   },
   "consensus": {
     "max_mempool_bytes": 500000000,
@@ -25,16 +28,23 @@
       "timeout_msec": 5000,
       "manual": true,
       "debug_time_between_steps_msec": 1000
-    }
+    },
+    "log_level": "all"
   },
   "pre_persistence": {
     "capacity": 99999,
     "mempool_max_bytes": 99999,
-    "mempool_max_txs": 99999
+    "mempool_max_txs": 99999,
+    "log_level": "info"
   },
   "persistence": {
     "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres",
-    "schema": "node1"
+    "schema": "node1",
+    "log_level": "info"
   },
-  "utility": {}
+  "utility": {},
+  "telemetry": {
+    "address": "0.0.0.0:9000",
+    "endpoint": "/metrics"
+  }
 }

build/config/config2.json

@@ -2,21 +2,24 @@
   "root_dir": "/go/src/github.com/pocket-network",
   "genesis": "build/config/genesis.json",
   "private_key": "2d00000000000000000000000000000000000000000000000000000000000000ee37d8c8e9cf42a34cfa75ff1141e2bc0ff2f37483f064dce47cb4d5e69db1d4",
+  "enable_telemetry": true,
   "pre2p": {
     "consensus_port": 8080,
     "use_raintree": true,
-    "connection_type": "tcp"
+    "connection_type": "tcp",
+    "log_level": "all"
   },
   "p2p": {
     "protocol": "tcp",
-    "address": "0.0.0.0:8084",
-    "external_ip": "172.18.0.1:8084",
+    "address": "0.0.0.0:8082",
+    "external_ip": "172.18.0.1:8082",
     "peers": [
       "172.18.0.1:8081",
       "172.18.0.1:8082",
       "172.18.0.1:8083",
       "172.18.0.1:8084"
-    ]
+    ],
+    "log_level": "debug"
   },
   "consensus": {
     "max_mempool_bytes": 500000000,
@@ -25,16 +28,23 @@
       "timeout_msec": 5000,
       "manual": true,
       "debug_time_between_steps_msec": 1000
-    }
+    },
+    "log_level": "debug"
   },
   "pre_persistence": {
     "capacity": 99999,
     "mempool_max_bytes": 99999,
-    "mempool_max_txs": 99999
+    "mempool_max_txs": 99999,
+    "log_level": "info"
   },
   "persistence": {
     "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres",
-    "schema": "node2"
+    "schema": "node2",
+    "log_level": "info"
   },
-  "utility": {}
+  "utility": {},
+  "telemetry": {
+    "address": "0.0.0.0:9000",
+    "endpoint": "/metrics"
+  }
 }

build/config/config3.json

@@ -2,21 +2,24 @@
   "root_dir": "/go/src/github.com/pocket-network",
   "genesis": "build/config/genesis.json",
   "private_key": "2b000000000000000000000000000000000000000000000000000000000000001ba66c6751506850ae0787244c69476b6d45fb857a914a5a0445a24253f7b810",
+  "enable_telemetry": true,
   "pre2p": {
     "consensus_port": 8080,
     "use_raintree": true,
-    "connection_type": "tcp"
+    "connection_type": "tcp",
+    "log_level": "debug"
   },
   "p2p": {
     "protocol": "tcp",
-    "address": "0.0.0.0:8082",
-    "external_ip": "172.18.0.1:8082",
+    "address": "0.0.0.0:8083",
+    "external_ip": "172.18.0.1:8083",
     "peers": [
       "172.18.0.1:8081",
       "172.18.0.1:8082",
       "172.18.0.1:8083",
       "172.18.0.1:8084"
-    ]
+    ],
+    "log_level": "debug"
   },
   "consensus": {
     "max_mempool_bytes": 500000000,
@@ -25,16 +28,23 @@
       "timeout_msec": 5000,
       "manual": true,
       "debug_time_between_steps_msec": 1000
-    }
+    },
+    "log_level": "all"
   },
   "pre_persistence": {
     "capacity": 99999,
     "mempool_max_bytes": 99999,
-    "mempool_max_txs": 99999
+    "mempool_max_txs": 99999,
+    "log_level": "info"
   },
   "persistence": {
     "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres",
-    "schema": "node3"
+    "schema": "node3",
+    "log_level": "info"
   },
-  "utility": {}
+  "utility": {},
+  "telemetry": {
+    "address": "0.0.0.0:9000",
+    "endpoint": "/metrics"
+  }
 }

build/config/config4.json

@@ -2,21 +2,24 @@
   "root_dir": "/go/src/github.com/pocket-network",
   "genesis": "build/config/genesis.json",
   "private_key": "2c00000000000000000000000000000000000000000000000000000000000000f868bcc508133899cc47b612e4f7d9d5dacc90ce1f28214a97b651baa00bf6e4",
+  "enable_telemetry": true,
   "pre2p": {
     "consensus_port": 8080,
     "use_raintree": true,
-    "connection_type": "tcp"
+    "connection_type": "tcp",
+    "log_level": "debug"
   },
   "p2p": {
     "protocol": "tcp",
-    "address": "0.0.0.0:8083",
-    "external_ip": "172.18.0.1:8083",
+    "address": "0.0.0.0:8084",
+    "external_ip": "172.18.0.1:8084",
     "peers": [
       "172.18.0.1:8081",
       "172.18.0.1:8082",
       "172.18.0.1:8083",
       "172.18.0.1:8084"
-    ]
+    ],
+    "log_level": "debug"
   },
   "consensus": {
     "max_mempool_bytes": 500000000,
@@ -25,16 +28,23 @@
       "timeout_msec": 5000,
       "manual": true,
       "debug_time_between_steps_msec": 1000
-    }
+    },
+    "log_level": "all"
   },
   "pre_persistence": {
     "capacity": 99999,
     "mempool_max_bytes": 99999,
-    "mempool_max_txs": 99999
+    "mempool_max_txs": 99999,
+    "log_level": "info"
   },
   "persistence": {
     "postgres_url": "postgres://postgres:postgres@pocket-db:5432/postgres",
-    "schema": "node4"
+    "schema": "node4",
+    "log_level": "info"
   },
-  "utility": {}
+  "utility": {},
+  "telemetry": {
+    "address": "0.0.0.0:9000",
+    "endpoint": "/metrics"
+  }
 }