Merge pull request #696 from yeya24/add-metrics-docs

docs: add a doc about how to add metrics

Author: allencloud

apis/swagger.yml

@@ -58,6 +58,17 @@ paths:
         500:
           $ref: "#/responses/500ErrorResponse"
 
+  /metrics:
+    get:
+      summary: "Get Prometheus metrics"
+      description: "Get Prometheus metrics"
+      responses:
+        200:
+          description: "no error"
+          schema:
+            type: "string"
+            example: "go_goroutines 1"
+
   /peer/registry:
     post:
       summary: "registry a task"

cmd/dfdaemon/app/version.go

@@ -31,7 +31,7 @@ func init() {
 
 // versionExample shows examples in version command, and is used in auto-generated cli docs.
 func versionExample() string {
-	return `dfdaemon, version  0.4.1
+	return `dfdaemon version  0.4.1
   Git commit:     6fd5c8f
   Build date:     20190717-15:57:52
   Go version:     go1.12.6

cmd/dfget/app/version.go

@@ -47,7 +47,7 @@ func init() {
 
 // versionExample shows examples in version command, and is used in auto-generated cli docs.
 func versionExample() string {
-	return `dfget, version  0.4.1
+	return `dfget version  0.4.1
   Git commit:     6fd5c8f
   Build date:     20190717-15:57:52
   Go version:     go1.12.6

cmd/supernode/app/version.go

@@ -31,7 +31,7 @@ func init() {
 
 // versionExample shows examples in version command, and is used in auto-generated cli docs.
 func versionExample() string {
-	return `supernode, version  0.4.1
+	return `supernode version  0.4.1
   Git commit:     6fd5c8f
   Build date:     20190717-15:57:52
   Go version:     go1.12.6

common/util/metrics_util.go

@@ -0,0 +1,64 @@
+package util
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	namespace = "dragonfly"
+)
+
+// NewCounter will auto-register a Counter metric to prometheus default registry and return it.
+func NewCounter(subsystem, name, help string, labels []string) *prometheus.CounterVec {
+	return promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+}
+
+// NewGauge will auto-register a Gauge metric to prometheus default registry and return it.
+func NewGauge(subsystem, name, help string, labels []string) *prometheus.GaugeVec {
+	return promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+}
+
+// NewSummary will auto-register a Summary metric to prometheus default registry and return it.
+func NewSummary(subsystem, name, help string, labels []string, objectives map[float64]float64) *prometheus.SummaryVec {
+	return promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  namespace,
+			Subsystem:  subsystem,
+			Name:       name,
+			Help:       help,
+			Objectives: objectives,
+		},
+		labels,
+	)
+}
+
+// NewHistogram will auto-register a Histogram metric to prometheus default registry and return it.
+func NewHistogram(subsystem, name, help string, labels []string, buckets []float64) *prometheus.HistogramVec {
+	return promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      name,
+			Help:      help,
+			Buckets:   buckets,
+		},
+		labels,
+	)
+}

docs/user_guide/monitoring.md

@@ -93,7 +93,9 @@ scrape_configs:
       - targets: ['localhost:8002', 'localhost:65001']
 ```
 
-If you are not familiar with Prometheus, you can modify `prometheus.yml` to this configuration above. Here we don't use any alert rules and alertmanager, so these parts is unset. After modifying this file, you can validate it via `promtool`.
+If you are not familiar with Prometheus, you can modify `prometheus.yml` to this configuration above. We add `localhost:8002` and `localhost:65001` to targets as an example, it represents the ip address of supernode and dfdaemon, which is accessible by Prometheus server. Most of the cases you may have to change to other ip address rather than using `localhost` because these components run in different machines or in docker containers.
+
+Here we don't use any alert rules and alertmanager, so these parts is unset. After modifying this file, you can validate it via `promtool`.
 
 ``` bash
 ./promtool check config prometheus.yml
@@ -111,4 +113,22 @@ Finally you can start Prometheus in the same directory. If Prometheus works well
 
 In Prometheus web ui, you can search Dragonfly metrics below. If you want to learn more about Prometheus query language, please check [promql](https://prometheus.io/docs/prometheus/latest/querying/basics/) for help.
 
-![dragonfly_metrics.png](../images/dragonfly_metrics.png)
+![dragonfly_metrics.png](../images/dragonfly_metrics.png)
+
+### Add your own metrics
+
+Sometimes maybe you want to add your own metrics to Dragonfly. First please ensure you know the basic concepts about Prometheus metrics. If you don't, please check [metrics types](https://prometheus.io/docs/concepts/metric_types/).
+
+We provide several functions to add metrics easily. Here is an example to add a Counter type metric.
+
+``` go
+import "github.com/dragonflyoss/Dragonfly/common/util"
+
+requestCounter := util.NewCounter("supernode", "http_requests_total",
+			"Counter of HTTP requests.", []string{"code"})
+requestCounter.WithLabelValues("200").Inc()
+```
+
+This function will auto-register metrics to Prometheus default registry and you can get `dragonfly_supernode_http_requests_total{code,handler,method}` in /metrics endpoint. Here we also add prefix `dragonfly` to metrics name by default. If you want to learn more about how to use these metrics after getting them, please check [prometheus/client_golang](https://github.com/prometheus/client_golang).
+
+As for naming of metric and label, it is better to follow the best practice. We suggest you to check this [metric and label naming](https://prometheus.io/docs/practices/naming/) guide for more detailed information.

supernode/config/constants.go

@@ -70,8 +70,8 @@ const (
 )
 
 const (
-	// Namespace is the prefix of the metrics' name of dragonfly
-	Namespace = "dragonfly"
-	// Subsystem represents metrics for supernode
-	Subsystem = "supernode"
+	// SubsystemSupernode represents metrics from supernode
+	SubsystemSupernode = "supernode"
+	// SubsystemDfget represents metrics from dfget
+	SubsystemDfget = "dfget"
 )

supernode/server/metrics.go

@@ -3,10 +3,10 @@ package server
 import (
 	"net/http"
 
+	"github.com/dragonflyoss/Dragonfly/common/util"
 	"github.com/dragonflyoss/Dragonfly/supernode/config"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -19,49 +19,23 @@ type metrics struct {
 }
 
 func newMetrics() *metrics {
-	m := &metrics{
-		requestCounter: promauto.NewCounterVec(
-			prometheus.CounterOpts{
-				Namespace: config.Namespace,
-				Subsystem: config.Subsystem,
-				Name:      "http_requests_total",
-				Help:      "Counter of HTTP requests.",
-			},
-			[]string{"code", "handler", "method"},
+	return &metrics{
+		requestCounter: util.NewCounter(config.SubsystemSupernode, "http_requests_total",
+			"Counter of HTTP requests.", []string{"code", "handler", "method"},
 		),
-		requestDuration: promauto.NewHistogramVec(
-			prometheus.HistogramOpts{
-				Namespace: config.Namespace,
-				Subsystem: config.Subsystem,
-				Name:      "http_request_duration_seconds",
-				Help:      "Histogram of latencies for HTTP requests.",
-				Buckets:   []float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
-			},
-			[]string{"code", "handler", "method"},
+		requestDuration: util.NewHistogram(config.SubsystemSupernode, "http_request_duration_seconds",
+			"Histogram of latencies for HTTP requests.", []string{"code", "handler", "method"},
+			[]float64{.1, .2, .4, 1, 3, 8, 20, 60, 120},
 		),
-		requestSize: promauto.NewHistogramVec(
-			prometheus.HistogramOpts{
-				Namespace: config.Namespace,
-				Subsystem: config.Subsystem,
-				Name:      "http_request_size_bytes",
-				Help:      "Histogram of request size for HTTP requests.",
-				Buckets:   prometheus.ExponentialBuckets(100, 10, 8),
-			},
-			[]string{"code", "handler", "method"},
+		requestSize: util.NewHistogram(config.SubsystemSupernode, "http_request_size_bytes",
+			"Histogram of request size for HTTP requests.", []string{"code", "handler", "method"},
+			prometheus.ExponentialBuckets(100, 10, 8),
 		),
-		responseSize: promauto.NewHistogramVec(
-			prometheus.HistogramOpts{
-				Namespace: config.Namespace,
-				Subsystem: config.Subsystem,
-				Name:      "http_response_size_bytes",
-				Help:      "Histogram of response size for HTTP requests.",
-				Buckets:   prometheus.ExponentialBuckets(100, 10, 8),
-			},
-			[]string{"code", "handler", "method"},
+		responseSize: util.NewHistogram(config.SubsystemSupernode, "http_response_size_bytes",
+			"Histogram of response size for HTTP requests.", []string{"code", "handler", "method"},
+			prometheus.ExponentialBuckets(100, 10, 8),
 		),
 	}
-
-	return m
 }
 
 // instrumentHandler will update metrics for every http request

test/util_api.go

@@ -32,7 +32,11 @@ func CheckMetric(c *check.C, metric string, value float64) {
 	lines := strings.Split(string(data), "\n")
 	for _, line := range lines {
 		if strings.Contains(line, metric) {
-			val, err = strconv.ParseFloat(strings.Split(line, " ")[1], 64)
+			vals := strings.Split(line, " ")
+			if len(vals) != 2 {
+				c.Errorf("bad metrics format")
+			}
+			val, err = strconv.ParseFloat(vals[1], 64)
 			c.Assert(err, check.IsNil)
 			c.Assert(val, check.Equals, value)
 			return

version/version.go

@@ -27,10 +27,8 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-
 	"github.com/dragonflyoss/Dragonfly/apis/types"
+	"github.com/dragonflyoss/Dragonfly/common/util"
 )
 
 var (
@@ -80,7 +78,7 @@ func init() {
 
 // versionInfoTmpl contains the template used by Info.
 var versionInfoTmpl = `
-{{.program}}, version  {{.version}}
+{{.program}} version  {{.version}}
   Git commit:     {{.revision}}
   Build date:     {{.buildDate}}
   Go version:     {{.goVersion}}
@@ -109,16 +107,9 @@ func Print(program string) string {
 
 // NewBuildInfo register a collector which exports metrics about version and build information.
 func NewBuildInfo(program string) {
-	buildInfo := promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: "dragonfly",
-			Subsystem: program,
-			Name:      "build_info",
-			Help: fmt.Sprintf(
-				"A metric with a constant '1' value labeled by version, revision, os, arch and goversion from which %s was built.",
-				program,
-			),
-		},
+	buildInfo := util.NewGauge(program, "build_info",
+		fmt.Sprintf("A metric with a constant '1' value labeled by version, revision, os, "+
+			"arch and goversion from which %s was built.", program),
 		[]string{"version", "revision", "os", "arch", "goversion"},
 	)
 	buildInfo.WithLabelValues(version, revision, os, arch, goVersion).Set(1)