feat: export pg query metrics to prometheus (#3118)

* expose prometheus metrics for pg query

* add env variable for flag

* remove query label from pg prom metrics

Author: vishal-dt

internal/middleware/instrument.go

@@ -33,6 +33,11 @@ var (
 	}, []string{"path", "method", "status"})
 )
 
+var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name: "pg_query_duration_seconds",
+	Help: "Duration of PG queries",
+}, []string{"label"})
+
 var CdDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
 	Name: "cd_duration_seconds",
 	Help: "Duration of CD process",

pkg/sql/connection.go

@@ -18,6 +18,7 @@
 package sql
 
 import (
+	"github.com/devtron-labs/devtron/internal/middleware"
 	"go.uber.org/zap"
 	"reflect"
 	"time"
@@ -35,6 +36,7 @@ type Config struct {
 	ApplicationName        string `env:"APP" envDefault:"orchestrator"`
 	LogQuery               bool   `env:"PG_LOG_QUERY" envDefault:"true"`
 	LogAllQuery            bool   `env:"PG_LOG_ALL_QUERY" envDefault:"false"`
+	ExportPromMetrics      bool   `env:"PG_EXPORT_PROM_METRICS" envDefault:"false"`
 	QueryDurationThreshold int64  `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000"`
 }
 
@@ -69,14 +71,19 @@ func NewDbConnection(cfg *Config, logger *zap.SugaredLogger) (*pg.DB, error) {
 
 		dbConnection.OnQueryProcessed(func(event *pg.QueryProcessedEvent) {
 			query, err := event.FormattedQuery()
+			queryDuration := time.Since(event.StartTime)
+
+			// Expose prom metrics
+			if cfg.ExportPromMetrics {
+				middleware.PgQueryDuration.WithLabelValues("value").Observe(queryDuration.Seconds())
+			}
+
 			if err != nil {
 				logger.Errorw("Error formatting query",
 					"err", err)
 				return
 			}
 
-			queryDuration := time.Since(event.StartTime)
-
 			if cfg.LogAllQuery || queryDuration.Milliseconds() > cfg.QueryDurationThreshold {
 				logger.Debugw("query time",
 					"duration", queryDuration.Seconds(),