feat: Add timer/counter telemetry for CI process (#3081)

* code change 1

* code change 2

* code-change-3

* code change 4

* add-expose-flag

* code change 5

* code change 6

* change-cd-telemetry-trigger-function

* add-deployment-type

* server-mode-change

* remove-loop-import

* add-wire-gen

* Update AppService function name

* remove finished on

* ci-telemetry

* add-flag-and-appName

* add-flag-and-appName

* add-app-name

* improve-ci-metrics-struct

* improve-ci-metrics-struct

* improve-ci-metrics-struct

* add-ci-pipeline

* Add check to Cache Upload

* Code-refactor

Author: Ashish-devtron

Wire.go

@@ -410,6 +410,7 @@ func InitializeApp() (*App, error) {
 		pubsub.NewApplicationStatusHandlerImpl,
 		wire.Bind(new(pubsub.ApplicationStatusHandler), new(*pubsub.ApplicationStatusHandlerImpl)),
 
+		pubsub.GetCiEventConfig,
 		pubsub.NewCiEventHandlerImpl,
 		wire.Bind(new(pubsub.CiEventHandler), new(*pubsub.CiEventHandlerImpl)),
 

api/router/pubsub/CiEventHandler.go

@@ -20,13 +20,25 @@ package pubsub
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/caarlos0/env/v6"
 	pubsub "github.com/devtron-labs/common-lib/pubsub-lib"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
+	"github.com/devtron-labs/devtron/util"
 	"go.uber.org/zap"
 )
 
+type CiEventConfig struct {
+	ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"`
+}
+
+func GetCiEventConfig() (*CiEventConfig, error) {
+	cfg := &CiEventConfig{}
+	err := env.Parse(cfg)
+	return cfg, err
+}
+
 type CiEventHandler interface {
 	Subscribe() error
 	BuildCiArtifactRequest(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error)
@@ -37,6 +49,7 @@ type CiEventHandlerImpl struct {
 	logger         *zap.SugaredLogger
 	pubsubClient   *pubsub.PubSubClientServiceImpl
 	webhookService pipeline.WebhookService
+	ciEventConfig  *CiEventConfig
 }
 
 type CiCompleteEvent struct {
@@ -49,13 +62,16 @@ type CiCompleteEvent struct {
 	PipelineName     string                      `json:"pipelineName"`
 	DataSource       string                      `json:"dataSource"`
 	MaterialType     string                      `json:"materialType"`
+	Metrics          util.CIMetrics              `json:"metrics"`
+	AppName          string                      `json:"appName"`
 }
 
-func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService) *CiEventHandlerImpl {
+func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl {
 	ciEventHandlerImpl := &CiEventHandlerImpl{
 		logger:         logger,
 		pubsubClient:   pubsubClient,
 		webhookService: webhookService,
+		ciEventConfig:  ciEventConfig,
 	}
 	err := ciEventHandlerImpl.Subscribe()
 	if err != nil {
@@ -71,6 +87,7 @@ func (impl *CiEventHandlerImpl) Subscribe() error {
 		//defer msg.Ack()
 		ciCompleteEvent := CiCompleteEvent{}
 		err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent)
+		util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName)
 		if err != nil {
 			impl.logger.Error("error while unmarshalling json data", "error", err)
 			return

api/router/pubsub/WorkflowStatusUpdateHandler.go

@@ -24,7 +24,6 @@ import (
 	"github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	"github.com/devtron-labs/devtron/api/bean"
 	client "github.com/devtron-labs/devtron/client/events"
-	//"github.com/devtron-labs/devtron/client/pubsub"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
 	util "github.com/devtron-labs/devtron/util/event"

internal/middleware/instrument.go

@@ -38,6 +38,42 @@ var CdDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
 	Help: "Duration of CD process",
 }, []string{"appName", "status", "envName", "deploymentType"})
 
+var CiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "ci_duration_seconds",
+	Help:    "Duration of CI process",
+	Buckets: prometheus.LinearBuckets(20, 20, 5),
+}, []string{"pipelineName", "appName"})
+
+var CacheDownloadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "cache_download_duration_seconds",
+	Help:    "Duration of Cache Download process",
+	Buckets: prometheus.LinearBuckets(20, 20, 5),
+}, []string{"pipelineName", "appName"})
+
+var PreCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "pre_ci_duration_seconds",
+	Help:    "Duration of Pre CI process",
+	Buckets: prometheus.LinearBuckets(20, 20, 5),
+}, []string{"pipelineName", "appName"})
+
+var BuildDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "build_duration_seconds",
+	Help:    "Duration of Build process",
+	Buckets: prometheus.LinearBuckets(20, 20, 5),
+}, []string{"pipelineName", "appName"})
+
+var PostCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "post_ci_duration_seconds",
+	Help:    "Duration of Post CI process",
+	Buckets: prometheus.LinearBuckets(20, 20, 5),
+}, []string{"pipelineName", "appName"})
+
+var CacheUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "cache_upload_duration_seconds",
+	Help:    "Duration of Cache Upload process",
+	Buckets: prometheus.LinearBuckets(20, 20, 5),
+}, []string{"pipelineName", "appName"})
+
 var requestCounter = promauto.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "orchestrator_http_requests_total",
@@ -60,7 +96,7 @@ var CdTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{
 
 var CiTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{
 	Name: "ci_trigger_counter",
-}, []string{"appName"})
+}, []string{"appName", "pipelineName"})
 
 // prometheusMiddleware implements mux.MiddlewareFunc.
 func PrometheusMiddleware(next http.Handler) http.Handler {

pkg/pipeline/CiService.go

@@ -157,7 +157,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) {
 	}
 	impl.Logger.Debugw("ci triggered", "wf name ", createdWf.Name, " pipeline ", trigger.PipelineId)
 
-	middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName).Inc()
+	middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc()
 	go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest)
 	return savedCiWf.Id, err
 }

util/helper.go

@@ -45,6 +45,21 @@ type CDMetrics struct {
 	Time            float64
 }
 
+type CIMetrics struct {
+	CacheDownDuration  float64   `json:"cacheDownDuration"`
+	PreCiDuration      float64   `json:"preCiDuration"`
+	BuildDuration      float64   `json:"buildDuration"`
+	PostCiDuration     float64   `json:"postCiDuration"`
+	CacheUpDuration    float64   `json:"cacheUpDuration"`
+	TotalDuration      float64   `json:"totalDuration"`
+	CacheDownStartTime time.Time `json:"cacheDownStartTime"`
+	PreCiStartTime     time.Time `json:"preCiStart"`
+	BuildStartTime     time.Time `json:"buildStartTime"`
+	PostCiStartTime    time.Time `json:"postCiStartTime"`
+	CacheUpStartTime   time.Time `json:"cacheUpStartTime"`
+	TotalStartTime     time.Time `json:"totalStartTime"`
+}
+
 func ContainsString(list []string, element string) bool {
 	if len(list) == 0 {
 		return false
@@ -226,3 +241,20 @@ func TriggerCDMetrics(wfr CDMetrics, exposeCDMetrics bool) {
 		middleware.CdDuration.WithLabelValues(wfr.AppName, wfr.Status, wfr.EnvironmentName, wfr.DeploymentType).Observe(wfr.Time)
 	}
 }
+
+func TriggerCIMetrics(Metrics CIMetrics, exposeCIMetrics bool, PipelineName string, AppName string) {
+	if exposeCIMetrics {
+		middleware.CacheDownloadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDownDuration)
+		middleware.CiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.TotalDuration)
+		if Metrics.CacheUpDuration != 0 {
+			middleware.CacheUploadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheUpDuration)
+		}
+		if Metrics.PostCiDuration != 0 {
+			middleware.PostCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PostCiDuration)
+		}
+		if Metrics.PreCiDuration != 0 {
+			middleware.PreCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PreCiDuration)
+		}
+		middleware.BuildDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.BuildDuration)
+	}
+}