From 40f816453be2b8892e5037cab4e33bc31f6729c3 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Tue, 21 Feb 2023 16:03:00 +0530 Subject: [PATCH 01/24] code change 1 --- internal/middleware/instrument.go | 9 +++++++-- pkg/app/AppService.go | 5 ++++- pkg/pipeline/CdHandler.go | 4 ++++ pkg/pipeline/CiService.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 4 ++++ 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 0fe8e7bdee..0f9360801f 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -33,6 +33,11 @@ var ( }, []string{"path", "method", "status"}) ) +var CdDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "cd_duration_seconds", + Help: "Duration of CD process", +}, []string{"appName", "status"}) + var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "orchestrator_http_requests_total", @@ -51,11 +56,11 @@ var AcdGetResourceCounter = promauto.NewCounterVec(prometheus.CounterOpts{ var CdTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "cd_trigger_counter", -}, []string{"appId", "envId", "pipelineId"}) +}, []string{"appName", "envName"}) var CiTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ci_trigger_counter", -}, []string{"appId", "pipelineId"}) +}, []string{"appName"}) // prometheusMiddleware implements mux.MiddlewareFunc. func PrometheusMiddleware(next http.Handler) http.Handler { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 12adbe3219..a9ac55b8e2 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1127,7 +1127,7 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR span.End() } } - middleware.CdTriggerCounter.WithLabelValues(strconv.Itoa(pipeline.AppId), strconv.Itoa(pipeline.EnvironmentId), strconv.Itoa(pipeline.Id)).Inc() + middleware.CdTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Environment.Name).Inc() return releaseId, saveErr } @@ -1887,6 +1887,9 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl impl.logger.Errorw("error on update cd workflow runner", "wfr", wfr, "app", app, "err", err) return err } + if wfr.Status == pipelineConfig.WorkflowSucceeded { + middleware.CdDuration.WithLabelValues(wfr.Name, wfr.Status).Observe(time.Since(wfr.UpdatedOn).Seconds()) + } return nil } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 2122acc026..94a43b70fd 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -30,6 +30,7 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/client/argocdServer/application" client2 "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -323,6 +324,9 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel impl.Logger.Errorw("error on update cd workflow runner", "wfr", wfr, "err", err) return err } + if wfr.Status == pipelineConfig.WorkflowSucceeded { + middleware.CdDuration.WithLabelValues(wfr.Name, wfr.Status).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) + } impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) if helmAppStatus == application.Healthy { pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByCdWorkflowId(wfr.CdWorkflowId) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 8e796ae39e..edc754c11e 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -157,7 +157,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { } impl.Logger.Debugw("ci triggered", "wf name ", createdWf.Name, " pipeline ", trigger.PipelineId) - middleware.CiTriggerCounter.WithLabelValues(strconv.Itoa(pipeline.AppId), strconv.Itoa(trigger.PipelineId)).Inc() + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName).Inc() go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest) return savedCiWf.Id, err } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 47d7fc537f..ec8417ecfe 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/argoproj/gitops-engine/pkg/health" blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/util/argo" "go.opentelemetry.io/otel" "strconv" @@ -1065,6 +1066,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo impl.logger.Errorw("error in updating status", "err", err) return err } + middleware.CdDuration.WithLabelValues(runner.Name, runner.Status).Observe(time.Since(runner.FinishedOn).Seconds()) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, @@ -1132,6 +1134,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne impl.logger.Errorw("error updating cd wf runner status", "err", err, "currentRunner", currentRunner) return err } + middleware.CdDuration.WithLabelValues(currentRunner.Name, currentRunner.Status).Observe(time.Since(currentRunner.FinishedOn).Seconds()) return nil //update current WF with error status } else { @@ -1409,6 +1412,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("err", "err", err) return 0, err } + middleware.CdDuration.WithLabelValues(runner.Name, runner.Status).Observe(time.Since(triggeredAt).Seconds()) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, From dc8146214676524ac52ff31a7227254bd3e10e51 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Tue, 21 Feb 2023 18:57:58 +0530 Subject: [PATCH 02/24] code change 2 --- internal/middleware/instrument.go | 2 +- pkg/app/AppService.go | 2 +- pkg/pipeline/CdHandler.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 0f9360801f..cb6d062cb4 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -36,7 +36,7 @@ var ( var CdDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cd_duration_seconds", Help: "Duration of CD process", -}, []string{"appName", "status"}) +}, []string{"appName", "status", "envName"}) var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index a9ac55b8e2..6f72f8d343 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1888,7 +1888,7 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl return err } if wfr.Status == pipelineConfig.WorkflowSucceeded { - middleware.CdDuration.WithLabelValues(wfr.Name, wfr.Status).Observe(time.Since(wfr.UpdatedOn).Seconds()) + middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) } return nil } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 94a43b70fd..c5babe87a3 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -325,7 +325,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel return err } if wfr.Status == pipelineConfig.WorkflowSucceeded { - middleware.CdDuration.WithLabelValues(wfr.Name, wfr.Status).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) + middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) } impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) if helmAppStatus == application.Healthy { diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index ec8417ecfe..d4ea38df0c 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -1066,7 +1066,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo impl.logger.Errorw("error in updating status", "err", err) return err } - middleware.CdDuration.WithLabelValues(runner.Name, runner.Status).Observe(time.Since(runner.FinishedOn).Seconds()) + middleware.CdDuration.WithLabelValues(runner.CdWorkflow.Pipeline.DeploymentAppName, runner.Status, runner.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(runner.FinishedOn).Seconds() - time.Since(runner.StartedOn).Seconds()) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, @@ -1134,7 +1134,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne impl.logger.Errorw("error updating cd wf runner status", "err", err, "currentRunner", currentRunner) return err } - middleware.CdDuration.WithLabelValues(currentRunner.Name, currentRunner.Status).Observe(time.Since(currentRunner.FinishedOn).Seconds()) + middleware.CdDuration.WithLabelValues(currentRunner.CdWorkflow.Pipeline.DeploymentAppName, currentRunner.Status, currentRunner.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(currentRunner.FinishedOn).Seconds() - time.Since(currentRunner.StartedOn).Seconds()) return nil //update current WF with error status } else { @@ -1412,7 +1412,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("err", "err", err) return 0, err } - middleware.CdDuration.WithLabelValues(runner.Name, runner.Status).Observe(time.Since(triggeredAt).Seconds()) + middleware.CdDuration.WithLabelValues(runner.CdWorkflow.Pipeline.DeploymentAppName, runner.Status, runner.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(triggeredAt).Seconds()) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, From bbe73fdb3c55e984605549fef80db4258a5544a8 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Wed, 22 Feb 2023 12:25:27 +0530 Subject: [PATCH 03/24] code-change-3 --- pkg/pipeline/CdHandler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index c5babe87a3..c4ad43e554 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -236,6 +236,7 @@ func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch impl.Logger.Errorw("error on update cd workflow runner", "cdWfr", cdWfr, "err", err) return err, isTimelineUpdated } + middleware.CdDuration.WithLabelValues(cdWfr.CdWorkflow.Pipeline.DeploymentAppName, cdWfr.Status, cdWfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(cdWfr.FinishedOn).Seconds() - time.Since(cdWfr.StartedOn).Seconds()) // creating cd pipeline status timeline timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: cdWfr.Id, From 576047d72c4a4d499f2ee5b6fb3c480141b13bc0 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 23 Feb 2023 18:25:11 +0530 Subject: [PATCH 04/24] code change 4 --- pkg/app/AppService.go | 2 +- pkg/pipeline/CdHandler.go | 8 +++++--- pkg/pipeline/WorkflowDagExecutor.go | 8 ++++---- util/helper.go | 6 ++++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 6f72f8d343..a02c55465b 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1888,7 +1888,7 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl return err } if wfr.Status == pipelineConfig.WorkflowSucceeded { - middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) + util2.CDDurationTelemetry(wfr) } return nil } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index c4ad43e554..f63e65dad0 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -30,7 +30,6 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/client/argocdServer/application" client2 "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -236,7 +235,7 @@ func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch impl.Logger.Errorw("error on update cd workflow runner", "cdWfr", cdWfr, "err", err) return err, isTimelineUpdated } - middleware.CdDuration.WithLabelValues(cdWfr.CdWorkflow.Pipeline.DeploymentAppName, cdWfr.Status, cdWfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(cdWfr.FinishedOn).Seconds() - time.Since(cdWfr.StartedOn).Seconds()) + util3.CDDurationTelemetry(&cdWfr) // creating cd pipeline status timeline timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: cdWfr.Id, @@ -326,7 +325,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel return err } if wfr.Status == pipelineConfig.WorkflowSucceeded { - middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) + util3.CDDurationTelemetry(wfr) } impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) if helmAppStatus == application.Healthy { @@ -451,6 +450,9 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) return 0, "", err } + if savedWorkflow.Status == pipelineConfig.WorkflowSucceeded || savedWorkflow.Status == pipelineConfig.WorkflowFailed { + util3.CDDurationTelemetry(savedWorkflow) + } if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index d4ea38df0c..c416a4f77a 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,7 +23,7 @@ import ( "fmt" "github.com/argoproj/gitops-engine/pkg/health" blob_storage "github.com/devtron-labs/common-lib/blob-storage" - "github.com/devtron-labs/devtron/internal/middleware" + util4 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "go.opentelemetry.io/otel" "strconv" @@ -1066,7 +1066,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo impl.logger.Errorw("error in updating status", "err", err) return err } - middleware.CdDuration.WithLabelValues(runner.CdWorkflow.Pipeline.DeploymentAppName, runner.Status, runner.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(runner.FinishedOn).Seconds() - time.Since(runner.StartedOn).Seconds()) + util4.CDDurationTelemetry(runner) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, @@ -1134,7 +1134,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne impl.logger.Errorw("error updating cd wf runner status", "err", err, "currentRunner", currentRunner) return err } - middleware.CdDuration.WithLabelValues(currentRunner.CdWorkflow.Pipeline.DeploymentAppName, currentRunner.Status, currentRunner.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(currentRunner.FinishedOn).Seconds() - time.Since(currentRunner.StartedOn).Seconds()) + util4.CDDurationTelemetry(currentRunner) return nil //update current WF with error status } else { @@ -1412,7 +1412,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("err", "err", err) return 0, err } - middleware.CdDuration.WithLabelValues(runner.CdWorkflow.Pipeline.DeploymentAppName, runner.Status, runner.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(triggeredAt).Seconds()) + util4.CDDurationTelemetry(runner) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, diff --git a/util/helper.go b/util/helper.go index 8563783fc8..e514f0e01f 100644 --- a/util/helper.go +++ b/util/helper.go @@ -22,6 +22,8 @@ import ( "compress/gzip" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/juju/errors" "io" "io/ioutil" @@ -208,3 +210,7 @@ func InterfaceToMapAdapter(resp interface{}) map[string]interface{} { } return dat } + +func CDDurationTelemetry(wfr *pipelineConfig.CdWorkflowRunner) { + middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) +} From 1d11900f96d12af5fab9a4acb5184c59c6422a2e Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 23 Feb 2023 19:46:49 +0530 Subject: [PATCH 05/24] add-expose-flag --- pkg/app/AppService.go | 27 ++++++--- pkg/app/integrationTest/AppService_test.go | 66 ++++++++++++++++++---- pkg/pipeline/CdConfig.go | 1 + pkg/pipeline/CdHandler.go | 6 +- pkg/pipeline/WorkflowDagExecutor.go | 6 +- util/helper.go | 7 ++- wire_gen.go | 2 +- 7 files changed, 88 insertions(+), 27 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index a02c55465b..99e3fdb9c5 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -30,6 +30,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/dockerRegistry" + "github.com/devtron-labs/devtron/pkg/pipeline" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/k8s" @@ -154,6 +155,7 @@ type AppServiceImpl struct { gitOpsConfigRepository repository.GitOpsConfigRepository appStatusService appStatus.AppStatusService k8sApplicationService k8s.K8sApplicationService + cdConfig *pipeline.CdConfig } type AppService interface { @@ -185,20 +187,26 @@ func NewAppService( appListingRepository repository.AppListingRepository, appRepository app.AppRepository, envRepository repository2.EnvironmentRepository, - pipelineConfigRepository chartConfig.PipelineConfigRepository, configMapRepository chartConfig.ConfigMapRepository, - appLevelMetricsRepository repository.AppLevelMetricsRepository, envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, + pipelineConfigRepository chartConfig.PipelineConfigRepository, + configMapRepository chartConfig.ConfigMapRepository, + appLevelMetricsRepository repository.AppLevelMetricsRepository, + envLevelMetricsRepository repository.EnvLevelAppMetricsRepository, chartRepository chartRepoRepository.ChartRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, commonService commonService.CommonService, - imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, imageScanHistoryRepository security.ImageScanHistoryRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + commonService commonService.CommonService, + imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, + imageScanHistoryRepository security.ImageScanHistoryRepository, ArgoK8sClient argocdServer.ArgoK8sClient, gitFactory *GitFactory, pipelineStrategyHistoryService history2.PipelineStrategyHistoryService, configMapHistoryService history2.ConfigMapHistoryService, deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService, - chartTemplateService ChartTemplateService, refChartDir chartRepoRepository.RefChartDir, + chartTemplateService ChartTemplateService, + refChartDir chartRepoRepository.RefChartDir, chartRefRepository chartRepoRepository.ChartRefRepository, - chartService chart.ChartService, helmAppClient client2.HelmAppClient, + chartService chart.ChartService, + helmAppClient client2.HelmAppClient, argoUserService argo.ArgoUserService, cdPipelineStatusTimelineRepo pipelineConfig.PipelineStatusTimelineRepository, appCrudOperationService AppCrudOperationService, @@ -211,7 +219,9 @@ func NewAppService( pipelineStatusTimelineService PipelineStatusTimelineService, appStatusConfig *AppStatusConfig, gitOpsConfigRepository repository.GitOpsConfigRepository, - appStatusService appStatus.AppStatusService, k8sApplicationService k8s.K8sApplicationService) *AppServiceImpl { + appStatusService appStatus.AppStatusService, + k8sApplicationService k8s.K8sApplicationService, + cdConfig *pipeline.CdConfig) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -265,6 +275,7 @@ func NewAppService( gitOpsConfigRepository: gitOpsConfigRepository, appStatusService: appStatusService, k8sApplicationService: k8sApplicationService, + cdConfig: cdConfig, } return appServiceImpl } @@ -1888,7 +1899,7 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl return err } if wfr.Status == pipelineConfig.WorkflowSucceeded { - util2.CDDurationTelemetry(wfr) + util2.CDDurationTelemetry(wfr, impl.cdConfig) } return nil } diff --git a/pkg/app/integrationTest/AppService_test.go b/pkg/app/integrationTest/AppService_test.go index f1507ed27e..703ded3e6b 100644 --- a/pkg/app/integrationTest/AppService_test.go +++ b/pkg/app/integrationTest/AppService_test.go @@ -20,6 +20,7 @@ import ( repository1 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/module" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" + "github.com/devtron-labs/devtron/pkg/pipeline" serverEnvConfig "github.com/devtron-labs/devtron/pkg/server/config" "github.com/devtron-labs/devtron/pkg/sql" repository2 "github.com/devtron-labs/devtron/pkg/user/repository" @@ -112,6 +113,9 @@ func InitAppService() *app2.AppServiceImpl { if err != nil { log.Fatal("error in getting db connection, AppService_test", "err", err) } + + cdConfig, _ := pipeline.GetCdConfig() + pipelineOverrideRepository := chartConfig.NewPipelineOverrideRepository(dbConnection) pipelineRepository := pipelineConfig.NewPipelineRepositoryImpl(dbConnection, logger) httpClient := util.NewHttpClient() @@ -132,7 +136,7 @@ func InitAppService() *app2.AppServiceImpl { log.Fatal("error in getting server helm client config, AppService_test", "err", err) } helmAppClient := client.NewHelmAppClientImpl(logger, helmClientConfig) - helmAppService := client.NewHelmAppServiceImpl(logger, clusterService, helmAppClient, nil, nil, nil, serverEnvConfig, nil, nil, nil, nil) + helmAppService := client.NewHelmAppServiceImpl(logger, clusterService, helmAppClient, nil, nil, nil, serverEnvConfig, nil, nil, nil, nil, nil, nil) moduleService := module.NewModuleServiceImpl(logger, serverEnvConfig, moduleRepositoryImpl, moduleActionAuditLogRepository, helmAppService, nil, nil, nil, nil, nil, nil) eventClient := client1.NewEventRESTClientImpl(logger, httpClient, eventClientConfig, pubSubClient, ciPipelineRepositoryImpl, pipelineRepository, attributesRepositoryImpl, moduleService) @@ -141,7 +145,7 @@ func InitAppService() *app2.AppServiceImpl { ciPipelineMaterialRepository := pipelineConfig.NewCiPipelineMaterialRepositoryImpl(dbConnection, logger) userRepository := repository2.NewUserRepositoryImpl(dbConnection, logger) eventFactory := client1.NewEventSimpleFactoryImpl(logger, cdWorkflowRepository, pipelineOverrideRepository, ciWorkflowRepository, - ciPipelineMaterialRepository, ciPipelineRepositoryImpl, pipelineRepository, userRepository) + ciPipelineMaterialRepository, ciPipelineRepositoryImpl, pipelineRepository, userRepository, nil) appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(logger) appListingRepository := repository.NewAppListingRepositoryImpl(logger, dbConnection, appListingRepositoryQueryBuilder) appRepository := app.NewAppRepositoryImpl(dbConnection, logger) @@ -153,13 +157,55 @@ func InitAppService() *app2.AppServiceImpl { pipelineStatusSyncDetailService := app2.NewPipelineStatusSyncDetailServiceImpl(logger, pipelineStatusSyncDetailRepository) pipelineStatusTimelineService := app2.NewPipelineStatusTimelineServiceImpl(logger, pipelineStatusTimelineRepository, cdWorkflowRepository, nil, pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService) refChartDir := chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts") - appService := app2.NewAppService(nil, pipelineOverrideRepository, nil, logger, nil, - pipelineRepository, nil, eventClient, eventFactory, nil, nil, nil, nil, nil, nil, - appListingRepository, appRepository, nil, nil, nil, nil, nil, - chartRepository, nil, cdWorkflowRepository, nil, nil, nil, nil, - nil, nil, nil, nil, nil, refChartDir, nil, - nil, nil, nil, pipelineStatusTimelineRepository, nil, nil, nil, - nil, nil, pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService, pipelineStatusTimelineService, - nil) + appService := app2.NewAppService( + nil, + pipelineOverrideRepository, + nil, + logger, + nil, + pipelineRepository, + nil, + eventClient, + eventFactory, + nil, + nil, + nil, + nil, + nil, + nil, + appListingRepository, + appRepository, + nil, + nil, + nil, + nil, + nil, + chartRepository, + nil, + cdWorkflowRepository, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + refChartDir, + nil, + nil, + nil, + nil, + pipelineStatusTimelineRepository, + nil, + nil, + nil, + nil, + nil, + pipelineStatusTimelineResourcesService, + pipelineStatusSyncDetailService, + pipelineStatusTimelineService, + nil, nil, nil, nil, cdConfig) return appService } diff --git a/pkg/pipeline/CdConfig.go b/pkg/pipeline/CdConfig.go index c45e26c896..ca7cc8aa4e 100644 --- a/pkg/pipeline/CdConfig.go +++ b/pkg/pipeline/CdConfig.go @@ -70,6 +70,7 @@ type CdConfig struct { BuildLogTTLValue int `env:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600"` DefaultAddressPoolBaseCidr string `env:"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR"` DefaultAddressPoolSize int `env:"CD_DEFAULT_ADDRESS_POOL_SIZE"` + ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` } func GetCdConfig() (*CdConfig, error) { diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index f63e65dad0..c83ebcabc5 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -235,7 +235,7 @@ func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch impl.Logger.Errorw("error on update cd workflow runner", "cdWfr", cdWfr, "err", err) return err, isTimelineUpdated } - util3.CDDurationTelemetry(&cdWfr) + util3.CDDurationTelemetry(&cdWfr, impl.cdConfig) // creating cd pipeline status timeline timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: cdWfr.Id, @@ -325,7 +325,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel return err } if wfr.Status == pipelineConfig.WorkflowSucceeded { - util3.CDDurationTelemetry(wfr) + util3.CDDurationTelemetry(wfr, impl.cdConfig) } impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) if helmAppStatus == application.Healthy { @@ -451,7 +451,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus return 0, "", err } if savedWorkflow.Status == pipelineConfig.WorkflowSucceeded || savedWorkflow.Status == pipelineConfig.WorkflowFailed { - util3.CDDurationTelemetry(savedWorkflow) + util3.CDDurationTelemetry(savedWorkflow, impl.cdConfig) } if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index c416a4f77a..1c09e2e887 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -1066,7 +1066,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo impl.logger.Errorw("error in updating status", "err", err) return err } - util4.CDDurationTelemetry(runner) + util4.CDDurationTelemetry(runner, impl.cdConfig) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, @@ -1134,7 +1134,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne impl.logger.Errorw("error updating cd wf runner status", "err", err, "currentRunner", currentRunner) return err } - util4.CDDurationTelemetry(currentRunner) + util4.CDDurationTelemetry(currentRunner, impl.cdConfig) return nil //update current WF with error status } else { @@ -1412,7 +1412,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("err", "err", err) return 0, err } - util4.CDDurationTelemetry(runner) + util4.CDDurationTelemetry(runner, impl.cdConfig) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, diff --git a/util/helper.go b/util/helper.go index e514f0e01f..a0b1893ee4 100644 --- a/util/helper.go +++ b/util/helper.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/juju/errors" "io" "io/ioutil" @@ -211,6 +212,8 @@ func InterfaceToMapAdapter(resp interface{}) map[string]interface{} { return dat } -func CDDurationTelemetry(wfr *pipelineConfig.CdWorkflowRunner) { - middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) +func CDDurationTelemetry(wfr *pipelineConfig.CdWorkflowRunner, cdConfig *pipeline.CdConfig) { + if cdConfig.ExposeCDMetrics { + middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) + } } diff --git a/wire_gen.go b/wire_gen.go index 1167298d1a..bf45982468 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -335,7 +335,6 @@ func InitializeApp() (*App, error) { k8sResourceHistoryRepositoryImpl := repository8.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) k8sApplicationServiceImpl := k8s.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, k8sClientServiceImpl, helmAppServiceImpl, k8sUtil, acdAuthConfig, k8sResourceHistoryServiceImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appStatusConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, k8sApplicationServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -344,6 +343,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appStatusConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, k8sApplicationServiceImpl,cdConfig) cdWorkflowServiceImpl := pipeline.NewCdWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, cdConfig, appServiceImpl) materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) deploymentGroupRepositoryImpl := repository.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) From 226eb312335e1095bf4200f87be92ea9c46e8b2e Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 24 Feb 2023 11:38:38 +0530 Subject: [PATCH 06/24] code change 5 --- pkg/app/AppService.go | 6 ++---- pkg/pipeline/CdHandler.go | 9 ++------- pkg/pipeline/WorkflowDagExecutor.go | 6 +++--- util/helper.go | 4 ++-- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 99e3fdb9c5..503f7285cd 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1881,12 +1881,12 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl impl.logger.Errorw("error on update cd workflow runner, fetch failed for runner type", "wfr", wfr, "app", app, "err", err) return err } - wfr.FinishedOn = time.Now() if updateTimedOutStatus { wfr.Status = pipelineConfig.WorkflowTimedOut } else { if app.Status.Health.Status == health.HealthStatusHealthy { wfr.Status = pipelineConfig.WorkflowSucceeded + wfr.FinishedOn = time.Now() } else { wfr.Status = pipelineConfig.WorkflowInProgress } @@ -1898,9 +1898,7 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl impl.logger.Errorw("error on update cd workflow runner", "wfr", wfr, "app", app, "err", err) return err } - if wfr.Status == pipelineConfig.WorkflowSucceeded { - util2.CDDurationTelemetry(wfr, impl.cdConfig) - } + util2.TriggerCDMetrics(wfr, impl.cdConfig) return nil } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index c83ebcabc5..3d102376f0 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -235,7 +235,6 @@ func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch impl.Logger.Errorw("error on update cd workflow runner", "cdWfr", cdWfr, "err", err) return err, isTimelineUpdated } - util3.CDDurationTelemetry(&cdWfr, impl.cdConfig) // creating cd pipeline status timeline timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: cdWfr.Id, @@ -324,9 +323,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel impl.Logger.Errorw("error on update cd workflow runner", "wfr", wfr, "err", err) return err } - if wfr.Status == pipelineConfig.WorkflowSucceeded { - util3.CDDurationTelemetry(wfr, impl.cdConfig) - } + util3.TriggerCDMetrics(wfr, impl.cdConfig) impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) if helmAppStatus == application.Healthy { pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByCdWorkflowId(wfr.CdWorkflowId) @@ -450,9 +447,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) return 0, "", err } - if savedWorkflow.Status == pipelineConfig.WorkflowSucceeded || savedWorkflow.Status == pipelineConfig.WorkflowFailed { - util3.CDDurationTelemetry(savedWorkflow, impl.cdConfig) - } + util3.TriggerCDMetrics(savedWorkflow, impl.cdConfig) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 1c09e2e887..f86f1d0885 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -1066,7 +1066,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo impl.logger.Errorw("error in updating status", "err", err) return err } - util4.CDDurationTelemetry(runner, impl.cdConfig) + util4.TriggerCDMetrics(runner, impl.cdConfig) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, @@ -1134,7 +1134,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne impl.logger.Errorw("error updating cd wf runner status", "err", err, "currentRunner", currentRunner) return err } - util4.CDDurationTelemetry(currentRunner, impl.cdConfig) + util4.TriggerCDMetrics(currentRunner, impl.cdConfig) return nil //update current WF with error status } else { @@ -1412,7 +1412,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("err", "err", err) return 0, err } - util4.CDDurationTelemetry(runner, impl.cdConfig) + util4.TriggerCDMetrics(runner, impl.cdConfig) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, diff --git a/util/helper.go b/util/helper.go index a0b1893ee4..0b61ce060b 100644 --- a/util/helper.go +++ b/util/helper.go @@ -212,8 +212,8 @@ func InterfaceToMapAdapter(resp interface{}) map[string]interface{} { return dat } -func CDDurationTelemetry(wfr *pipelineConfig.CdWorkflowRunner, cdConfig *pipeline.CdConfig) { - if cdConfig.ExposeCDMetrics { +func TriggerCDMetrics(wfr *pipelineConfig.CdWorkflowRunner, cdConfig *pipeline.CdConfig) { + if cdConfig.ExposeCDMetrics && (wfr.Status == pipelineConfig.WorkflowFailed || wfr.Status == pipelineConfig.WorkflowSucceeded) { middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) } } From baeb8b7b9b64a390720cc5c95ae4c58b2e80ee9a Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 24 Feb 2023 12:33:40 +0530 Subject: [PATCH 07/24] code change 6 --- util/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/helper.go b/util/helper.go index 0b61ce060b..362f098bd7 100644 --- a/util/helper.go +++ b/util/helper.go @@ -214,6 +214,6 @@ func InterfaceToMapAdapter(resp interface{}) map[string]interface{} { func TriggerCDMetrics(wfr *pipelineConfig.CdWorkflowRunner, cdConfig *pipeline.CdConfig) { if cdConfig.ExposeCDMetrics && (wfr.Status == pipelineConfig.WorkflowFailed || wfr.Status == pipelineConfig.WorkflowSucceeded) { - middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Namespace).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) + middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Name).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) } } From bdb2e5c4cd26dad01a74760fb11fe85aaf4ce520 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 24 Feb 2023 17:38:04 +0530 Subject: [PATCH 08/24] change-cd-telemetry-trigger-function --- .../cron/CdApplicationStatusUpdateHandler.go | 4 +- pkg/app/AppService.go | 25 ++++---- pkg/app/integrationTest/AppService_test.go | 61 +++---------------- pkg/pipeline/CdHandler.go | 16 ++++- pkg/pipeline/WorkflowDagExecutor.go | 26 ++++++-- util/helper.go | 15 +++-- 6 files changed, 71 insertions(+), 76 deletions(-) diff --git a/client/cron/CdApplicationStatusUpdateHandler.go b/client/cron/CdApplicationStatusUpdateHandler.go index fceb398729..10b7ee3a68 100644 --- a/client/cron/CdApplicationStatusUpdateHandler.go +++ b/client/cron/CdApplicationStatusUpdateHandler.go @@ -33,7 +33,7 @@ type CdApplicationStatusUpdateHandlerImpl struct { workflowDagExecutor pipeline.WorkflowDagExecutor installedAppService service.InstalledAppService CdHandler pipeline.CdHandler - AppStatusConfig *app.AppStatusConfig + AppStatusConfig *app.AppServiceConfig pubsubClient *pubsub.PubSubClientServiceImpl pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository eventClient client2.EventClient @@ -44,7 +44,7 @@ type CdApplicationStatusUpdateHandlerImpl struct { func NewCdApplicationStatusUpdateHandlerImpl(logger *zap.SugaredLogger, appService app.AppService, workflowDagExecutor pipeline.WorkflowDagExecutor, installedAppService service.InstalledAppService, - CdHandler pipeline.CdHandler, AppStatusConfig *app.AppStatusConfig, pubsubClient *pubsub.PubSubClientServiceImpl, + CdHandler pipeline.CdHandler, AppStatusConfig *app.AppServiceConfig, pubsubClient *pubsub.PubSubClientServiceImpl, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, eventClient client2.EventClient, appListingRepository repository.AppListingRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index c72a43e138..ff32615d97 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -30,7 +30,6 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/dockerRegistry" - "github.com/devtron-labs/devtron/pkg/pipeline" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/k8s" @@ -83,16 +82,17 @@ import ( "google.golang.org/grpc/status" ) -type AppStatusConfig struct { +type AppServiceConfig struct { CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *"` CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *"` CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20"` //in minutes PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10"` //in minutes HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120"` //in seconds + ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` } -func GetAppStatusConfig() (*AppStatusConfig, error) { - cfg := &AppStatusConfig{} +func GetAppStatusConfig() (*AppServiceConfig, error) { + cfg := &AppServiceConfig{} err := env.Parse(cfg) if err != nil { fmt.Println("failed to parse server app status config: " + err.Error()) @@ -151,11 +151,10 @@ type AppServiceImpl struct { pipelineStatusTimelineResourcesService PipelineStatusTimelineResourcesService pipelineStatusSyncDetailService PipelineStatusSyncDetailService pipelineStatusTimelineService PipelineStatusTimelineService - appStatusConfig *AppStatusConfig + appStatusConfig *AppServiceConfig gitOpsConfigRepository repository.GitOpsConfigRepository appStatusService appStatus.AppStatusService k8sApplicationService k8s.K8sApplicationService - cdConfig *pipeline.CdConfig } type AppService interface { @@ -217,11 +216,10 @@ func NewAppService( pipelineStatusTimelineResourcesService PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService PipelineStatusSyncDetailService, pipelineStatusTimelineService PipelineStatusTimelineService, - appStatusConfig *AppStatusConfig, + appStatusConfig *AppServiceConfig, gitOpsConfigRepository repository.GitOpsConfigRepository, appStatusService appStatus.AppStatusService, - k8sApplicationService k8s.K8sApplicationService, - cdConfig *pipeline.CdConfig) *AppServiceImpl { + k8sApplicationService k8s.K8sApplicationService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -275,7 +273,6 @@ func NewAppService( gitOpsConfigRepository: gitOpsConfigRepository, appStatusService: appStatusService, k8sApplicationService: k8sApplicationService, - cdConfig: cdConfig, } return appServiceImpl } @@ -1898,7 +1895,13 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl impl.logger.Errorw("error on update cd workflow runner", "wfr", wfr, "app", app, "err", err) return err } - util2.TriggerCDMetrics(wfr, impl.cdConfig) + cdMetrics := util2.CDMetrics{ + AppName: wfr.CdWorkflow.Pipeline.DeploymentAppName, + Status: wfr.Status, + DeploymentType: wfr.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: wfr.CdWorkflow.Pipeline.Environment.Name, + } + util2.TriggerCDMetrics(cdMetrics, impl.appStatusConfig.ExposeCDMetrics) return nil } diff --git a/pkg/app/integrationTest/AppService_test.go b/pkg/app/integrationTest/AppService_test.go index 703ded3e6b..40d6d56766 100644 --- a/pkg/app/integrationTest/AppService_test.go +++ b/pkg/app/integrationTest/AppService_test.go @@ -20,7 +20,6 @@ import ( repository1 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/module" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" - "github.com/devtron-labs/devtron/pkg/pipeline" serverEnvConfig "github.com/devtron-labs/devtron/pkg/server/config" "github.com/devtron-labs/devtron/pkg/sql" repository2 "github.com/devtron-labs/devtron/pkg/user/repository" @@ -114,8 +113,6 @@ func InitAppService() *app2.AppServiceImpl { log.Fatal("error in getting db connection, AppService_test", "err", err) } - cdConfig, _ := pipeline.GetCdConfig() - pipelineOverrideRepository := chartConfig.NewPipelineOverrideRepository(dbConnection) pipelineRepository := pipelineConfig.NewPipelineRepositoryImpl(dbConnection, logger) httpClient := util.NewHttpClient() @@ -157,55 +154,13 @@ func InitAppService() *app2.AppServiceImpl { pipelineStatusSyncDetailService := app2.NewPipelineStatusSyncDetailServiceImpl(logger, pipelineStatusSyncDetailRepository) pipelineStatusTimelineService := app2.NewPipelineStatusTimelineServiceImpl(logger, pipelineStatusTimelineRepository, cdWorkflowRepository, nil, pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService) refChartDir := chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts") - appService := app2.NewAppService( - nil, - pipelineOverrideRepository, - nil, - logger, - nil, - pipelineRepository, - nil, - eventClient, - eventFactory, - nil, - nil, - nil, - nil, - nil, - nil, - appListingRepository, - appRepository, - nil, - nil, - nil, - nil, - nil, - chartRepository, - nil, - cdWorkflowRepository, - nil, - nil, - nil, - nil, - nil, - nil, - nil, - nil, - nil, - refChartDir, - nil, - nil, - nil, - nil, - pipelineStatusTimelineRepository, - nil, - nil, - nil, - nil, - nil, - pipelineStatusTimelineResourcesService, - pipelineStatusSyncDetailService, - pipelineStatusTimelineService, - nil, nil, nil, nil, cdConfig) + appService := app2.NewAppService(nil, pipelineOverrideRepository, nil, logger, nil, + pipelineRepository, nil, eventClient, eventFactory, nil, nil, nil, nil, nil, nil, + appListingRepository, appRepository, nil, nil, nil, nil, nil, + chartRepository, nil, cdWorkflowRepository, nil, nil, nil, nil, + nil, nil, nil, nil, nil, refChartDir, nil, + nil, nil, nil, pipelineStatusTimelineRepository, nil, nil, nil, + nil, nil, pipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService, pipelineStatusTimelineService, + nil, nil, nil, nil) return appService } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index aa1d608716..6752829466 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -334,7 +334,13 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel impl.Logger.Errorw("error on update cd workflow runner", "wfr", wfr, "err", err) return err } - util3.TriggerCDMetrics(wfr, impl.cdConfig) + cdMetrics := util3.CDMetrics{ + AppName: wfr.CdWorkflow.Pipeline.DeploymentAppName, + Status: wfr.Status, + DeploymentType: wfr.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: wfr.CdWorkflow.Pipeline.Environment.Name, + } + util3.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) if helmAppStatus == application.Healthy { pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByCdWorkflowId(wfr.CdWorkflowId) @@ -458,7 +464,13 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) return 0, "", err } - util3.TriggerCDMetrics(savedWorkflow, impl.cdConfig) + cdMetrics := util3.CDMetrics{ + AppName: savedWorkflow.CdWorkflow.Pipeline.DeploymentAppName, + Status: savedWorkflow.Status, + DeploymentType: savedWorkflow.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: savedWorkflow.CdWorkflow.Pipeline.Environment.Name, + } + util3.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 16ee474785..6e50807cb2 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -23,8 +23,8 @@ import ( "fmt" "github.com/argoproj/gitops-engine/pkg/health" blob_storage "github.com/devtron-labs/common-lib/blob-storage" - util4 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/client/gitSensor" + util4 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "go.opentelemetry.io/otel" "strconv" @@ -1092,7 +1092,13 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo impl.logger.Errorw("error in updating status", "err", err) return err } - util4.TriggerCDMetrics(runner, impl.cdConfig) + cdMetrics := util4.CDMetrics{ + AppName: runner.CdWorkflow.Pipeline.DeploymentAppName, + Status: runner.Status, + DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, + } + util4.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, @@ -1160,7 +1166,13 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne impl.logger.Errorw("error updating cd wf runner status", "err", err, "currentRunner", currentRunner) return err } - util4.TriggerCDMetrics(currentRunner, impl.cdConfig) + cdMetrics := util4.CDMetrics{ + AppName: currentRunner.CdWorkflow.Pipeline.DeploymentAppName, + Status: currentRunner.Status, + DeploymentType: currentRunner.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: currentRunner.CdWorkflow.Pipeline.Environment.Name, + } + util4.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) return nil //update current WF with error status } else { @@ -1438,7 +1450,13 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value impl.logger.Errorw("err", "err", err) return 0, err } - util4.TriggerCDMetrics(runner, impl.cdConfig) + cdMetrics := util4.CDMetrics{ + AppName: runner.CdWorkflow.Pipeline.DeploymentAppName, + Status: runner.Status, + DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, + EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, + } + util4.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) // creating cd pipeline status timeline for deployment failed timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: runner.Id, diff --git a/util/helper.go b/util/helper.go index 362f098bd7..d293c8148a 100644 --- a/util/helper.go +++ b/util/helper.go @@ -24,7 +24,6 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/juju/errors" "io" "io/ioutil" @@ -39,6 +38,14 @@ import ( "go.uber.org/zap" ) +type CDMetrics struct { + AppName string + DeploymentType string + Status string + EnvironmentName string + Time float64 +} + func ContainsString(list []string, element string) bool { if len(list) == 0 { return false @@ -212,8 +219,8 @@ func InterfaceToMapAdapter(resp interface{}) map[string]interface{} { return dat } -func TriggerCDMetrics(wfr *pipelineConfig.CdWorkflowRunner, cdConfig *pipeline.CdConfig) { - if cdConfig.ExposeCDMetrics && (wfr.Status == pipelineConfig.WorkflowFailed || wfr.Status == pipelineConfig.WorkflowSucceeded) { - middleware.CdDuration.WithLabelValues(wfr.CdWorkflow.Pipeline.DeploymentAppName, wfr.Status, wfr.CdWorkflow.Pipeline.Environment.Name).Observe(time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds()) +func TriggerCDMetrics(wfr CDMetrics, exposeCDMetrics bool) { + if exposeCDMetrics && (wfr.Status == pipelineConfig.WorkflowFailed || wfr.Status == pipelineConfig.WorkflowSucceeded) { + middleware.CdDuration.WithLabelValues(wfr.AppName, wfr.Status, wfr.EnvironmentName, wfr.DeploymentType).Observe(wfr.Time) } } From 61ca29f58c4b5c2d50ff2b0ca1b38c26cf8113b6 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 24 Feb 2023 17:40:12 +0530 Subject: [PATCH 09/24] add-deployment-type --- internal/middleware/instrument.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index cb6d062cb4..6d293a8d68 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -36,7 +36,7 @@ var ( var CdDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cd_duration_seconds", Help: "Duration of CD process", -}, []string{"appName", "status", "envName"}) +}, []string{"appName", "status", "envName", "deploymentType"}) var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ From 2acf68f11aebdccc7d92a28633264304260dfb0c Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 24 Feb 2023 18:01:54 +0530 Subject: [PATCH 10/24] server-mode-change --- pkg/app/AppService.go | 1 + pkg/pipeline/CdHandler.go | 2 ++ pkg/pipeline/WorkflowDagExecutor.go | 3 +++ 3 files changed, 6 insertions(+) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index ff32615d97..fc45255ba1 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1900,6 +1900,7 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl Status: wfr.Status, DeploymentType: wfr.CdWorkflow.Pipeline.DeploymentAppType, EnvironmentName: wfr.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds(), } util2.TriggerCDMetrics(cdMetrics, impl.appStatusConfig.ExposeCDMetrics) return nil diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 6752829466..fa0e0b221d 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -339,6 +339,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel Status: wfr.Status, DeploymentType: wfr.CdWorkflow.Pipeline.DeploymentAppType, EnvironmentName: wfr.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(wfr.StartedOn).Seconds() - time.Since(wfr.FinishedOn).Seconds(), } util3.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) impl.Logger.Infow("updated workflow runner status for helm app", "wfr", wfr) @@ -469,6 +470,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus Status: savedWorkflow.Status, DeploymentType: savedWorkflow.CdWorkflow.Pipeline.DeploymentAppType, EnvironmentName: savedWorkflow.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(savedWorkflow.StartedOn).Seconds() - time.Since(savedWorkflow.FinishedOn).Seconds(), } util3.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 6e50807cb2..c87db8df21 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -1097,6 +1097,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo Status: runner.Status, DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(runner.StartedOn).Seconds() - time.Since(runner.FinishedOn).Seconds(), } util4.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) // creating cd pipeline status timeline for deployment failed @@ -1171,6 +1172,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne Status: currentRunner.Status, DeploymentType: currentRunner.CdWorkflow.Pipeline.DeploymentAppType, EnvironmentName: currentRunner.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(currentRunner.StartedOn).Seconds() - time.Since(currentRunner.FinishedOn).Seconds(), } util4.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) return nil @@ -1455,6 +1457,7 @@ func (impl *WorkflowDagExecutorImpl) ManualCdTrigger(overrideRequest *bean.Value Status: runner.Status, DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, + Time: time.Since(runner.StartedOn).Seconds() - time.Since(runner.FinishedOn).Seconds(), } util4.TriggerCDMetrics(cdMetrics, impl.cdConfig.ExposeCDMetrics) // creating cd pipeline status timeline for deployment failed From dd1953c235f75d271fcdc67029cab5bb064e8fc7 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 24 Feb 2023 18:28:08 +0530 Subject: [PATCH 11/24] remove-loop-import --- util/helper.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/helper.go b/util/helper.go index d293c8148a..41ff44f8bb 100644 --- a/util/helper.go +++ b/util/helper.go @@ -23,7 +23,6 @@ import ( "encoding/json" "fmt" "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/juju/errors" "io" "io/ioutil" @@ -220,7 +219,7 @@ func InterfaceToMapAdapter(resp interface{}) map[string]interface{} { } func TriggerCDMetrics(wfr CDMetrics, exposeCDMetrics bool) { - if exposeCDMetrics && (wfr.Status == pipelineConfig.WorkflowFailed || wfr.Status == pipelineConfig.WorkflowSucceeded) { + if exposeCDMetrics && (wfr.Status == WorkflowFailed || wfr.Status == WorkflowSucceeded) { middleware.CdDuration.WithLabelValues(wfr.AppName, wfr.Status, wfr.EnvironmentName, wfr.DeploymentType).Observe(wfr.Time) } } From 4c07c2913d8e4ed9f3f432482bcd4980a76139ef Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Tue, 28 Feb 2023 14:12:17 +0530 Subject: [PATCH 12/24] add-wire-gen --- wire_gen.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wire_gen.go b/wire_gen.go index b1836be8de..e3b920dba2 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -327,7 +327,7 @@ func InitializeApp() (*App, error) { pipelineStatusSyncDetailRepositoryImpl := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(db, sugaredLogger) pipelineStatusSyncDetailServiceImpl := app2.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) pipelineStatusTimelineServiceImpl := app2.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl) - appStatusConfig, err := app2.GetAppStatusConfig() + appServiceConfig, err := app2.GetAppStatusConfig() if err != nil { return nil, err } @@ -349,7 +349,7 @@ func InitializeApp() (*App, error) { k8sResourceHistoryRepositoryImpl := repository8.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) k8sApplicationServiceImpl := k8s.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, k8sClientServiceImpl, helmAppServiceImpl, k8sUtil, acdAuthConfig, k8sResourceHistoryServiceImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appStatusConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -453,7 +453,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - cdApplicationStatusUpdateHandlerImpl := cron.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, cdHandlerImpl, appStatusConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl) + cdApplicationStatusUpdateHandlerImpl := cron.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, cdHandlerImpl, appServiceConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl) appListingRestHandlerImpl := restHandler.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, clusterServiceImplExtended, helmAppServiceImpl, argoUserServiceImpl, k8sApplicationServiceImpl, installedAppServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl) appListingRouterImpl := router.NewAppListingRouterImpl(appListingRestHandlerImpl) chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sUtil, clusterServiceImplExtended, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) From 1ca3974d1225b9488aaeafeee640ffb8329cb28e Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Tue, 28 Feb 2023 15:14:27 +0530 Subject: [PATCH 13/24] Update AppService function name --- Wire.go | 2 +- pkg/app/AppService.go | 2 +- wire_gen.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Wire.go b/Wire.go index 3932ba141d..300a71b6d3 100644 --- a/Wire.go +++ b/Wire.go @@ -173,7 +173,7 @@ func InitializeApp() (*App, error) { restHandler.NewPipelineRestHandler, wire.Bind(new(restHandler.PipelineTriggerRestHandler), new(*restHandler.PipelineTriggerRestHandlerImpl)), - app.GetAppStatusConfig, + app.GetAppServiceConfig, app.NewAppService, wire.Bind(new(app.AppService), new(*app.AppServiceImpl)), diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index f520790a77..99e841d997 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -93,7 +93,7 @@ type AppServiceConfig struct { ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` } -func GetAppStatusConfig() (*AppServiceConfig, error) { +func GetAppServiceConfig() (*AppServiceConfig, error) { cfg := &AppServiceConfig{} err := env.Parse(cfg) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index e3b920dba2..b3667df757 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -327,7 +327,7 @@ func InitializeApp() (*App, error) { pipelineStatusSyncDetailRepositoryImpl := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(db, sugaredLogger) pipelineStatusSyncDetailServiceImpl := app2.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) pipelineStatusTimelineServiceImpl := app2.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl) - appServiceConfig, err := app2.GetAppStatusConfig() + appServiceConfig, err := app2.GetAppServiceConfig() if err != nil { return nil, err } From 41f7e131f5d2a2b0a0876db64b039ad24defebb8 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Tue, 28 Feb 2023 15:19:14 +0530 Subject: [PATCH 14/24] remove finished on --- pkg/app/AppService.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 99e841d997..48e3822726 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1886,7 +1886,6 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl impl.logger.Errorw("error on update cd workflow runner, fetch failed for runner type", "wfr", wfr, "app", app, "err", err) return err } - wfr.FinishedOn = time.Now() if updateTimedOutStatus { wfr.Status = pipelineConfig.WorkflowTimedOut } else { From dd93feb49bbd8f978d1eb9ad5f82731a34abfbb9 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Wed, 1 Mar 2023 18:25:23 +0530 Subject: [PATCH 15/24] ci-telemetry --- api/router/pubsub/CiEventHandler.go | 23 ++++++++++++++++++ internal/middleware/instrument.go | 36 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index 3fd2f0db83..c956c48372 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -39,6 +40,15 @@ type CiEventHandlerImpl struct { webhookService pipeline.WebhookService } +type Metrics struct { + CacheDown float64 `json:"cache_down"` + PreCi float64 `json:"pre_ci"` + Build float64 `json:"build"` + PostCi float64 `json:"post_ci"` + CacheUp float64 `json:"cache_up"` + Total float64 `json:"total"` +} + type CiCompleteEvent struct { CiProjectDetails []pipeline.CiProjectDetails `json:"ciProjectDetails"` DockerImage string `json:"dockerImage" validate:"required,image-validator"` @@ -49,6 +59,7 @@ type CiCompleteEvent struct { PipelineName string `json:"pipelineName"` DataSource string `json:"dataSource"` MaterialType string `json:"materialType"` + Metrics Metrics `json:"metrics"` } func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService) *CiEventHandlerImpl { @@ -71,6 +82,18 @@ func (impl *CiEventHandlerImpl) Subscribe() error { //defer msg.Ack() ciCompleteEvent := CiCompleteEvent{} err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent) + + middleware.CacheDownloadDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.CacheDown) + middleware.CiDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.Total) + middleware.CacheUploadDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.CacheUp) + if ciCompleteEvent.Metrics.PostCi != 0 { + middleware.PostCiDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.PostCi) + } + if ciCompleteEvent.Metrics.PreCi != 0 { + middleware.PreCiDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.PreCi) + } + middleware.BuildDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.CacheDown) + if err != nil { impl.logger.Error("error while unmarshalling json data", "error", err) return diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 6d293a8d68..21b3d318b6 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -38,6 +38,42 @@ var CdDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Help: "Duration of CD process", }, []string{"appName", "status", "envName", "deploymentType"}) +var CiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "ci_duration_seconds", + Help: "Duration of CI process", + Buckets: prometheus.LinearBuckets(20, 20, 5), +}, []string{"pipelineName"}) + +var CacheDownloadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "cache_download_duration_seconds", + Help: "Duration of Cache Download process", + Buckets: prometheus.LinearBuckets(20, 20, 5), +}, []string{"pipelineName"}) + +var PreCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pre_ci_duration_seconds", + Help: "Duration of Pre CI process", + Buckets: prometheus.LinearBuckets(20, 20, 5), +}, []string{"pipelineName"}) + +var BuildDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "build_duration_seconds", + Help: "Duration of Build process", + Buckets: prometheus.LinearBuckets(20, 20, 5), +}, []string{"pipelineName"}) + +var PostCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "post_ci_duration_seconds", + Help: "Duration of Post CI process", + Buckets: prometheus.LinearBuckets(20, 20, 5), +}, []string{"pipelineName"}) + +var CacheUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "cache_upload_duration_seconds", + Help: "Duration of Cache Upload process", + Buckets: prometheus.LinearBuckets(20, 20, 5), +}, []string{"pipelineName"}) + var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "orchestrator_http_requests_total", From aa42596102995e7e233961bd5d27acc1b62061e9 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 3 Mar 2023 15:50:42 +0530 Subject: [PATCH 16/24] add-flag-and-appName --- Wire.go | 1 + api/router/pubsub/CiEventHandler.go | 42 ++++++++----------- .../pubsub/WorkflowStatusUpdateHandler.go | 1 - internal/middleware/instrument.go | 2 +- util/helper.go | 24 +++++++++++ 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/Wire.go b/Wire.go index 300a71b6d3..39e138e897 100644 --- a/Wire.go +++ b/Wire.go @@ -410,6 +410,7 @@ func InitializeApp() (*App, error) { pubsub.NewApplicationStatusHandlerImpl, wire.Bind(new(pubsub.ApplicationStatusHandler), new(*pubsub.ApplicationStatusHandlerImpl)), + pubsub.GetCiEventConfig(), pubsub.NewCiEventHandlerImpl, wire.Bind(new(pubsub.CiEventHandler), new(*pubsub.CiEventHandlerImpl)), diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index c956c48372..aec60561fa 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -20,14 +20,25 @@ package pubsub import ( "encoding/json" "fmt" + "github.com/caarlos0/env/v6" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/util" "go.uber.org/zap" ) +type CiEventConfig struct { + ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"` +} + +func GetCiEventConfig() (*CiEventConfig, error) { + cfg := &CiEventConfig{} + err := env.Parse(cfg) + return cfg, err +} + type CiEventHandler interface { Subscribe() error BuildCiArtifactRequest(event CiCompleteEvent) (*pipeline.CiArtifactWebhookRequest, error) @@ -38,15 +49,7 @@ type CiEventHandlerImpl struct { logger *zap.SugaredLogger pubsubClient *pubsub.PubSubClientServiceImpl webhookService pipeline.WebhookService -} - -type Metrics struct { - CacheDown float64 `json:"cache_down"` - PreCi float64 `json:"pre_ci"` - Build float64 `json:"build"` - PostCi float64 `json:"post_ci"` - CacheUp float64 `json:"cache_up"` - Total float64 `json:"total"` + ciEventConfig *CiEventConfig } type CiCompleteEvent struct { @@ -59,14 +62,16 @@ type CiCompleteEvent struct { PipelineName string `json:"pipelineName"` DataSource string `json:"dataSource"` MaterialType string `json:"materialType"` - Metrics Metrics `json:"metrics"` + Metrics util.CIMetrics `json:"metrics"` + AppName string `json:"appName"` } -func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService) *CiEventHandlerImpl { +func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl { ciEventHandlerImpl := &CiEventHandlerImpl{ logger: logger, pubsubClient: pubsubClient, webhookService: webhookService, + ciEventConfig: ciEventConfig, } err := ciEventHandlerImpl.Subscribe() if err != nil { @@ -82,18 +87,7 @@ func (impl *CiEventHandlerImpl) Subscribe() error { //defer msg.Ack() ciCompleteEvent := CiCompleteEvent{} err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent) - - middleware.CacheDownloadDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.CacheDown) - middleware.CiDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.Total) - middleware.CacheUploadDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.CacheUp) - if ciCompleteEvent.Metrics.PostCi != 0 { - middleware.PostCiDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.PostCi) - } - if ciCompleteEvent.Metrics.PreCi != 0 { - middleware.PreCiDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.PreCi) - } - middleware.BuildDuration.WithLabelValues(ciCompleteEvent.PipelineName).Observe(ciCompleteEvent.Metrics.CacheDown) - + util.TriggerCIMetrics(ciCompleteEvent.Metrics, impl.ciEventConfig.ExposeCiMetrics, ciCompleteEvent.PipelineName, ciCompleteEvent.AppName) if err != nil { impl.logger.Error("error while unmarshalling json data", "error", err) return diff --git a/api/router/pubsub/WorkflowStatusUpdateHandler.go b/api/router/pubsub/WorkflowStatusUpdateHandler.go index 76e5fc1c92..532c72e545 100644 --- a/api/router/pubsub/WorkflowStatusUpdateHandler.go +++ b/api/router/pubsub/WorkflowStatusUpdateHandler.go @@ -24,7 +24,6 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/client/events" - //"github.com/devtron-labs/devtron/client/pubsub" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline" util "github.com/devtron-labs/devtron/util/event" diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 21b3d318b6..02a89be082 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -60,7 +60,7 @@ var BuildDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "build_duration_seconds", Help: "Duration of Build process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName"}) +}, []string{"pipelineName", "AppName"}) var PostCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "post_ci_duration_seconds", diff --git a/util/helper.go b/util/helper.go index 41ff44f8bb..eb521ff6ac 100644 --- a/util/helper.go +++ b/util/helper.go @@ -45,6 +45,15 @@ type CDMetrics struct { Time float64 } +type CIMetrics struct { + CacheDown float64 `json:"cache_down"` + PreCi float64 `json:"pre_ci"` + Build float64 `json:"build"` + PostCi float64 `json:"post_ci"` + CacheUp float64 `json:"cache_up"` + Total float64 `json:"total"` +} + func ContainsString(list []string, element string) bool { if len(list) == 0 { return false @@ -223,3 +232,18 @@ func TriggerCDMetrics(wfr CDMetrics, exposeCDMetrics bool) { middleware.CdDuration.WithLabelValues(wfr.AppName, wfr.Status, wfr.EnvironmentName, wfr.DeploymentType).Observe(wfr.Time) } } + +func TriggerCIMetrics(Metrics CIMetrics, exposeCIMetrics bool, PipelineName string, AppName string) { + if exposeCIMetrics { + middleware.CacheDownloadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDown) + middleware.CiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.Total) + middleware.CacheUploadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheUp) + if Metrics.PostCi != 0 { + middleware.PostCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PostCi) + } + if Metrics.PreCi != 0 { + middleware.PreCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PreCi) + } + middleware.BuildDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDown) + } +} From 6b7d00ba248db1cd9ec88911daeb46d97b557363 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 3 Mar 2023 15:58:33 +0530 Subject: [PATCH 17/24] add-flag-and-appName --- Wire.go | 2 +- wire_gen.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Wire.go b/Wire.go index 39e138e897..96152d2d8d 100644 --- a/Wire.go +++ b/Wire.go @@ -410,7 +410,7 @@ func InitializeApp() (*App, error) { pubsub.NewApplicationStatusHandlerImpl, wire.Bind(new(pubsub.ApplicationStatusHandler), new(*pubsub.ApplicationStatusHandlerImpl)), - pubsub.GetCiEventConfig(), + pubsub.GetCiEventConfig, pubsub.NewCiEventHandlerImpl, wire.Bind(new(pubsub.CiEventHandler), new(*pubsub.CiEventHandlerImpl)), diff --git a/wire_gen.go b/wire_gen.go index b3667df757..8a4c25077e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -466,7 +466,11 @@ func InitializeApp() (*App, error) { gitWebhookServiceImpl := git.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, appServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, workflowDagExecutorImpl, ciHandlerImpl) - ciEventHandlerImpl := pubsub.NewCiEventHandlerImpl(sugaredLogger, pubSubClientServiceImpl, webhookServiceImpl) + ciEventConfig, err := pubsub.GetCiEventConfig() + if err != nil { + return nil, err + } + ciEventHandlerImpl := pubsub.NewCiEventHandlerImpl(sugaredLogger, pubSubClientServiceImpl, webhookServiceImpl, ciEventConfig) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, webhookServiceImpl, ciEventHandlerImpl, validate, userServiceImpl, enforcerImpl, enforcerUtilImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, cdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) From ce69c91aea2e3382e7501dd54045c5c4d0887239 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Fri, 3 Mar 2023 18:22:42 +0530 Subject: [PATCH 18/24] add-app-name --- internal/middleware/instrument.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 02a89be082..f2db01bc87 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -42,19 +42,19 @@ var CiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "ci_duration_seconds", Help: "Duration of CI process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName"}) +}, []string{"pipelineName", "AppName"}) var CacheDownloadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cache_download_duration_seconds", Help: "Duration of Cache Download process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName"}) +}, []string{"pipelineName", "AppName"}) var PreCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pre_ci_duration_seconds", Help: "Duration of Pre CI process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName"}) +}, []string{"pipelineName", "AppName"}) var BuildDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "build_duration_seconds", @@ -66,13 +66,13 @@ var PostCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "post_ci_duration_seconds", Help: "Duration of Post CI process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName"}) +}, []string{"pipelineName", "AppName"}) var CacheUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cache_upload_duration_seconds", Help: "Duration of Cache Upload process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName"}) +}, []string{"pipelineName", "AppName"}) var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ From b22f41ba6650b978b60cf00ae6a76821ce70a1ba Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Mon, 6 Mar 2023 10:40:53 +0530 Subject: [PATCH 19/24] improve-ci-metrics-struct --- util/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/version.go b/util/version.go index f64cb32956..417b3fb010 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: ServerMode} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE_FULL} } func IsBaseStack() bool { From 80ab760a730ec0cd9fbcfe5d14244e609facd012 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Mon, 6 Mar 2023 10:43:16 +0530 Subject: [PATCH 20/24] improve-ci-metrics-struct --- util/helper.go | 34 ++++++++++++++++++++-------------- util/version.go | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/util/helper.go b/util/helper.go index eb521ff6ac..6707fd1863 100644 --- a/util/helper.go +++ b/util/helper.go @@ -46,12 +46,18 @@ type CDMetrics struct { } type CIMetrics struct { - CacheDown float64 `json:"cache_down"` - PreCi float64 `json:"pre_ci"` - Build float64 `json:"build"` - PostCi float64 `json:"post_ci"` - CacheUp float64 `json:"cache_up"` - Total float64 `json:"total"` + CacheDownDuration float64 `json:"cache_down_duration"` + PreCiDuration float64 `json:"pre_ci_duration"` + BuildDuration float64 `json:"build_duration"` + PostCiDuration float64 `json:"post_ci_duration"` + CacheUpDuration float64 `json:"cache_up_duration"` + TotalDuration float64 `json:"total_duration"` + CacheDownStart time.Time `json:"cache_down_start"` + PreCiStart time.Time `json:"pre_ci_start"` + BuildStart time.Time `json:"build_start"` + PostCiStart time.Time `json:"post_ci_start"` + CacheUpStart time.Time `json:"cache_up_start"` + TotalStart time.Time `json:"total_start"` } func ContainsString(list []string, element string) bool { @@ -235,15 +241,15 @@ func TriggerCDMetrics(wfr CDMetrics, exposeCDMetrics bool) { func TriggerCIMetrics(Metrics CIMetrics, exposeCIMetrics bool, PipelineName string, AppName string) { if exposeCIMetrics { - middleware.CacheDownloadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDown) - middleware.CiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.Total) - middleware.CacheUploadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheUp) - if Metrics.PostCi != 0 { - middleware.PostCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PostCi) + middleware.CacheDownloadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDownDuration) + middleware.CiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.TotalDuration) + middleware.CacheUploadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheUpDuration) + if Metrics.PostCiDuration != 0 { + middleware.PostCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PostCiDuration) } - if Metrics.PreCi != 0 { - middleware.PreCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PreCi) + if Metrics.PreCiDuration != 0 { + middleware.PreCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PreCiDuration) } - middleware.BuildDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDown) + middleware.BuildDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.BuildDuration) } } diff --git a/util/version.go b/util/version.go index 417b3fb010..f64cb32956 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE_FULL} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: ServerMode} } func IsBaseStack() bool { From 7fb89e717eea2cab29c96d63b2b17f0efcf8f70f Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Mon, 6 Mar 2023 13:24:34 +0530 Subject: [PATCH 21/24] improve-ci-metrics-struct --- util/helper.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/util/helper.go b/util/helper.go index 6707fd1863..55ed5cddc5 100644 --- a/util/helper.go +++ b/util/helper.go @@ -46,18 +46,18 @@ type CDMetrics struct { } type CIMetrics struct { - CacheDownDuration float64 `json:"cache_down_duration"` - PreCiDuration float64 `json:"pre_ci_duration"` - BuildDuration float64 `json:"build_duration"` - PostCiDuration float64 `json:"post_ci_duration"` - CacheUpDuration float64 `json:"cache_up_duration"` - TotalDuration float64 `json:"total_duration"` - CacheDownStart time.Time `json:"cache_down_start"` - PreCiStart time.Time `json:"pre_ci_start"` - BuildStart time.Time `json:"build_start"` - PostCiStart time.Time `json:"post_ci_start"` - CacheUpStart time.Time `json:"cache_up_start"` - TotalStart time.Time `json:"total_start"` + CacheDownDuration float64 `json:"CacheDownDuration"` + PreCiDuration float64 `json:"PreCiDuration"` + BuildDuration float64 `json:"BuildDuration"` + PostCiDuration float64 `json:"PostCiDuration"` + CacheUpDuration float64 `json:"CacheUpDuration"` + TotalDuration float64 `json:"TotalDuration"` + CacheDownStartTime time.Time `json:"CacheDownStartTime"` + PreCiStartTime time.Time `json:"pre_ci_start"` + BuildStartTime time.Time `json:"BuildStartTime"` + PostCiStartTime time.Time `json:"PostCiStartTime"` + CacheUpStartTime time.Time `json:"CacheUpStartTime"` + TotalStartTime time.Time `json:"TotalStartTime"` } func ContainsString(list []string, element string) bool { From fe0976e1176cd6a15528211e7180177e872ff20e Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 9 Mar 2023 17:25:49 +0530 Subject: [PATCH 22/24] add-ci-pipeline --- internal/middleware/instrument.go | 2 +- pkg/pipeline/CiService.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index f2db01bc87..1c98c7ffe9 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -96,7 +96,7 @@ var CdTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{ var CiTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ci_trigger_counter", -}, []string{"appName"}) +}, []string{"appName", "PipelineName"}) // prometheusMiddleware implements mux.MiddlewareFunc. func PrometheusMiddleware(next http.Handler) http.Handler { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 81d09356b4..97c349fea5 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -157,7 +157,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { } impl.Logger.Debugw("ci triggered", "wf name ", createdWf.Name, " pipeline ", trigger.PipelineId) - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName).Inc() + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest) return savedCiWf.Id, err } From 69e67a584efe99e725c81a65522c128e5105bbfa Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 9 Mar 2023 20:21:54 +0530 Subject: [PATCH 23/24] Add check to Cache Upload --- util/helper.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/helper.go b/util/helper.go index 55ed5cddc5..2aadca06d1 100644 --- a/util/helper.go +++ b/util/helper.go @@ -243,7 +243,9 @@ func TriggerCIMetrics(Metrics CIMetrics, exposeCIMetrics bool, PipelineName stri if exposeCIMetrics { middleware.CacheDownloadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheDownDuration) middleware.CiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.TotalDuration) - middleware.CacheUploadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheUpDuration) + if Metrics.CacheUpDuration != 0 { + middleware.CacheUploadDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.CacheUpDuration) + } if Metrics.PostCiDuration != 0 { middleware.PostCiDuration.WithLabelValues(PipelineName, AppName).Observe(Metrics.PostCiDuration) } From 69df5d01e6b7c91447d3288e145b56a52d39a240 Mon Sep 17 00:00:00 2001 From: Ashish-devtron Date: Thu, 9 Mar 2023 20:53:25 +0530 Subject: [PATCH 24/24] Code-refactor --- internal/middleware/instrument.go | 14 +++++++------- util/helper.go | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 1c98c7ffe9..faa41b89b9 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -42,37 +42,37 @@ var CiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "ci_duration_seconds", Help: "Duration of CI process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName", "AppName"}) +}, []string{"pipelineName", "appName"}) var CacheDownloadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cache_download_duration_seconds", Help: "Duration of Cache Download process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName", "AppName"}) +}, []string{"pipelineName", "appName"}) var PreCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "pre_ci_duration_seconds", Help: "Duration of Pre CI process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName", "AppName"}) +}, []string{"pipelineName", "appName"}) var BuildDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "build_duration_seconds", Help: "Duration of Build process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName", "AppName"}) +}, []string{"pipelineName", "appName"}) var PostCiDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "post_ci_duration_seconds", Help: "Duration of Post CI process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName", "AppName"}) +}, []string{"pipelineName", "appName"}) var CacheUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "cache_upload_duration_seconds", Help: "Duration of Cache Upload process", Buckets: prometheus.LinearBuckets(20, 20, 5), -}, []string{"pipelineName", "AppName"}) +}, []string{"pipelineName", "appName"}) var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ @@ -96,7 +96,7 @@ var CdTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{ var CiTriggerCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ci_trigger_counter", -}, []string{"appName", "PipelineName"}) +}, []string{"appName", "pipelineName"}) // prometheusMiddleware implements mux.MiddlewareFunc. func PrometheusMiddleware(next http.Handler) http.Handler { diff --git a/util/helper.go b/util/helper.go index 2aadca06d1..83bca97882 100644 --- a/util/helper.go +++ b/util/helper.go @@ -46,18 +46,18 @@ type CDMetrics struct { } type CIMetrics struct { - CacheDownDuration float64 `json:"CacheDownDuration"` - PreCiDuration float64 `json:"PreCiDuration"` - BuildDuration float64 `json:"BuildDuration"` - PostCiDuration float64 `json:"PostCiDuration"` - CacheUpDuration float64 `json:"CacheUpDuration"` - TotalDuration float64 `json:"TotalDuration"` - CacheDownStartTime time.Time `json:"CacheDownStartTime"` - PreCiStartTime time.Time `json:"pre_ci_start"` - BuildStartTime time.Time `json:"BuildStartTime"` - PostCiStartTime time.Time `json:"PostCiStartTime"` - CacheUpStartTime time.Time `json:"CacheUpStartTime"` - TotalStartTime time.Time `json:"TotalStartTime"` + CacheDownDuration float64 `json:"cacheDownDuration"` + PreCiDuration float64 `json:"preCiDuration"` + BuildDuration float64 `json:"buildDuration"` + PostCiDuration float64 `json:"postCiDuration"` + CacheUpDuration float64 `json:"cacheUpDuration"` + TotalDuration float64 `json:"totalDuration"` + CacheDownStartTime time.Time `json:"cacheDownStartTime"` + PreCiStartTime time.Time `json:"preCiStart"` + BuildStartTime time.Time `json:"buildStartTime"` + PostCiStartTime time.Time `json:"postCiStartTime"` + CacheUpStartTime time.Time `json:"cacheUpStartTime"` + TotalStartTime time.Time `json:"totalStartTime"` } func ContainsString(list []string, element string) bool {