diff --git a/api/appStore/InstalledAppRestHandler.go b/api/appStore/InstalledAppRestHandler.go index f6a513c365..f293da280d 100644 --- a/api/appStore/InstalledAppRestHandler.go +++ b/api/appStore/InstalledAppRestHandler.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/middleware" util2 "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" @@ -43,6 +44,7 @@ import ( "net/http" "strconv" "strings" + "time" ) type InstalledAppRestHandler interface { @@ -234,10 +236,10 @@ func (handler InstalledAppRestHandlerImpl) GetAllInstalledApp(w http.ResponseWri } } - + start := time.Now() resultObjectMap1 := handler.enforcer.EnforceByEmailInBatch(userEmailId, casbin.ResourceHelmApp, casbin.ActionGet, objectArray1) resultObjectMap2 := handler.enforcer.EnforceByEmailInBatch(userEmailId, casbin.ResourceHelmApp, casbin.ActionGet, objectArray2) - + middleware.AppListingDuration.WithLabelValues("enforceByEmailInBatch", "helm").Observe(time.Since(start).Seconds()) authorizedAppIdSet := make(map[string]bool) //O(n) time loop , at max we will only iterate through all the apps for obj, ok := range resultObjectMap1 { diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index 1275697bf1..04e6f75cc0 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -8,6 +8,7 @@ import ( openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/k8s/application" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" @@ -141,14 +142,18 @@ func (impl *HelmAppServiceImpl) listApplications(ctx context.Context, clusterIds func (impl *HelmAppServiceImpl) ListHelmApplications(ctx context.Context, clusterIds []int, w http.ResponseWriter, token string, helmAuth func(token string, object string) bool) { var helmCdPipelines []*pipelineConfig.Pipeline var installedHelmApps []*repository.InstalledApps + start := time.Now() appStream, err := impl.listApplications(ctx, clusterIds) + middleware.AppListingDuration.WithLabelValues("listApplications", "helm").Observe(time.Since(start).Seconds()) if err != nil { impl.logger.Errorw("error in fetching app list", "clusters", clusterIds, "err", err) } if err == nil && len(clusterIds) > 0 { // get helm apps which are created using cd_pipelines newCtx, span := otel.Tracer("pipelineRepository").Start(ctx, "GetAppAndEnvDetailsForDeploymentAppTypePipeline") + start = time.Now() helmCdPipelines, err = impl.pipelineRepository.GetAppAndEnvDetailsForDeploymentAppTypePipeline(util.PIPELINE_DEPLOYMENT_TYPE_HELM, clusterIds) + middleware.AppListingDuration.WithLabelValues("getAppAndEnvDetailsForDeploymentAppTypePipeline", "helm").Observe(time.Since(start).Seconds()) span.End() if err != nil { impl.logger.Errorw("error in fetching helm app list from DB created using cd_pipelines", "clusters", clusterIds, "err", err) @@ -157,7 +162,9 @@ func (impl *HelmAppServiceImpl) ListHelmApplications(ctx context.Context, cluste // if not hyperion mode, then fetch from installed_apps whose deployment_app_type is helm (as in hyperion mode, these apps should be treated as external-apps) if !util2.IsBaseStack() { newCtx, span = otel.Tracer("pipelineRepository").Start(newCtx, "GetAppAndEnvDetailsForDeploymentAppTypePipeline") + start = time.Now() installedHelmApps, err = impl.installedAppRepository.GetAppAndEnvDetailsForDeploymentAppTypeInstalledApps(util.PIPELINE_DEPLOYMENT_TYPE_HELM, clusterIds) + middleware.AppListingDuration.WithLabelValues("getAppAndEnvDetailsForDeploymentAppTypeInstalledApps", "helm").Observe(time.Since(start).Seconds()) span.End() if err != nil { impl.logger.Errorw("error in fetching helm app list from DB created from app store", "clusters", clusterIds, "err", err) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index d53c4b3d73..5d895af3d6 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" @@ -296,7 +297,9 @@ func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseW } newCtx, span = otel.Tracer("appListingService").Start(newCtx, "FetchAppsByEnvironment") + start := time.Now() envContainers, err := handler.appListingService.FetchAppsByEnvironment(fetchAppListingRequest, w, r, token) + middleware.AppListingDuration.WithLabelValues("fetchAppsByEnvironment", "devtron").Observe(time.Since(start).Seconds()) span.End() if err != nil { handler.logger.Errorw("service err, FetchAppsByEnvironment", "err", err, "payload", fetchAppListingRequest) @@ -333,7 +336,9 @@ func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseW } newCtx, span = otel.Tracer("enforcer").Start(newCtx, "EnforceByEmailInBatchForTeams") + start = time.Now() resultMap := handler.enforcer.EnforceByEmailInBatch(userEmailId, casbin.ResourceTeam, casbin.ActionGet, objectArray) + middleware.AppListingDuration.WithLabelValues("enforceByEmailInBatchResourceTeam", "devtron").Observe(time.Since(start).Seconds()) span.End() for teamId, teamName := range uniqueTeams { object := strings.ToLower(teamName) @@ -362,7 +367,9 @@ func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseW } newCtx, span = otel.Tracer("enforcer").Start(newCtx, "EnforceByEmailInBatchForApps") + start = time.Now() resultMap = handler.enforcer.EnforceByEmailInBatch(userEmailId, casbin.ResourceApplications, casbin.ActionGet, objectArray) + middleware.AppListingDuration.WithLabelValues("enforceByEmailInBatchResourceApplication", "devtron").Observe(time.Since(start).Seconds()) span.End() for _, filteredAppEnvContainer := range filteredAppEnvContainers { if fetchAppListingRequest.DeploymentGroupId > 0 { diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 7e75fd5d99..619a7edcfd 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -84,6 +84,11 @@ var CacheUploadDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Buckets: prometheus.LinearBuckets(20, 20, 5), }, []string{"pipelineName", "appName"}) +var AppListingDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "app_listing_duration_seconds", + Help: "Duration of App Listing process", +}, []string{"MethodName", "AppType"}) + var requestCounter = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "orchestrator_http_requests_total", diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 793eaa543b..3c738fde26 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -24,6 +24,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/internal/middleware" "go.opentelemetry.io/otel" "strconv" "strings" @@ -132,7 +133,9 @@ func (impl AppListingRepositoryImpl) FetchAppsByEnvironment(appListingFilter hel impl.Logger.Debugw("basic app detail query: ", query) var lastDeployedTimeDTO []*bean.AppEnvironmentContainer lastDeployedTimeMap := map[int]*bean.AppEnvironmentContainer{} + start := time.Now() _, err := impl.dbConnection.Query(&lastDeployedTimeDTO, query) + middleware.AppListingDuration.WithLabelValues("buildAppListingQueryLastDeploymentTime", "devtron").Observe(time.Since(start).Seconds()) if err != nil { impl.Logger.Error(err) return appEnvArr, err @@ -152,12 +155,13 @@ func (impl AppListingRepositoryImpl) FetchAppsByEnvironment(appListingFilter hel var appEnvContainer []*bean.AppEnvironmentContainer appsEnvquery := impl.appListingRepositoryQueryBuilder.BuildAppListingQuery(appListingFilter) impl.Logger.Debugw("basic app detail query: ", appsEnvquery) + start = time.Now() _, appsErr := impl.dbConnection.Query(&appEnvContainer, appsEnvquery) + middleware.AppListingDuration.WithLabelValues("buildAppListingQuery", "devtron").Observe(time.Since(start).Seconds()) if appsErr != nil { impl.Logger.Error(appsErr) return appEnvContainer, appsErr } - latestDeploymentStatusMap := map[string]*bean.AppEnvironmentContainer{} for _, item := range appEnvContainer { if item.EnvironmentId > 0 && item.PipelineId > 0 && item.Active == false { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index e8750434b9..b051f0a7e6 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "github.com/argoproj/gitops-engine/pkg/health" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/app" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" @@ -366,12 +367,16 @@ func (impl AppListingServiceImpl) GetReleaseCount(appId, envId int) (int, error) } func (impl AppListingServiceImpl) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) { + start := time.Now() appEnvMapping, err := impl.fetchACDAppStatus(fetchAppListingRequest, envContainers) + middleware.AppListingDuration.WithLabelValues("fetchACDAppStatus", "devtron").Observe(time.Since(start).Seconds()) if err != nil { impl.Logger.Errorw("error in fetching app statuses", "error", err) return []*bean.AppContainer{}, err } + start = time.Now() appContainerResponses, err := impl.appListingViewBuilder.BuildView(fetchAppListingRequest, appEnvMapping) + middleware.AppListingDuration.WithLabelValues("buildView", "devtron").Observe(time.Since(start).Seconds()) return appContainerResponses, err } func GetCIPipelineIDs(jobContainers []*bean.JobListingContainer) []int { diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index 04e94e10fd..f2ecb396f9 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -25,6 +25,7 @@ import ( openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/pkg/appStatus" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" @@ -184,7 +185,9 @@ func (impl InstalledAppServiceImpl) GetAll(filter *appStoreBean.AppStoreFilter) ApplicationType: &applicationType, ClusterIds: &clusterIdsConverted, } + start := time.Now() installedApps, err := impl.installedAppRepository.GetAllInstalledApps(filter) + middleware.AppListingDuration.WithLabelValues("getAllInstalledApps", "helm").Observe(time.Since(start).Seconds()) if err != nil && !util.IsErrNoRows(err) { impl.logger.Error(err) return installedAppsResponse, err