fix: Optimize app grouping apis (#3125)

* workflow status api optimized

* optimzation on db queries for app grouping api's

* app workflow listing for app grouping optimized

* ci pipelines fetch for build n deploy for app grouping

* fixing db query

* fix for rbac objects for app grouping api

* fix

* fix

* ci workflow status fix

* added otel tracer for app grouping api's

* review changes

* fix

* review changes

Author: vikramdevtron

api/restHandler/AppWorkflowRestHandler.go

@@ -29,6 +29,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
 	"github.com/devtron-labs/devtron/util/rbac"
 	"github.com/gorilla/mux"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
 	"net/http"
 	"strconv"
@@ -239,7 +240,9 @@ func (impl AppWorkflowRestHandlerImpl) FindAppWorkflowByEnvironment(w http.Respo
 		return
 	}
 	workflows := make(map[string]interface{})
+	_, span := otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchAppWorkflowsInAppGrouping")
 	workflowsList, err := impl.appWorkflowService.FindAppWorkflowsByEnvironmentId(envId, userEmailId, impl.checkAuthBatch)
+	span.End()
 	if err != nil {
 		impl.Logger.Errorw("error in fetching workflows for app", "err", err)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/BuildPipelineRestHandler.go

@@ -16,6 +16,7 @@ import (
 	bean1 "github.com/devtron-labs/devtron/pkg/pipeline/bean"
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
 	"github.com/gorilla/mux"
+	"go.opentelemetry.io/otel"
 	"io"
 	"net/http"
 	"strconv"
@@ -1333,7 +1334,9 @@ func (handler PipelineConfigRestHandlerImpl) GetCiPipelineByEnvironment(w http.R
 		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
-	ciConf, err := handler.pipelineBuilder.GetCiPipelineByEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	_, span := otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchCiPipelinesForAppGrouping")
+	ciConf, err := handler.pipelineBuilder.GetCiPipelineByEnvironment(envId, userEmailId, handler.checkAuthBatch, r.Context())
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err, GetCiPipeline", "err", err, "envId", envId)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/DeploymentPipelineRestHandler.go

@@ -1961,7 +1961,9 @@ func (handler PipelineConfigRestHandlerImpl) GetCdPipelinesByEnvironment(w http.
 		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
+	_, span := otel.Tracer("orchestrator").Start(context.Background(), "ciHandler.FetchCdPipelinesForAppGrouping")
 	results, err := handler.pipelineBuilder.GetCdPipelinesByEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err, GetCdPipelines", "err", err, "envId", envId)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/PipelineConfigRestHandler.go

@@ -25,6 +25,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/helper"
 	"github.com/devtron-labs/devtron/pkg/chart"
 	"github.com/devtron-labs/devtron/util/argo"
+	"go.opentelemetry.io/otel"
 	"io"
 	"net/http"
 	"strconv"
@@ -593,7 +594,9 @@ func (handler PipelineConfigRestHandlerImpl) FetchAppWorkflowStatusForTriggerVie
 		return
 	}
 	triggerWorkflowStatus := pipelineConfig.TriggerWorkflowStatus{}
+	_, span := otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchCiStatusForBuildAndDeployInAppGrouping")
 	ciWorkflowStatus, err := handler.ciHandler.FetchCiStatusForTriggerViewForEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err", "err", err)
 		if util.IsErrNoRows(err) {
@@ -605,7 +608,9 @@ func (handler PipelineConfigRestHandlerImpl) FetchAppWorkflowStatusForTriggerVie
 		return
 	}
 
+	_, span = otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchCdStatusForBuildAndDeployInAppGrouping")
 	cdWorkflowStatus, err := handler.cdHandler.FetchAppWorkflowStatusForTriggerViewForEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err, FetchAppWorkflowStatusForTriggerView", "err", err)
 		if util.IsErrNoRows(err) {

cmd/external-app/wire_gen.go

@@ -56,6 +56,9 @@ type AppWorkflowRepository interface {
 	FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(envIds []int, appIds []int) ([]*AppWorkflowMapping, error)
 	FindCiPipelineIdsFromAppWfIds(appWfIds []int) ([]int, error)
 	FindChildCDIdsByParentCDPipelineId(cdPipelineId int) ([]int, error)
+	FindByCDPipelineIds(cdPipelineIds []int) ([]*AppWorkflowMapping, error)
+	FindByWorkflowIds(workflowIds []int) ([]*AppWorkflowMapping, error)
+	FindMappingByAppIds(appIds []int) ([]*AppWorkflowMapping, error)
 }
 
 type AppWorkflowRepositoryImpl struct {
@@ -323,6 +326,7 @@ func (impl AppWorkflowRepositoryImpl) FindWFCDMappingByParentCDPipelineId(cdPipe
 		Select()
 	return appWorkflowsMapping, err
 }
+
 func (impl AppWorkflowRepositoryImpl) DeleteAppWorkflowMapping(appWorkflow *AppWorkflowMapping, tx *pg.Tx) error {
 	appWorkflow.Active = false
 	err := tx.Update(appWorkflow)
@@ -400,3 +404,32 @@ func (impl AppWorkflowRepositoryImpl) FindChildCDIdsByParentCDPipelineId(cdPipel
 	_, err := impl.dbConnection.Query(&ids, query, cdPipelineId, CDPIPELINE, CDPIPELINE, true)
 	return ids, err
 }
+
+func (impl AppWorkflowRepositoryImpl) FindByCDPipelineIds(cdPipelineIds []int) ([]*AppWorkflowMapping, error) {
+	var appWorkflowsMapping []*AppWorkflowMapping
+	err := impl.dbConnection.Model(&appWorkflowsMapping).
+		Where("component_id in (?)", pg.In(cdPipelineIds)).
+		Where("type = ?", CDPIPELINE).
+		Where("active = ?", true).
+		Select()
+	return appWorkflowsMapping, err
+}
+
+func (impl AppWorkflowRepositoryImpl) FindByWorkflowIds(workflowIds []int) ([]*AppWorkflowMapping, error) {
+	var appWorkflowsMapping []*AppWorkflowMapping
+	err := impl.dbConnection.Model(&appWorkflowsMapping).
+		Where("app_workflow_id in (?)", pg.In(workflowIds)).
+		Where("active = ?", true).
+		Select()
+	return appWorkflowsMapping, err
+}
+
+func (impl AppWorkflowRepositoryImpl) FindMappingByAppIds(appIds []int) ([]*AppWorkflowMapping, error) {
+	var appWorkflowsMapping []*AppWorkflowMapping
+	err := impl.dbConnection.Model(&appWorkflowsMapping).Column("app_workflow_mapping.*", "AppWorkflow").
+		Where("app_workflow.app_id in (?)", pg.In(appIds)).
+		Where("app_workflow.active = ?", true).
+		Where("app_workflow_mapping.active = ?", true).
+		Select()
+	return appWorkflowsMapping, err
+}

internal/sql/repository/appWorkflow/AppWorkflowRepository.go

@@ -44,6 +44,7 @@ type PipelineConfigRepository interface {
 	GetAllStrategyByPipelineId(pipelineId int) ([]*PipelineStrategy, error)
 	GetDefaultStrategyByPipelineId(pipelineId int) (pipelineStrategy *PipelineStrategy, err error)
 	Delete(pipelineStrategy *PipelineStrategy, tx *pg.Tx) error
+	GetAllStrategyByPipelineIds(pipelineIds []int) ([]*PipelineStrategy, error)
 }
 
 type PipelineConfigRepositoryImpl struct {
@@ -117,3 +118,16 @@ func (impl PipelineConfigRepositoryImpl) GetDefaultStrategyByPipelineId(pipeline
 func (impl PipelineConfigRepositoryImpl) Delete(pipelineStrategy *PipelineStrategy, tx *pg.Tx) error {
 	return tx.Delete(pipelineStrategy)
 }
+
+func (impl PipelineConfigRepositoryImpl) GetAllStrategyByPipelineIds(pipelineIds []int) ([]*PipelineStrategy, error) {
+	var pipelineStrategies []*PipelineStrategy
+	err := impl.dbConnection.
+		Model(&pipelineStrategies).
+		Where("pipeline_id in (?)", pg.In(pipelineIds)).
+		Where("deleted = ?", false).
+		Select()
+	if pg.ErrNoRows == err {
+		return nil, errors.NotFoundf(err.Error())
+	}
+	return pipelineStrategies, err
+}

internal/sql/repository/chartConfig/PipelineConfigRepository.go

@@ -98,6 +98,7 @@ type CiPipelineRepository interface {
 	FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error)
 	FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error)
 	FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error)
+	FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error)
 }
 type CiPipelineRepositoryImpl struct {
 	dbConnection *pg.DB
@@ -377,3 +378,12 @@ func (impl CiPipelineRepositoryImpl) FindNumberOfAppsWithCiPipeline(appIds []int
 
 	return count, nil
 }
+
+func (impl CiPipelineRepositoryImpl) FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error) {
+	var ciPipelines []*CiPipeline
+	err := impl.dbConnection.Model(&ciPipelines).Column("ci_pipeline.*", "App", "App.Team").
+		Where("ci_pipeline.id in(?)", pg.In(ciPipelineIds)).
+		Where("ci_pipeline.deleted = ?", false).
+		Select()
+	return ciPipelines, err
+}

internal/sql/repository/pipelineConfig/CiPipelineRepository.go

@@ -207,13 +207,12 @@ func (impl *CiWorkflowRepositoryImpl) UpdateWorkFlow(wf *CiWorkflow) error {
 }
 
 func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) {
-	var workflow []*CiWorkflow
-	err = impl.dbConnection.Model(workflow).
+	err = impl.dbConnection.Model(&ciWorkflow).
 		Column("ci_workflow.*", "CiPipeline").
-		Where("ci_workflow.ci_pipeline_id = ? ", pg.In(pipelineId)).
+		Where("ci_workflow.ci_pipeline_id in (?) ", pg.In(pipelineId)).
 		Order("ci_workflow.started_on Desc").
 		Select()
-	return workflow, err
+	return ciWorkflow, err
 }
 
 func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) {

internal/sql/repository/pipelineConfig/CiWorkflowRepository.go

@@ -104,6 +104,7 @@ type PipelineRepository interface {
 	GetArgoPipelineByArgoAppName(argoAppName string) (Pipeline, error)
 	GetPartiallyDeletedPipelineByStatus(appId int, envId int) (Pipeline, error)
 	FindActiveByAppIds(appIds []int) (pipelines []*Pipeline, err error)
+	FindAppAndEnvironmentAndProjectByPipelineIds(pipelineIds []int) (pipelines []*Pipeline, err error)
 }
 
 type CiArtifactDTO struct {
@@ -602,3 +603,11 @@ func (impl PipelineRepositoryImpl) FindActiveByAppIds(appIds []int) (pipelines [
 		Select()
 	return pipelines, err
 }
+
+func (impl PipelineRepositoryImpl) FindAppAndEnvironmentAndProjectByPipelineIds(pipelineIds []int) (pipelines []*Pipeline, err error) {
+	err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment", "App.Team").
+		Where("pipeline.id in(?)", pg.In(pipelineIds)).
+		Where("pipeline.deleted = ?", false).
+		Select()
+	return pipelines, err
+}