fix: Optimize app grouping apis

api/restHandler/AppWorkflowRestHandler.go

@@ -29,6 +29,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
 	"github.com/devtron-labs/devtron/util/rbac"
 	"github.com/gorilla/mux"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
 	"net/http"
 	"strconv"
@@ -239,7 +240,9 @@ func (impl AppWorkflowRestHandlerImpl) FindAppWorkflowByEnvironment(w http.Respo
 		return
 	}
 	workflows := make(map[string]interface{})
+	_, span := otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchAppWorkflowsInAppGrouping")
 	workflowsList, err := impl.appWorkflowService.FindAppWorkflowsByEnvironmentId(envId, userEmailId, impl.checkAuthBatch)
+	span.End()
 	if err != nil {
 		impl.Logger.Errorw("error in fetching workflows for app", "err", err)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/BuildPipelineRestHandler.go

@@ -16,6 +16,7 @@ import (
 	bean1 "github.com/devtron-labs/devtron/pkg/pipeline/bean"
 	"github.com/devtron-labs/devtron/pkg/user/casbin"
 	"github.com/gorilla/mux"
+	"go.opentelemetry.io/otel"
 	"io"
 	"net/http"
 	"strconv"
@@ -1333,7 +1334,9 @@ func (handler PipelineConfigRestHandlerImpl) GetCiPipelineByEnvironment(w http.R
 		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
-	ciConf, err := handler.pipelineBuilder.GetCiPipelineByEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	_, span := otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchCiPipelinesForAppGrouping")
+	ciConf, err := handler.pipelineBuilder.GetCiPipelineByEnvironment(envId, userEmailId, handler.checkAuthBatch, r.Context())
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err, GetCiPipeline", "err", err, "envId", envId)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/DeploymentPipelineRestHandler.go

@@ -1961,7 +1961,9 @@ func (handler PipelineConfigRestHandlerImpl) GetCdPipelinesByEnvironment(w http.
 		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 		return
 	}
+	_, span := otel.Tracer("orchestrator").Start(context.Background(), "ciHandler.FetchCdPipelinesForAppGrouping")
 	results, err := handler.pipelineBuilder.GetCdPipelinesByEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err, GetCdPipelines", "err", err, "envId", envId)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/app/PipelineConfigRestHandler.go

@@ -25,6 +25,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/helper"
 	"github.com/devtron-labs/devtron/pkg/chart"
 	"github.com/devtron-labs/devtron/util/argo"
+	"go.opentelemetry.io/otel"
 	"io"
 	"net/http"
 	"strconv"
@@ -593,7 +594,9 @@ func (handler PipelineConfigRestHandlerImpl) FetchAppWorkflowStatusForTriggerVie
 		return
 	}
 	triggerWorkflowStatus := pipelineConfig.TriggerWorkflowStatus{}
+	_, span := otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchCiStatusForBuildAndDeployInAppGrouping")
 	ciWorkflowStatus, err := handler.ciHandler.FetchCiStatusForTriggerViewForEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err", "err", err)
 		if util.IsErrNoRows(err) {
@@ -605,7 +608,9 @@ func (handler PipelineConfigRestHandlerImpl) FetchAppWorkflowStatusForTriggerVie
 		return
 	}
 
+	_, span = otel.Tracer("orchestrator").Start(r.Context(), "ciHandler.FetchCdStatusForBuildAndDeployInAppGrouping")
 	cdWorkflowStatus, err := handler.cdHandler.FetchAppWorkflowStatusForTriggerViewForEnvironment(envId, userEmailId, handler.checkAuthBatch)
+	span.End()
 	if err != nil {
 		handler.Logger.Errorw("service err, FetchAppWorkflowStatusForTriggerView", "err", err)
 		if util.IsErrNoRows(err) {

internal/sql/repository/appWorkflow/AppWorkflowRepository.go

@@ -56,6 +56,9 @@ type AppWorkflowRepository interface {
 	FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(envIds []int, appIds []int) ([]*AppWorkflowMapping, error)
 	FindCiPipelineIdsFromAppWfIds(appWfIds []int) ([]int, error)
 	FindChildCDIdsByParentCDPipelineId(cdPipelineId int) ([]int, error)
+	FindByCDPipelineIds(cdPipelineIds []int) ([]*AppWorkflowMapping, error)
+	FindByWorkflowIds(workflowIds []int) ([]*AppWorkflowMapping, error)
+	FindMappingByAppIds(appIds []int) ([]*AppWorkflowMapping, error)
 }
 
 type AppWorkflowRepositoryImpl struct {
@@ -323,6 +326,7 @@ func (impl AppWorkflowRepositoryImpl) FindWFCDMappingByParentCDPipelineId(cdPipe
 		Select()
 	return appWorkflowsMapping, err
 }
+
 func (impl AppWorkflowRepositoryImpl) DeleteAppWorkflowMapping(appWorkflow *AppWorkflowMapping, tx *pg.Tx) error {
 	appWorkflow.Active = false
 	err := tx.Update(appWorkflow)
@@ -400,3 +404,32 @@ func (impl AppWorkflowRepositoryImpl) FindChildCDIdsByParentCDPipelineId(cdPipel
 	_, err := impl.dbConnection.Query(&ids, query, cdPipelineId, CDPIPELINE, CDPIPELINE, true)
 	return ids, err
 }
+
+func (impl AppWorkflowRepositoryImpl) FindByCDPipelineIds(cdPipelineIds []int) ([]*AppWorkflowMapping, error) {
+	var appWorkflowsMapping []*AppWorkflowMapping
+	err := impl.dbConnection.Model(&appWorkflowsMapping).
+		Where("component_id in (?)", pg.In(cdPipelineIds)).
+		Where("type = ?", CDPIPELINE).
+		Where("active = ?", true).
+		Select()
+	return appWorkflowsMapping, err
+}
+
+func (impl AppWorkflowRepositoryImpl) FindByWorkflowIds(workflowIds []int) ([]*AppWorkflowMapping, error) {
+	var appWorkflowsMapping []*AppWorkflowMapping
+	err := impl.dbConnection.Model(&appWorkflowsMapping).
+		Where("app_workflow_id in (?)", pg.In(workflowIds)).
+		Where("active = ?", true).
+		Select()
+	return appWorkflowsMapping, err
+}
+
+func (impl AppWorkflowRepositoryImpl) FindMappingByAppIds(appIds []int) ([]*AppWorkflowMapping, error) {
+	var appWorkflowsMapping []*AppWorkflowMapping
+	err := impl.dbConnection.Model(&appWorkflowsMapping).Column("app_workflow_mapping.*", "AppWorkflow").
+		Where("app_workflow.app_id in (?)", pg.In(appIds)).
+		Where("app_workflow.active = ?", true).
+		Where("app_workflow_mapping.active = ?", true).
+		Select()
+	return appWorkflowsMapping, err
+}

internal/sql/repository/chartConfig/PipelineConfigRepository.go

@@ -44,6 +44,7 @@ type PipelineConfigRepository interface {
 	GetAllStrategyByPipelineId(pipelineId int) ([]*PipelineStrategy, error)
 	GetDefaultStrategyByPipelineId(pipelineId int) (pipelineStrategy *PipelineStrategy, err error)
 	Delete(pipelineStrategy *PipelineStrategy, tx *pg.Tx) error
+	GetAllStrategyByPipelineIds(pipelineIds []int) ([]*PipelineStrategy, error)
 }
 
 type PipelineConfigRepositoryImpl struct {
@@ -117,3 +118,16 @@ func (impl PipelineConfigRepositoryImpl) GetDefaultStrategyByPipelineId(pipeline
 func (impl PipelineConfigRepositoryImpl) Delete(pipelineStrategy *PipelineStrategy, tx *pg.Tx) error {
 	return tx.Delete(pipelineStrategy)
 }
+
+func (impl PipelineConfigRepositoryImpl) GetAllStrategyByPipelineIds(pipelineIds []int) ([]*PipelineStrategy, error) {
+	var pipelineStrategies []*PipelineStrategy
+	err := impl.dbConnection.
+		Model(&pipelineStrategies).
+		Where("pipeline_id in (?)", pg.In(pipelineIds)).
+		Where("deleted = ?", false).
+		Select()
+	if pg.ErrNoRows == err {
+		return nil, errors.NotFoundf(err.Error())
+	}
+	return pipelineStrategies, err
+}

internal/sql/repository/pipelineConfig/CiPipelineRepository.go

@@ -98,6 +98,7 @@ type CiPipelineRepository interface {
 	FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error)
 	FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error)
 	FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error)
+	FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error)
 }
 type CiPipelineRepositoryImpl struct {
 	dbConnection *pg.DB
@@ -377,3 +378,12 @@ func (impl CiPipelineRepositoryImpl) FindNumberOfAppsWithCiPipeline(appIds []int
 
 	return count, nil
 }
+
+func (impl CiPipelineRepositoryImpl) FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error) {
+	var ciPipelines []*CiPipeline
+	err := impl.dbConnection.Model(&ciPipelines).Column("ci_pipeline.*", "App", "App.Team").
+		Where("ci_pipeline.id in(?)", pg.In(ciPipelineIds)).
+		Where("ci_pipeline.deleted = ?", false).
+		Select()
+	return ciPipelines, err
+}

internal/sql/repository/pipelineConfig/CiWorkflowRepository.go

@@ -207,13 +207,12 @@ func (impl *CiWorkflowRepositoryImpl) UpdateWorkFlow(wf *CiWorkflow) error {
 }
 
 func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) {
-	var workflow []*CiWorkflow
-	err = impl.dbConnection.Model(workflow).
+	err = impl.dbConnection.Model(&ciWorkflow).
 		Column("ci_workflow.*", "CiPipeline").
-		Where("ci_workflow.ci_pipeline_id = ? ", pg.In(pipelineId)).
+		Where("ci_workflow.ci_pipeline_id in (?) ", pg.In(pipelineId)).
 		Order("ci_workflow.started_on Desc").
 		Select()
-	return workflow, err
+	return ciWorkflow, err
 }
 
 func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) {

internal/sql/repository/pipelineConfig/PipelineRepository.go

@@ -104,6 +104,7 @@ type PipelineRepository interface {
 	GetArgoPipelineByArgoAppName(argoAppName string) (Pipeline, error)
 	GetPartiallyDeletedPipelineByStatus(appId int, envId int) (Pipeline, error)
 	FindActiveByAppIds(appIds []int) (pipelines []*Pipeline, err error)
+	FindAppAndEnvironmentAndProjectByPipelineIds(pipelineIds []int) (pipelines []*Pipeline, err error)
 }
 
 type CiArtifactDTO struct {
@@ -602,3 +603,11 @@ func (impl PipelineRepositoryImpl) FindActiveByAppIds(appIds []int) (pipelines [
 		Select()
 	return pipelines, err
 }
+
+func (impl PipelineRepositoryImpl) FindAppAndEnvironmentAndProjectByPipelineIds(pipelineIds []int) (pipelines []*Pipeline, err error) {
+	err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment", "App.Team").
+		Where("pipeline.id in(?)", pg.In(pipelineIds)).
+		Where("pipeline.deleted = ?", false).
+		Select()
+	return pipelines, err
+}

pkg/appWorkflow/AppWorkflowService.go

@@ -48,7 +48,7 @@ type AppWorkflowService interface {
 	FindAppWorkflowByName(name string, appId int) (AppWorkflowDto, error)
 
 	FindAllWorkflowsComponentDetails(appId int) (*AllAppWorkflowComponentDetails, error)
-	FindAppWorkflowsByEnvironmentId(envId int, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) ([]AppWorkflowDto, error)
+	FindAppWorkflowsByEnvironmentId(envId int, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) ([]*AppWorkflowDto, error)
 }
 
 type AppWorkflowServiceImpl struct {
@@ -301,6 +301,53 @@ func (impl AppWorkflowServiceImpl) FindAppWorkflowMapping(workflowId int) ([]App
 	return workflows, err
 }
 
+func (impl AppWorkflowServiceImpl) FindAppWorkflowMappingForEnv(appIds []int) (map[int]*AppWorkflowDto, error) {
+	appWorkflowMappings, err := impl.appWorkflowRepository.FindMappingByAppIds(appIds)
+	if err != nil && err != pg.ErrNoRows {
+		impl.Logger.Errorw("err", err)
+		return nil, err
+	}
+	pipelineIds := make([]int, 0)
+	for _, w := range appWorkflowMappings {
+		if w.Type == "CD_PIPELINE" {
+			pipelineIds = append(pipelineIds, w.ComponentId)
+		}
+	}
+	pipelines, err := impl.pipelineRepository.FindByIdsIn(pipelineIds)
+	if err != nil && err != pg.ErrNoRows {
+		impl.Logger.Errorw("err", "err", err)
+		return nil, err
+	}
+	pipelineMap := make(map[int]*pipelineConfig.Pipeline)
+	for _, pipeline := range pipelines {
+		pipelineMap[pipeline.Id] = pipeline
+	}
+	workflowMappings := make(map[int][]AppWorkflowMappingDto)
+	workflows := make(map[int]*AppWorkflowDto)
+	for _, w := range appWorkflowMappings {
+		if _, ok := workflows[w.AppWorkflowId]; !ok {
+			workflows[w.AppWorkflowId] = &AppWorkflowDto{
+				Id:    w.AppWorkflowId,
+				AppId: w.AppWorkflow.AppId,
+			}
+		}
+		workflow := AppWorkflowMappingDto{
+			Id:            w.Id,
+			ParentId:      w.ParentId,
+			ComponentId:   w.ComponentId,
+			Type:          w.Type,
+			AppWorkflowId: w.AppWorkflowId,
+			ParentType:    w.ParentType,
+		}
+		if w.Type == "CD_PIPELINE" {
+			workflow.DeploymentAppDeleteRequest = pipelineMap[w.ComponentId].DeploymentAppDeleteRequest
+		}
+		workflowMappings[w.AppWorkflowId] = append(workflowMappings[w.AppWorkflowId], workflow)
+		workflows[w.AppWorkflowId].AppWorkflowMappingDto = workflowMappings[w.AppWorkflowId]
+	}
+	return workflows, err
+}
+
 func (impl AppWorkflowServiceImpl) FindAppWorkflowMappingByComponent(id int, compType string) ([]*appWorkflow.AppWorkflowMapping, error) {
 	appWorkflowMappings, err := impl.appWorkflowRepository.FindByComponent(id, compType)
 	if err != nil && err != pg.ErrNoRows {
@@ -398,8 +445,8 @@ func (impl AppWorkflowServiceImpl) FindAllWorkflowsComponentDetails(appId int) (
 	return resp, nil
 }
 
-func (impl AppWorkflowServiceImpl) FindAppWorkflowsByEnvironmentId(envId int, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) ([]AppWorkflowDto, error) {
-	workflows := make([]AppWorkflowDto, 0)
+func (impl AppWorkflowServiceImpl) FindAppWorkflowsByEnvironmentId(envId int, emailId string, checkAuthBatch func(emailId string, appObject []string, envObject []string) (map[string]bool, map[string]bool)) ([]*AppWorkflowDto, error) {
+	workflows := make([]*AppWorkflowDto, 0)
 	pipelines, err := impl.pipelineRepository.FindActiveByEnvId(envId)
 	if err != nil && err != pg.ErrNoRows {
 		impl.Logger.Errorw("error fetching pipelines for env id", "err", err)
@@ -409,20 +456,25 @@ func (impl AppWorkflowServiceImpl) FindAppWorkflowsByEnvironmentId(envId int, em
 	appNamesMap := make(map[int]string)
 	var appIds []int
 	//authorization block starts here
-	var envObjectArr []string
-	var appObjectArr []string
-	rbacObjectMap := make(map[int][]string)
+	pipelineIds := make([]int, 0)
 	for _, pipeline := range pipelines {
-		appObject := impl.enforcerUtil.GetAppRBACName(pipeline.App.AppName)
-		envObject := impl.enforcerUtil.GetEnvRBACNameByCdPipelineIdAndEnvId(pipeline.Id)
-		appObjectArr = append(appObjectArr, appObject)
-		envObjectArr = append(envObjectArr, envObject)
-		rbacObjectMap[pipeline.Id] = []string{appObject, envObject}
+		pipelineIds = append(pipelineIds, pipeline.Id)
+	}
+	if len(pipelineIds) == 0 {
+		return workflows, fmt.Errorf("no pipeline found for this environment")
+	}
+	var appObjectArr []string
+	var envObjectArr []string
+	objects := impl.enforcerUtil.GetAppAndEnvObjectByPipelineIds(pipelineIds)
+	pipelineIds = []int{}
+	for _, object := range objects {
+		appObjectArr = append(appObjectArr, object[0])
+		envObjectArr = append(envObjectArr, object[1])
 	}
 	appResults, envResults := checkAuthBatch(emailId, appObjectArr, envObjectArr)
 	for _, pipeline := range pipelines {
-		appObject := rbacObjectMap[pipeline.Id][0]
-		envObject := rbacObjectMap[pipeline.Id][1]
+		appObject := objects[pipeline.Id][0]
+		envObject := objects[pipeline.Id][1]
 		if !(appResults[appObject] && envResults[envObject]) {
 			//if user unauthorized, skip items
 			continue
@@ -437,23 +489,15 @@ func (impl AppWorkflowServiceImpl) FindAppWorkflowsByEnvironmentId(envId int, em
 		impl.Logger.Warnw("there is no app id found for fetching app workflows", "envId", envId)
 		return workflows, nil
 	}
-	appWorkflow, err := impl.appWorkflowRepository.FindByAppIds(appIds)
-	if err != nil && err != pg.ErrNoRows {
-		impl.Logger.Errorw("error fetching app workflows by app ids", "err", err)
+	appWorkflows, err := impl.FindAppWorkflowMappingForEnv(appIds)
+	if err != nil {
+		impl.Logger.Errorw("error fetching app workflow mapping by wf id", "err", err)
 		return nil, err
 	}
-	for _, w := range appWorkflow {
-		appName := appNamesMap[w.AppId]
-		workflow := AppWorkflowDto{
-			Id:    w.Id,
-			Name:  appName, // here workflow name is app name, only for environment app grouping view
-			AppId: w.AppId,
-		}
-		mappings, err := impl.FindAppWorkflowMapping(w.Id)
-		if err != nil {
-			impl.Logger.Errorw("error fetching app workflow mapping by wf id", "err", err)
-			return nil, err
-		}
+	for _, appWorkflow := range appWorkflows {
+		appName := appNamesMap[appWorkflow.AppId]
+		appWorkflow.Name = appName
+		mappings := appWorkflow.AppWorkflowMappingDto
 		valid := false
 		for _, mapping := range mappings {
 			if mapping.Type == CD_PIPELINE_TYPE {
@@ -464,8 +508,7 @@ func (impl AppWorkflowServiceImpl) FindAppWorkflowsByEnvironmentId(envId int, em
 		}
 		//if there is no matching pipeline for requested environment, skip from workflow listing
 		if valid {
-			workflow.AppWorkflowMappingDto = mappings
-			workflows = append(workflows, workflow)
+			workflows = append(workflows, appWorkflow)
 		}
 	}
 	return workflows, err