fix: optimised the artifacts listing query and added query versioned support (#4375)

* add v2 query for fetching artifacts

* query update

Author: gireesh-naidu

api/bean/ValuesOverrideRequest.go

@@ -113,6 +113,9 @@ type ArtifactsListFilterOptions struct {
 	PipelineId int
 	StageType  WorkflowType
 
+	// CiPipelineId is id of ci-pipeline present in the same app-workflow of PipelineId
+	CiPipelineId int
+
 	//parent satge data
 	ParentCdId      int
 	ParentId        int
@@ -126,4 +129,7 @@ type ArtifactsListFilterOptions struct {
 
 	//pluginStage
 	PluginStage string
+
+	// UseCdStageQueryV2 is to set query version
+	UseCdStageQueryV2 bool
 }

internal/sql/repository/CiArtifactRepository.go

@@ -713,13 +713,22 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByParentCiWorkflowId(parentCiWo
 func (impl CiArtifactRepositoryImpl) FindArtifactByListFilter(listingFilterOptions *bean.ArtifactsListFilterOptions) ([]CiArtifact, int, error) {
 
 	var ciArtifactsResp []CiArtifactWithExtraData
-	var ciArtifacts []CiArtifact
+	ciArtifacts := make([]CiArtifact, 0)
 	totalCount := 0
 	finalQuery := BuildQueryForArtifactsForCdStage(*listingFilterOptions)
 	_, err := impl.dbConnection.Query(&ciArtifactsResp, finalQuery)
 	if err == pg.ErrNoRows || len(ciArtifactsResp) == 0 {
 		return ciArtifacts, totalCount, nil
 	}
+
+	if listingFilterOptions.UseCdStageQueryV2 {
+		for _, cia := range ciArtifactsResp {
+			totalCount = cia.TotalCount
+			ciArtifacts = append(ciArtifacts, cia.CiArtifact)
+		}
+		return ciArtifacts, totalCount, err
+	}
+
 	artifactIds := make([]int, len(ciArtifactsResp))
 	for i, af := range ciArtifactsResp {
 		artifactIds[i] = af.Id

internal/sql/repository/CiArtifactsListingQueryBuilder.go

@@ -50,6 +50,10 @@ func BuildQueryForArtifactsForCdStage(listingFilterOptions bean.ArtifactsListFil
 	// expected result -> will fetch all successfully deployed  artifacts ar parent stage plus its own stage. Along with this it will
 	// also fetch all artifacts generated by plugin at pre_cd or post_cd process (will use data_source in ci artifact table for this)
 
+	if listingFilterOptions.UseCdStageQueryV2 {
+		return buildQueryForArtifactsForCdStageV2(listingFilterOptions)
+	}
+
 	commonQuery := " from ci_artifact LEFT JOIN cd_workflow ON ci_artifact.id = cd_workflow.ci_artifact_id" +
 		" LEFT JOIN cd_workflow_runner ON cd_workflow_runner.cd_workflow_id=cd_workflow.id " +
 		" Where (((cd_workflow_runner.id in (select MAX(cd_workflow_runner.id) OVER (PARTITION BY cd_workflow.ci_artifact_id) FROM cd_workflow_runner inner join cd_workflow on cd_workflow.id=cd_workflow_runner.cd_workflow_id))" +
@@ -72,6 +76,36 @@ func BuildQueryForArtifactsForCdStage(listingFilterOptions bean.ArtifactsListFil
 	return finalQuery
 }
 
+func buildQueryForArtifactsForCdStageV2(listingFilterOptions bean.ArtifactsListFilterOptions) string {
+	whereCondition := fmt.Sprintf(" WHERE (id IN ("+
+		" SELECT DISTINCT(cd_workflow.ci_artifact_id) as ci_artifact_id "+
+		" FROM cd_workflow_runner"+
+		" INNER JOIN cd_workflow ON cd_workflow.id = cd_workflow_runner.cd_workflow_id "+
+		" AND (cd_workflow.pipeline_id = %d OR cd_workflow.pipeline_id = %d)"+
+		"    WHERE ("+
+		"            (cd_workflow.pipeline_id = %d AND cd_workflow_runner.workflow_type = '%s')"+
+		"            OR"+
+		"            (cd_workflow.pipeline_id = %d"+
+		"                AND cd_workflow_runner.workflow_type = '%s'"+
+		"                AND cd_workflow_runner.status IN ('Healthy','Succeeded')"+
+		"           )"+
+		"      )   ) ", listingFilterOptions.PipelineId, listingFilterOptions.ParentId, listingFilterOptions.PipelineId, listingFilterOptions.StageType, listingFilterOptions.ParentId, listingFilterOptions.ParentStageType)
+
+	whereCondition = fmt.Sprintf(" %s OR (ci_artifact.component_id = %d  AND ci_artifact.data_source= '%s' ))", whereCondition, listingFilterOptions.ParentId, listingFilterOptions.PluginStage)
+	if listingFilterOptions.SearchString != EmptyLikeRegex {
+		whereCondition = whereCondition + fmt.Sprintf(" AND ci_artifact.image LIKE '%s' ", listingFilterOptions.SearchString)
+	}
+	if len(listingFilterOptions.ExcludeArtifactIds) > 0 {
+		whereCondition = whereCondition + fmt.Sprintf(" AND ( ci_artifact.id NOT IN (%s))", helper.GetCommaSepratedString(listingFilterOptions.ExcludeArtifactIds))
+	}
+
+	selectQuery := fmt.Sprintf(" SELECT ci_artifact.* ,COUNT(id) OVER() AS total_count " +
+		" FROM ci_artifact")
+	ordeyByAndPaginated := fmt.Sprintf(" ORDER BY id DESC LIMIT %d OFFSET %d ", listingFilterOptions.Limit, listingFilterOptions.Offset)
+	finalQuery := selectQuery + whereCondition + ordeyByAndPaginated
+	return finalQuery
+}
+
 func BuildQueryForArtifactsForRollback(listingFilterOptions bean.ArtifactsListFilterOptions) string {
 	commonQuery := " FROM cd_workflow_runner cdwr " +
 		" INNER JOIN cd_workflow cdw ON cdw.id=cdwr.cd_workflow_id " +

pkg/pipeline/AppArtifactManager.go

@@ -25,6 +25,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	bean2 "github.com/devtron-labs/devtron/pkg/bean"
 	repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository"
+	"github.com/devtron-labs/devtron/pkg/pipeline/types"
 	"github.com/devtron-labs/devtron/pkg/user"
 	"github.com/go-pg/pg"
 	"go.uber.org/zap"
@@ -56,6 +57,7 @@ type AppArtifactManagerImpl struct {
 	ciArtifactRepository    repository.CiArtifactRepository
 	ciWorkflowRepository    pipelineConfig.CiWorkflowRepository
 	pipelineStageService    PipelineStageService
+	config                  *types.CdConfig
 	cdPipelineConfigService CdPipelineConfigService
 	dockerArtifactRegistry  dockerArtifactStoreRegistry.DockerArtifactStoreRepository
 	CiPipelineRepository    pipelineConfig.CiPipelineRepository
@@ -74,7 +76,10 @@ func NewAppArtifactManagerImpl(
 	dockerArtifactRegistry dockerArtifactStoreRegistry.DockerArtifactStoreRepository,
 	CiPipelineRepository pipelineConfig.CiPipelineRepository,
 	ciTemplateService CiTemplateService) *AppArtifactManagerImpl {
-
+	cdConfig, err := types.GetCdConfig()
+	if err != nil {
+		return nil
+	}
 	return &AppArtifactManagerImpl{
 		logger:                  logger,
 		cdWorkflowRepository:    cdWorkflowRepository,
@@ -84,6 +89,7 @@ func NewAppArtifactManagerImpl(
 		ciWorkflowRepository:    ciWorkflowRepository,
 		cdPipelineConfigService: cdPipelineConfigService,
 		pipelineStageService:    pipelineStageService,
+		config:                  cdConfig,
 		dockerArtifactRegistry:  dockerArtifactRegistry,
 		CiPipelineRepository:    CiPipelineRepository,
 		ciTemplateService:       ciTemplateService,
@@ -606,11 +612,14 @@ func (impl *AppArtifactManagerImpl) RetrieveArtifactsByCDPipelineV2(pipeline *pi
 	ciArtifactsResponse := &bean2.CiArtifactResponse{}
 
 	artifactListingFilterOpts.PipelineId = pipeline.Id
+	// this will be 0 for external-ci cases, note: do not refer this for external-ci cases
+	artifactListingFilterOpts.CiPipelineId = pipeline.CiPipelineId
 	artifactListingFilterOpts.ParentId = parentId
 	artifactListingFilterOpts.ParentCdId = parentCdId
 	artifactListingFilterOpts.ParentStageType = parentType
 	artifactListingFilterOpts.StageType = stage
 	artifactListingFilterOpts.SearchString = "%" + artifactListingFilterOpts.SearchString + "%"
+	artifactListingFilterOpts.UseCdStageQueryV2 = impl.config.UseArtifactListingQueryV2
 	ciArtifactsRefs, latestWfArtifactId, latestWfArtifactStatus, totalCount, err := impl.BuildArtifactsList(artifactListingFilterOpts)
 	if err != nil && err != pg.ErrNoRows {
 		impl.logger.Errorw("error in getting artifacts for child cd stage", "err", err, "stage", stage)

pkg/pipeline/types/CiCdConfig.go

@@ -121,6 +121,7 @@ type CiCdConfig struct {
 	BuildxProvenanceMode           string                       `env:"BUILDX_PROVENANCE_MODE" envDefault:""` //provenance is set to false if this flag is not set
 	ExtBlobStorageCmName           string                       `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"`
 	ExtBlobStorageSecretName       string                       `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"`
+	UseArtifactListingQueryV2      bool                         `env:"USE_ARTIFACT_LISTING_QUERY_V2" envDefault:"true"`
 }
 
 type CiConfig struct {