feat: copy container images

Wire.go

@@ -964,6 +964,9 @@ func InitializeApp() (*App, error) {
 
 		argocdServer.NewArgoClientWrapperServiceImpl,
 		wire.Bind(new(argocdServer.ArgoClientWrapperService), new(*argocdServer.ArgoClientWrapperServiceImpl)),
+
+		pipeline.NewPluginInputVariableParserImpl,
+		wire.Bind(new(pipeline.PluginInputVariableParser), new(*pipeline.PluginInputVariableParserImpl)),
 	)
 	return &App{}, nil
 }

api/bean/CustomTag.go

@@ -6,6 +6,7 @@ type CustomTag struct {
 	TagPattern           string `json:"tagPattern"`
 	AutoIncreasingNumber int    `json:"counterX"`
 	Metadata             string `json:"metadata"`
+	Enabled              bool   `json:"enabled"`
 }
 
 type CustomTagErrorResponse struct {

api/bean/ValuesOverrideRequest.go

@@ -116,4 +116,8 @@ type ArtifactsListFilterOptions struct {
 
 	//excludeWfRunners
 	ExcludeWfrIds []int
+
+
+	//pluginStage
+	PluginStage string
 }

api/restHandler/PipelineTriggerRestHandler.go

@@ -133,7 +133,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit
 	span.End()
 	if err != nil {
 		handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest)
-		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
+		common.WriteJsonResp(w, err, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	res := map[string]interface{}{"releaseId": mergeResp}

api/restHandler/app/BuildPipelineRestHandler.go

@@ -596,7 +596,7 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr
 	resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest)
 	if errors.Is(err, bean1.ErrImagePathInUse) {
 		handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest)
-		common.WriteJsonResp(w, err, response, http.StatusConflict)
+		common.WriteJsonResp(w, err, err, http.StatusConflict)
 		return
 	}
 	if err != nil {
@@ -1976,27 +1976,49 @@ func (handler PipelineConfigRestHandlerImpl) extractCipipelineMetaForImageTags(a
 	externalCi = false
 	ciPipelineId = 0
 	appId = 0
-
-	ciPipeline, err := handler.ciPipelineRepository.GetCiPipelineByArtifactId(artifactId)
-	var externalCiPipeline *pipelineConfig.ExternalCiPipeline
+	ciArtifact, err := handler.ciArtifactRepository.Get(artifactId)
 	if err != nil {
-		if err == pg.ErrNoRows {
-			handler.Logger.Infow("no ciPipeline found by artifact Id, fetching external ci-pipeline ", "artifactId", artifactId)
-			externalCiPipeline, err = handler.ciPipelineRepository.GetExternalCiPipelineByArtifactId(artifactId)
-		}
+		handler.Logger.Errorw("Error in fetching ci artifact by ci artifact id", "err", err)
+		return externalCi, ciPipelineId, appId, err
+	}
+	if ciArtifact.DataSource == repository.POST_CI {
+		ciPipelineId = ciArtifact.ComponentId
+		ciPipeline, err := handler.pipelineBuilder.GetCiPipelineById(ciPipelineId)
 		if err != nil {
-			handler.Logger.Errorw("error occurred in fetching ciPipeline/externalCiPipeline by artifact Id ", "err", err, "artifactId", artifactId)
+			handler.Logger.Errorw("no ci pipeline found for given artifact", "err", err, "artifactId", artifactId, "ciPipelineId", ciPipelineId)
 			return externalCi, ciPipelineId, appId, err
 		}
-	}
-
-	if ciPipeline.Id != 0 {
-		ciPipelineId = ciPipeline.Id
 		appId = ciPipeline.AppId
+	} else if ciArtifact.DataSource == repository.PRE_CD || ciArtifact.DataSource == repository.POST_CD {
+		cdPipelineId := ciArtifact.ComponentId
+		cdPipeline, err := handler.pipelineBuilder.GetCdPipelineById(cdPipelineId)
+		if err != nil {
+			handler.Logger.Errorw("no cd pipeline found for given artifact", "err", err, "artifactId", artifactId, "cdPipelineId", cdPipelineId)
+			return externalCi, ciPipelineId, appId, err
+		}
+		ciPipelineId = cdPipeline.CiPipelineId
+		appId = cdPipeline.AppId
 	} else {
-		externalCi = true
-		ciPipelineId = externalCiPipeline.Id
-		appId = externalCiPipeline.AppId
+		ciPipeline, err := handler.ciPipelineRepository.GetCiPipelineByArtifactId(artifactId)
+		var externalCiPipeline *pipelineConfig.ExternalCiPipeline
+		if err != nil {
+			if err == pg.ErrNoRows {
+				handler.Logger.Infow("no ciPipeline found by artifact Id, fetching external ci-pipeline ", "artifactId", artifactId)
+				externalCiPipeline, err = handler.ciPipelineRepository.GetExternalCiPipelineByArtifactId(artifactId)
+			}
+			if err != nil {
+				handler.Logger.Errorw("error occurred in fetching ciPipeline/externalCiPipeline by artifact Id ", "err", err, "artifactId", artifactId)
+				return externalCi, ciPipelineId, appId, err
+			}
+		}
+		if ciPipeline.Id != 0 {
+			ciPipelineId = ciPipeline.Id
+			appId = ciPipeline.AppId
+		} else {
+			externalCi = true
+			ciPipelineId = externalCiPipeline.Id
+			appId = externalCiPipeline.AppId
+		}
 	}
 	return externalCi, ciPipelineId, appId, nil
 }

api/restHandler/app/PipelineConfigRestHandler.go

@@ -130,6 +130,7 @@ type PipelineConfigRestHandlerImpl struct {
 	imageTaggingService          pipeline.ImageTaggingService
 	deploymentTemplateService    generateManifest.DeploymentTemplateService
 	pipelineRestHandlerEnvConfig *PipelineRestHandlerEnvConfig
+	ciArtifactRepository         repository.CiArtifactRepository
 }
 
 func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger,
@@ -153,7 +154,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger
 	materialRepository pipelineConfig.MaterialRepository, policyService security2.PolicyService,
 	scanResultRepository security.ImageScanResultRepository, gitProviderRepo repository.GitProviderRepository,
 	argoUserService argo.ArgoUserService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository,
-	imageTaggingService pipeline.ImageTaggingService) *PipelineConfigRestHandlerImpl {
+	imageTaggingService pipeline.ImageTaggingService,
+	ciArtifactRepository repository.CiArtifactRepository) *PipelineConfigRestHandlerImpl {
 	envConfig := &PipelineRestHandlerEnvConfig{}
 	err := env.Parse(envConfig)
 	if err != nil {
@@ -190,6 +192,7 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger
 		imageTaggingService:          imageTaggingService,
 		deploymentTemplateService:    deploymentTemplateService,
 		pipelineRestHandlerEnvConfig: envConfig,
+		ciArtifactRepository:         ciArtifactRepository,
 	}
 }
 

api/router/ApplicationRouter.go

@@ -79,7 +79,8 @@ func (r ApplicationRouterImpl) initApplicationRouter(router *mux.Router) {
 	router.Path("/{applicationName}/managed-resources").
 		Methods("GET").
 		HandlerFunc(r.handler.ManagedResources)
-	router.Path("/{name}/rollback").
+	router.Path("/{name}" +
+		"/rollback").
 		Methods("GET").
 		HandlerFunc(r.handler.Rollback)
 

api/router/pubsub/CiEventHandler.go

@@ -61,20 +61,22 @@ type ImageDetailsFromCR struct {
 }
 
 type CiCompleteEvent struct {
-	CiProjectDetails   []bean2.CiProjectDetails `json:"ciProjectDetails"`
-	DockerImage        string                   `json:"dockerImage" validate:"required,image-validator"`
-	Digest             string                   `json:"digest"`
-	PipelineId         int                      `json:"pipelineId"`
-	WorkflowId         *int                     `json:"workflowId"`
-	TriggeredBy        int32                    `json:"triggeredBy"`
-	PipelineName       string                   `json:"pipelineName"`
-	DataSource         string                   `json:"dataSource"`
-	MaterialType       string                   `json:"materialType"`
-	Metrics            util.CIMetrics           `json:"metrics"`
-	AppName            string                   `json:"appName"`
-	IsArtifactUploaded bool                     `json:"isArtifactUploaded"`
-	FailureReason      string                   `json:"failureReason"`
-	ImageDetailsFromCR *ImageDetailsFromCR      `json:"imageDetailsFromCR"`
+	CiProjectDetails              []bean2.CiProjectDetails `json:"ciProjectDetails"`
+	DockerImage                   string                   `json:"dockerImage" validate:"required,image-validator"`
+	Digest                        string                   `json:"digest"`
+	PipelineId                    int                      `json:"pipelineId"`
+	WorkflowId                    *int                     `json:"workflowId"`
+	TriggeredBy                   int32                    `json:"triggeredBy"`
+	PipelineName                  string                   `json:"pipelineName"`
+	DataSource                    string                   `json:"dataSource"`
+	MaterialType                  string                   `json:"materialType"`
+	Metrics                       util.CIMetrics           `json:"metrics"`
+	AppName                       string                   `json:"appName"`
+	IsArtifactUploaded            bool                     `json:"isArtifactUploaded"`
+	FailureReason                 string                   `json:"failureReason"`
+	ImageDetailsFromCR            *ImageDetailsFromCR      `json:"imageDetailsFromCR"`
+	PluginRegistryArtifactDetails map[string][]string      `json:"PluginRegistryArtifactDetails"`
+	PluginArtifactStage           string                   `json:"pluginArtifactStage"`
 }
 
 func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl {
@@ -214,14 +216,16 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (*
 	}
 
 	request := &pipeline.CiArtifactWebhookRequest{
-		Image:              event.DockerImage,
-		ImageDigest:        event.Digest,
-		DataSource:         event.DataSource,
-		PipelineName:       event.PipelineName,
-		MaterialInfo:       rawMaterialInfo,
-		UserId:             event.TriggeredBy,
-		WorkflowId:         event.WorkflowId,
-		IsArtifactUploaded: event.IsArtifactUploaded,
+		Image:                         event.DockerImage,
+		ImageDigest:                   event.Digest,
+		DataSource:                    event.DataSource,
+		PipelineName:                  event.PipelineName,
+		MaterialInfo:                  rawMaterialInfo,
+		UserId:                        event.TriggeredBy,
+		WorkflowId:                    event.WorkflowId,
+		IsArtifactUploaded:            event.IsArtifactUploaded,
+		PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails,
+		PluginArtifactStage:           event.PluginArtifactStage,
 	}
 	return request, nil
 }

api/router/pubsub/WorkflowStatusUpdateHandler.go

@@ -125,6 +125,14 @@ func (impl *WorkflowStatusUpdateHandlerImpl) SubscribeCD() error {
 			impl.logger.Errorw("could not get wf runner", "err", err)
 			return
 		}
+		if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) {
+			if len(wfr.ImagePathReservationIds) > 0 {
+				err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds)
+				if err != nil {
+					impl.logger.Errorw("error in removing image path reservation ")
+				}
+			}
+		}
 		if wfrStatus == string(v1alpha1.NodeSucceeded) || wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) {
 			eventType := util.EventType(0)
 			if wfrStatus == string(v1alpha1.NodeSucceeded) {

internal/sql/repository/CiArtifactRepository.go

@@ -31,6 +31,22 @@ import (
 	"go.uber.org/zap"
 )
 
+type credentialsSource = string
+type artifactsSourceType = string
+
+const (
+	GLOBAL_CONTAINER_REGISTRY credentialsSource = "global_container_registry"
+)
+const (
+	CI_RUNNER artifactsSourceType = "CI-RUNNER"
+	WEBHOOK   artifactsSourceType = "EXTERNAL"
+	PRE_CD    artifactsSourceType = "pre_cd"
+	POST_CD   artifactsSourceType = "post_cd"
+	PRE_CI    artifactsSourceType = "pre_ci"
+	POST_CI   artifactsSourceType = "post_ci"
+	GOCD      artifactsSourceType = "GOCD"
+)
+
 type CiArtifactWithExtraData struct {
 	CiArtifact
 	PayloadSchema      string
@@ -41,23 +57,26 @@ type CiArtifactWithExtraData struct {
 }
 
 type CiArtifact struct {
-	tableName            struct{}  `sql:"ci_artifact" pg:",discard_unknown_columns"`
-	Id                   int       `sql:"id,pk"`
-	PipelineId           int       `sql:"pipeline_id"` //id of the ci pipeline from which this webhook was triggered
-	Image                string    `sql:"image,notnull"`
-	ImageDigest          string    `sql:"image_digest,notnull"`
-	MaterialInfo         string    `sql:"material_info"` //git material metadata json array string
-	DataSource           string    `sql:"data_source,notnull"`
-	WorkflowId           *int      `sql:"ci_workflow_id"`
-	ParentCiArtifact     int       `sql:"parent_ci_artifact"`
-	ScanEnabled          bool      `sql:"scan_enabled,notnull"`
-	Scanned              bool      `sql:"scanned,notnull"`
-	ExternalCiPipelineId int       `sql:"external_ci_pipeline_id"`
-	IsArtifactUploaded   bool      `sql:"is_artifact_uploaded"`
-	DeployedTime         time.Time `sql:"-"`
-	Deployed             bool      `sql:"-"`
-	Latest               bool      `sql:"-"`
-	RunningOnParent      bool      `sql:"-"`
+	tableName             struct{}  `sql:"ci_artifact" pg:",discard_unknown_columns"`
+	Id                    int       `sql:"id,pk"`
+	PipelineId            int       `sql:"pipeline_id"` //id of the ci pipeline from which this webhook was triggered
+	Image                 string    `sql:"image,notnull"`
+	ImageDigest           string    `sql:"image_digest,notnull"`
+	MaterialInfo          string    `sql:"material_info"`       //git material metadata json array string
+	DataSource            string    `sql:"data_source,notnull"` // possible values -> (CI_RUNNER,ext,post_ci,pre_cd,post_cd) CI_runner is for normal build ci
+	WorkflowId            *int      `sql:"ci_workflow_id"`
+	ParentCiArtifact      int       `sql:"parent_ci_artifact"`
+	ScanEnabled           bool      `sql:"scan_enabled,notnull"`
+	Scanned               bool      `sql:"scanned,notnull"`
+	ExternalCiPipelineId  int       `sql:"external_ci_pipeline_id"`
+	IsArtifactUploaded    bool      `sql:"is_artifact_uploaded"`
+	CredentialsSourceType string    `sql:"credentials_source_type"`
+	CredentialSourceValue string    `sql:"credentials_source_value"`
+	ComponentId           int       `sql:"component_id"`
+	DeployedTime          time.Time `sql:"-"`
+	Deployed              bool      `sql:"-"`
+	Latest                bool      `sql:"-"`
+	RunningOnParent       bool      `sql:"-"`
 	sql.AuditLog
 }
 
@@ -83,6 +102,8 @@ type CiArtifactRepository interface {
 	GetArtifactsByParentCiWorkflowId(parentCiWorkflowId int) ([]string, error)
 	FetchArtifactsByCdPipelineIdV2(listingFilterOptions bean.ArtifactsListFilterOptions) ([]CiArtifactWithExtraData, int, error)
 	FindArtifactByListFilter(listingFilterOptions *bean.ArtifactsListFilterOptions) ([]CiArtifact, int, error)
+	GetArtifactsByDataSourceAndComponentId(dataSource string, componentId int) ([]CiArtifact, error)
+	FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error)
 }
 
 type CiArtifactRepositoryImpl struct {
@@ -128,7 +149,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactParentCiAndWorkflowDetailsByIds(
 	}
 
 	err := impl.dbConnection.Model(&artifacts).
-		Column("ci_artifact.ci_workflow_id", "ci_artifact.parent_ci_artifact", "ci_artifact.external_ci_pipeline_id", "ci_artifact.id").
+		Column("ci_artifact.ci_workflow_id", "ci_artifact.parent_ci_artifact", "ci_artifact.external_ci_pipeline_id", "ci_artifact.id", "ci_artifact.pipeline_id").
 		Where("ci_artifact.id in (?)", pg.In(ids)).
 		Select()
 
@@ -554,7 +575,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV2(cdPipelineId int
 }
 
 func GetCiMaterialInfo(materialInfo string, source string) ([]CiMaterialInfo, error) {
-	if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" {
+	if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" && source != "post_ci" && source != "pre_cd" && source != "post_cd" {
 		return nil, fmt.Errorf("datasource: %s not supported", source)
 	}
 	var ciMaterials []CiMaterialInfo
@@ -697,3 +718,29 @@ func (impl CiArtifactRepositoryImpl) FetchArtifactsByCdPipelineIdV2(listingFilte
 	}
 	return wfrList, totalCount, nil
 }
+
+func (impl CiArtifactRepositoryImpl) GetArtifactsByDataSourceAndComponentId(dataSource string, componentId int) ([]CiArtifact, error) {
+	var ciArtifacts []CiArtifact
+	err := impl.dbConnection.
+		Model(&ciArtifacts).
+		Where(" data_source=? and component_id=? ", dataSource, componentId).
+		Select()
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("error in getting ci artifacts by data_source and component_id")
+		return ciArtifacts, err
+	}
+	return ciArtifacts, nil
+}
+
+func (impl CiArtifactRepositoryImpl) FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error) {
+	var ciArtifacts []CiArtifact
+	err := impl.dbConnection.
+		Model(&ciArtifacts).
+		Where(" image in (?) ", pg.In(images)).
+		Select()
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("error in getting ci artifacts by data_source and component_id")
+		return ciArtifacts, err
+	}
+	return ciArtifacts, nil
+}