diff --git a/Wire.go b/Wire.go index 7fc361014e..57cdb92462 100644 --- a/Wire.go +++ b/Wire.go @@ -964,6 +964,9 @@ func InitializeApp() (*App, error) { argocdServer.NewArgoClientWrapperServiceImpl, wire.Bind(new(argocdServer.ArgoClientWrapperService), new(*argocdServer.ArgoClientWrapperServiceImpl)), + + pipeline.NewPluginInputVariableParserImpl, + wire.Bind(new(pipeline.PluginInputVariableParser), new(*pipeline.PluginInputVariableParserImpl)), ) return &App{}, nil } diff --git a/api/bean/CustomTag.go b/api/bean/CustomTag.go index 8d398be5be..7c39ff4175 100644 --- a/api/bean/CustomTag.go +++ b/api/bean/CustomTag.go @@ -6,6 +6,7 @@ type CustomTag struct { TagPattern string `json:"tagPattern"` AutoIncreasingNumber int `json:"counterX"` Metadata string `json:"metadata"` + Enabled bool `json:"enabled"` } type CustomTagErrorResponse struct { diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index 100cc49b28..0e4615ffa7 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -116,4 +116,8 @@ type ArtifactsListFilterOptions struct { //excludeWfRunners ExcludeWfrIds []int + + + //pluginStage + PluginStage string } diff --git a/api/restHandler/PipelineTriggerRestHandler.go b/api/restHandler/PipelineTriggerRestHandler.go index bd6235c478..bf34401970 100644 --- a/api/restHandler/PipelineTriggerRestHandler.go +++ b/api/restHandler/PipelineTriggerRestHandler.go @@ -133,7 +133,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + common.WriteJsonResp(w, err, err.Error(), http.StatusInternalServerError) return } res := map[string]interface{}{"releaseId": mergeResp} diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 225fef0dcc..78eda463e6 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -596,7 +596,7 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) - common.WriteJsonResp(w, err, response, http.StatusConflict) + common.WriteJsonResp(w, err, err, http.StatusConflict) return } if err != nil { @@ -1976,27 +1976,49 @@ func (handler PipelineConfigRestHandlerImpl) extractCipipelineMetaForImageTags(a externalCi = false ciPipelineId = 0 appId = 0 - - ciPipeline, err := handler.ciPipelineRepository.GetCiPipelineByArtifactId(artifactId) - var externalCiPipeline *pipelineConfig.ExternalCiPipeline + ciArtifact, err := handler.ciArtifactRepository.Get(artifactId) if err != nil { - if err == pg.ErrNoRows { - handler.Logger.Infow("no ciPipeline found by artifact Id, fetching external ci-pipeline ", "artifactId", artifactId) - externalCiPipeline, err = handler.ciPipelineRepository.GetExternalCiPipelineByArtifactId(artifactId) - } + handler.Logger.Errorw("Error in fetching ci artifact by ci artifact id", "err", err) + return externalCi, ciPipelineId, appId, err + } + if ciArtifact.DataSource == repository.POST_CI { + ciPipelineId = ciArtifact.ComponentId + ciPipeline, err := handler.pipelineBuilder.GetCiPipelineById(ciPipelineId) if err != nil { - handler.Logger.Errorw("error occurred in fetching ciPipeline/externalCiPipeline by artifact Id ", "err", err, "artifactId", artifactId) + handler.Logger.Errorw("no ci pipeline found for given artifact", "err", err, "artifactId", artifactId, "ciPipelineId", ciPipelineId) return externalCi, ciPipelineId, appId, err } - } - - if ciPipeline.Id != 0 { - ciPipelineId = ciPipeline.Id appId = ciPipeline.AppId + } else if ciArtifact.DataSource == repository.PRE_CD || ciArtifact.DataSource == repository.POST_CD { + cdPipelineId := ciArtifact.ComponentId + cdPipeline, err := handler.pipelineBuilder.GetCdPipelineById(cdPipelineId) + if err != nil { + handler.Logger.Errorw("no cd pipeline found for given artifact", "err", err, "artifactId", artifactId, "cdPipelineId", cdPipelineId) + return externalCi, ciPipelineId, appId, err + } + ciPipelineId = cdPipeline.CiPipelineId + appId = cdPipeline.AppId } else { - externalCi = true - ciPipelineId = externalCiPipeline.Id - appId = externalCiPipeline.AppId + ciPipeline, err := handler.ciPipelineRepository.GetCiPipelineByArtifactId(artifactId) + var externalCiPipeline *pipelineConfig.ExternalCiPipeline + if err != nil { + if err == pg.ErrNoRows { + handler.Logger.Infow("no ciPipeline found by artifact Id, fetching external ci-pipeline ", "artifactId", artifactId) + externalCiPipeline, err = handler.ciPipelineRepository.GetExternalCiPipelineByArtifactId(artifactId) + } + if err != nil { + handler.Logger.Errorw("error occurred in fetching ciPipeline/externalCiPipeline by artifact Id ", "err", err, "artifactId", artifactId) + return externalCi, ciPipelineId, appId, err + } + } + if ciPipeline.Id != 0 { + ciPipelineId = ciPipeline.Id + appId = ciPipeline.AppId + } else { + externalCi = true + ciPipelineId = externalCiPipeline.Id + appId = externalCiPipeline.AppId + } } return externalCi, ciPipelineId, appId, nil } diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 693bafedac..e020ceb95f 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -130,6 +130,7 @@ type PipelineConfigRestHandlerImpl struct { imageTaggingService pipeline.ImageTaggingService deploymentTemplateService generateManifest.DeploymentTemplateService pipelineRestHandlerEnvConfig *PipelineRestHandlerEnvConfig + ciArtifactRepository repository.CiArtifactRepository } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -153,7 +154,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger materialRepository pipelineConfig.MaterialRepository, policyService security2.PolicyService, scanResultRepository security.ImageScanResultRepository, gitProviderRepo repository.GitProviderRepository, argoUserService argo.ArgoUserService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - imageTaggingService pipeline.ImageTaggingService) *PipelineConfigRestHandlerImpl { + imageTaggingService pipeline.ImageTaggingService, + ciArtifactRepository repository.CiArtifactRepository) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) if err != nil { @@ -190,6 +192,7 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger imageTaggingService: imageTaggingService, deploymentTemplateService: deploymentTemplateService, pipelineRestHandlerEnvConfig: envConfig, + ciArtifactRepository: ciArtifactRepository, } } diff --git a/api/router/ApplicationRouter.go b/api/router/ApplicationRouter.go index 789a92c675..729f7f5242 100644 --- a/api/router/ApplicationRouter.go +++ b/api/router/ApplicationRouter.go @@ -79,7 +79,8 @@ func (r ApplicationRouterImpl) initApplicationRouter(router *mux.Router) { router.Path("/{applicationName}/managed-resources"). Methods("GET"). HandlerFunc(r.handler.ManagedResources) - router.Path("/{name}/rollback"). + router.Path("/{name}" + + "/rollback"). Methods("GET"). HandlerFunc(r.handler.Rollback) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index cf0e1fa030..1766d9be09 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -61,20 +61,22 @@ type ImageDetailsFromCR struct { } type CiCompleteEvent struct { - CiProjectDetails []bean2.CiProjectDetails `json:"ciProjectDetails"` - DockerImage string `json:"dockerImage" validate:"required,image-validator"` - Digest string `json:"digest"` - PipelineId int `json:"pipelineId"` - WorkflowId *int `json:"workflowId"` - TriggeredBy int32 `json:"triggeredBy"` - PipelineName string `json:"pipelineName"` - DataSource string `json:"dataSource"` - MaterialType string `json:"materialType"` - Metrics util.CIMetrics `json:"metrics"` - AppName string `json:"appName"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - FailureReason string `json:"failureReason"` - ImageDetailsFromCR *ImageDetailsFromCR `json:"imageDetailsFromCR"` + CiProjectDetails []bean2.CiProjectDetails `json:"ciProjectDetails"` + DockerImage string `json:"dockerImage" validate:"required,image-validator"` + Digest string `json:"digest"` + PipelineId int `json:"pipelineId"` + WorkflowId *int `json:"workflowId"` + TriggeredBy int32 `json:"triggeredBy"` + PipelineName string `json:"pipelineName"` + DataSource string `json:"dataSource"` + MaterialType string `json:"materialType"` + Metrics util.CIMetrics `json:"metrics"` + AppName string `json:"appName"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + FailureReason string `json:"failureReason"` + ImageDetailsFromCR *ImageDetailsFromCR `json:"imageDetailsFromCR"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` + PluginArtifactStage string `json:"pluginArtifactStage"` } func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl { @@ -214,14 +216,16 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* } request := &pipeline.CiArtifactWebhookRequest{ - Image: event.DockerImage, - ImageDigest: event.Digest, - DataSource: event.DataSource, - PipelineName: event.PipelineName, - MaterialInfo: rawMaterialInfo, - UserId: event.TriggeredBy, - WorkflowId: event.WorkflowId, - IsArtifactUploaded: event.IsArtifactUploaded, + Image: event.DockerImage, + ImageDigest: event.Digest, + DataSource: event.DataSource, + PipelineName: event.PipelineName, + MaterialInfo: rawMaterialInfo, + UserId: event.TriggeredBy, + WorkflowId: event.WorkflowId, + IsArtifactUploaded: event.IsArtifactUploaded, + PluginRegistryArtifactDetails: event.PluginRegistryArtifactDetails, + PluginArtifactStage: event.PluginArtifactStage, } return request, nil } diff --git a/api/router/pubsub/WorkflowStatusUpdateHandler.go b/api/router/pubsub/WorkflowStatusUpdateHandler.go index d2a9f9ffe1..12a587cf57 100644 --- a/api/router/pubsub/WorkflowStatusUpdateHandler.go +++ b/api/router/pubsub/WorkflowStatusUpdateHandler.go @@ -125,6 +125,14 @@ func (impl *WorkflowStatusUpdateHandlerImpl) SubscribeCD() error { impl.logger.Errorw("could not get wf runner", "err", err) return } + if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + if len(wfr.ImagePathReservationIds) > 0 { + err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("error in removing image path reservation ") + } + } + } if wfrStatus == string(v1alpha1.NodeSucceeded) || wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { eventType := util.EventType(0) if wfrStatus == string(v1alpha1.NodeSucceeded) { diff --git a/assets/ic-plugin-copy-container-image.png b/assets/ic-plugin-copy-container-image.png new file mode 100644 index 0000000000..1b5a3c47e5 Binary files /dev/null and b/assets/ic-plugin-copy-container-image.png differ diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index d670f5ee29..5a1318757f 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -31,6 +31,22 @@ import ( "go.uber.org/zap" ) +type credentialsSource = string +type artifactsSourceType = string + +const ( + GLOBAL_CONTAINER_REGISTRY credentialsSource = "global_container_registry" +) +const ( + CI_RUNNER artifactsSourceType = "CI-RUNNER" + WEBHOOK artifactsSourceType = "EXTERNAL" + PRE_CD artifactsSourceType = "pre_cd" + POST_CD artifactsSourceType = "post_cd" + PRE_CI artifactsSourceType = "pre_ci" + POST_CI artifactsSourceType = "post_ci" + GOCD artifactsSourceType = "GOCD" +) + type CiArtifactWithExtraData struct { CiArtifact PayloadSchema string @@ -41,23 +57,26 @@ type CiArtifactWithExtraData struct { } type CiArtifact struct { - tableName struct{} `sql:"ci_artifact" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - PipelineId int `sql:"pipeline_id"` //id of the ci pipeline from which this webhook was triggered - Image string `sql:"image,notnull"` - ImageDigest string `sql:"image_digest,notnull"` - MaterialInfo string `sql:"material_info"` //git material metadata json array string - DataSource string `sql:"data_source,notnull"` - WorkflowId *int `sql:"ci_workflow_id"` - ParentCiArtifact int `sql:"parent_ci_artifact"` - ScanEnabled bool `sql:"scan_enabled,notnull"` - Scanned bool `sql:"scanned,notnull"` - ExternalCiPipelineId int `sql:"external_ci_pipeline_id"` - IsArtifactUploaded bool `sql:"is_artifact_uploaded"` - DeployedTime time.Time `sql:"-"` - Deployed bool `sql:"-"` - Latest bool `sql:"-"` - RunningOnParent bool `sql:"-"` + tableName struct{} `sql:"ci_artifact" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + PipelineId int `sql:"pipeline_id"` //id of the ci pipeline from which this webhook was triggered + Image string `sql:"image,notnull"` + ImageDigest string `sql:"image_digest,notnull"` + MaterialInfo string `sql:"material_info"` //git material metadata json array string + DataSource string `sql:"data_source,notnull"` // possible values -> (CI_RUNNER,ext,post_ci,pre_cd,post_cd) CI_runner is for normal build ci + WorkflowId *int `sql:"ci_workflow_id"` + ParentCiArtifact int `sql:"parent_ci_artifact"` + ScanEnabled bool `sql:"scan_enabled,notnull"` + Scanned bool `sql:"scanned,notnull"` + ExternalCiPipelineId int `sql:"external_ci_pipeline_id"` + IsArtifactUploaded bool `sql:"is_artifact_uploaded"` + CredentialsSourceType string `sql:"credentials_source_type"` + CredentialSourceValue string `sql:"credentials_source_value"` + ComponentId int `sql:"component_id"` + DeployedTime time.Time `sql:"-"` + Deployed bool `sql:"-"` + Latest bool `sql:"-"` + RunningOnParent bool `sql:"-"` sql.AuditLog } @@ -83,6 +102,8 @@ type CiArtifactRepository interface { GetArtifactsByParentCiWorkflowId(parentCiWorkflowId int) ([]string, error) FetchArtifactsByCdPipelineIdV2(listingFilterOptions bean.ArtifactsListFilterOptions) ([]CiArtifactWithExtraData, int, error) FindArtifactByListFilter(listingFilterOptions *bean.ArtifactsListFilterOptions) ([]CiArtifact, int, error) + GetArtifactsByDataSourceAndComponentId(dataSource string, componentId int) ([]CiArtifact, error) + FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error) } type CiArtifactRepositoryImpl struct { @@ -128,7 +149,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactParentCiAndWorkflowDetailsByIds( } err := impl.dbConnection.Model(&artifacts). - Column("ci_artifact.ci_workflow_id", "ci_artifact.parent_ci_artifact", "ci_artifact.external_ci_pipeline_id", "ci_artifact.id"). + Column("ci_artifact.ci_workflow_id", "ci_artifact.parent_ci_artifact", "ci_artifact.external_ci_pipeline_id", "ci_artifact.id", "ci_artifact.pipeline_id"). Where("ci_artifact.id in (?)", pg.In(ids)). Select() @@ -554,7 +575,7 @@ func (impl CiArtifactRepositoryImpl) GetArtifactsByCDPipelineV2(cdPipelineId int } func GetCiMaterialInfo(materialInfo string, source string) ([]CiMaterialInfo, error) { - if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" { + if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" && source != "post_ci" && source != "pre_cd" && source != "post_cd" { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []CiMaterialInfo @@ -697,3 +718,29 @@ func (impl CiArtifactRepositoryImpl) FetchArtifactsByCdPipelineIdV2(listingFilte } return wfrList, totalCount, nil } + +func (impl CiArtifactRepositoryImpl) GetArtifactsByDataSourceAndComponentId(dataSource string, componentId int) ([]CiArtifact, error) { + var ciArtifacts []CiArtifact + err := impl.dbConnection. + Model(&ciArtifacts). + Where(" data_source=? and component_id=? ", dataSource, componentId). + Select() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting ci artifacts by data_source and component_id") + return ciArtifacts, err + } + return ciArtifacts, nil +} + +func (impl CiArtifactRepositoryImpl) FindCiArtifactByImagePaths(images []string) ([]CiArtifact, error) { + var ciArtifacts []CiArtifact + err := impl.dbConnection. + Model(&ciArtifacts). + Where(" image in (?) ", pg.In(images)). + Select() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting ci artifacts by data_source and component_id") + return ciArtifacts, err + } + return ciArtifacts, nil +} diff --git a/internal/sql/repository/CiArtifactsListingQueryBuilder.go b/internal/sql/repository/CiArtifactsListingQueryBuilder.go index cddec86121..a7ff12332c 100644 --- a/internal/sql/repository/CiArtifactsListingQueryBuilder.go +++ b/internal/sql/repository/CiArtifactsListingQueryBuilder.go @@ -9,17 +9,14 @@ import ( const EmptyLikeRegex = "%%" func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilterOptions) string { - commonPaginatedQueryPart := "" - //if listingFilterOpts.SearchString != EmptyLikeRegex { - commonPaginatedQueryPart = fmt.Sprintf(" cia.image LIKE '%v' ", listingFilterOpts.SearchString) - //} + commonPaginatedQueryPart := fmt.Sprintf(" cia.image LIKE '%v'", listingFilterOpts.SearchString) orderByClause := " ORDER BY cia.id DESC" limitOffsetQueryPart := fmt.Sprintf(" LIMIT %v OFFSET %v", listingFilterOpts.Limit, listingFilterOpts.Offset) finalQuery := "" if listingFilterOpts.ParentStageType == bean.CI_WORKFLOW_TYPE { selectQuery := " SELECT cia.* " remainingQuery := " FROM ci_artifact cia" + - " INNER JOIN ci_pipeline cp ON cp.id=cia.pipeline_id" + + " INNER JOIN ci_pipeline cp ON (cp.id=cia.pipeline_id or (cp.id=cia.component_id and cia.data_source='post_ci' ) )" + " INNER JOIN pipeline p ON p.ci_pipeline_id = cp.id and p.id=%v" + " WHERE " remainingQuery = fmt.Sprintf(remainingQuery, listingFilterOpts.PipelineId) @@ -50,27 +47,28 @@ func BuildQueryForParentTypeCIOrWebhook(listingFilterOpts bean.ArtifactsListFilt } func BuildQueryForArtifactsForCdStage(listingFilterOptions bean.ArtifactsListFilterOptions) string { - commonQuery := " FROM cd_workflow_runner " + - " INNER JOIN cd_workflow ON cd_workflow.id=cd_workflow_runner.cd_workflow_id " + - " INNER JOIN ci_artifact cia ON cia.id = cd_workflow.ci_artifact_id " + - " WHERE ((cd_workflow.pipeline_id = %v AND cd_workflow_runner.workflow_type = '%v') " + - " OR (cd_workflow.pipeline_id = %v AND cd_workflow_runner.workflow_type = '%v' AND cd_workflow_runner.status IN ('Healthy','Succeeded'))) " + // expected result -> will fetch all successfully deployed artifacts ar parent stage plus its own stage. Along with this it will + // also fetch all artifacts generated by plugin at pre_cd or post_cd process (will use data_source in ci artifact table for this) - if listingFilterOptions.SearchString != EmptyLikeRegex { - commonQuery += " AND cia.image LIKE '%v' " - } + commonQuery := " from ci_artifact LEFT JOIN cd_workflow ON ci_artifact.id = cd_workflow.ci_artifact_id" + + " LEFT JOIN cd_workflow_runner ON cd_workflow_runner.cd_workflow_id=cd_workflow.id " + + " Where (((cd_workflow_runner.id in (select MAX(cd_workflow_runner.id) OVER (PARTITION BY cd_workflow.ci_artifact_id) FROM cd_workflow_runner inner join cd_workflow on cd_workflow.id=cd_workflow_runner.cd_workflow_id))" + + " AND ((cd_workflow.pipeline_id= %v and cd_workflow_runner.workflow_type = '%v' ) OR (cd_workflow.pipeline_id = %v AND cd_workflow_runner.workflow_type = '%v' AND cd_workflow_runner.status IN ('Healthy','Succeeded') )))" + + " OR (ci_artifact.component_id = %v and ci_artifact.data_source= '%v' ))" + + " AND (ci_artifact.image LIKE '%v' )" - commonQuery = fmt.Sprintf(commonQuery, listingFilterOptions.PipelineId, listingFilterOptions.StageType, listingFilterOptions.ParentId, listingFilterOptions.ParentStageType, listingFilterOptions.SearchString) + commonQuery = fmt.Sprintf(commonQuery, listingFilterOptions.PipelineId, listingFilterOptions.StageType, listingFilterOptions.ParentId, listingFilterOptions.ParentStageType, listingFilterOptions.ParentId, listingFilterOptions.PluginStage, listingFilterOptions.SearchString) if len(listingFilterOptions.ExcludeArtifactIds) > 0 { - commonQuery = commonQuery + fmt.Sprintf(" AND cd_workflow.ci_artifact_id NOT IN (%v)", helper.GetCommaSepratedString(listingFilterOptions.ExcludeArtifactIds)) + commonQuery = commonQuery + fmt.Sprintf(" AND ( ci_artifact.id NOT IN (%v))", helper.GetCommaSepratedString(listingFilterOptions.ExcludeArtifactIds)) } - totalCountQuery := "SELECT COUNT(DISTINCT ci_artifact_id) as total_count " + commonQuery - selectQuery := fmt.Sprintf("SELECT cia.id , (%v) ", totalCountQuery) - GroupByQuery := " GROUP BY cia.id " - limitOffSetQuery := fmt.Sprintf(" LIMIT %v OFFSET %v", listingFilterOptions.Limit, listingFilterOptions.Offset) + totalCountQuery := "SELECT COUNT(DISTINCT ci_artifact.id) as total_count " + commonQuery + selectQuery := fmt.Sprintf("SELECT DISTINCT(ci_artifact.id) , (%v) ", totalCountQuery) + //GroupByQuery := " GROUP BY cia.id " + limitOffSetQuery := fmt.Sprintf(" order by ci_artifact.id desc LIMIT %v OFFSET %v", listingFilterOptions.Limit, listingFilterOptions.Offset) - finalQuery := selectQuery + commonQuery + GroupByQuery + limitOffSetQuery + //finalQuery := selectQuery + commonQuery + GroupByQuery + limitOffSetQuery + finalQuery := selectQuery + commonQuery + limitOffSetQuery return finalQuery } @@ -79,10 +77,11 @@ func BuildQueryForArtifactsForRollback(listingFilterOptions bean.ArtifactsListFi " INNER JOIN cd_workflow cdw ON cdw.id=cdwr.cd_workflow_id " + " INNER JOIN ci_artifact cia ON cia.id=cdw.ci_artifact_id " + " WHERE cdw.pipeline_id=%v AND cdwr.workflow_type = '%v' " + + commonQuery = fmt.Sprintf(commonQuery, listingFilterOptions.PipelineId, listingFilterOptions.StageType) if listingFilterOptions.SearchString != EmptyLikeRegex { - commonQuery += " AND cia.image LIKE '%v' " + commonQuery += fmt.Sprintf(" AND cia.image LIKE '%v' ", listingFilterOptions.SearchString) } - commonQuery = fmt.Sprintf(commonQuery, listingFilterOptions.PipelineId, listingFilterOptions.StageType, listingFilterOptions.SearchString) if len(listingFilterOptions.ExcludeWfrIds) > 0 { commonQuery = fmt.Sprintf(" %s AND cdwr.id NOT IN (%s)", commonQuery, helper.GetCommaSepratedString(listingFilterOptions.ExcludeWfrIds)) } diff --git a/internal/sql/repository/CustomTagRepository.go b/internal/sql/repository/CustomTagRepository.go index b075491db0..abe19f4740 100644 --- a/internal/sql/repository/CustomTagRepository.go +++ b/internal/sql/repository/CustomTagRepository.go @@ -14,6 +14,7 @@ type CustomTag struct { AutoIncreasingNumber int `sql:"auto_increasing_number, notnull"` Active bool `sql:"active"` Metadata string `sql:"metadata"` + Enabled bool `sql:"enabled, notnull"` } type ImagePathReservation struct { @@ -35,6 +36,9 @@ type ImageTagRepository interface { DeleteByEntityKeyAndValue(entityKey int, entityValue string) error DeactivateImagePathReservation(id int) error FetchActiveCustomTagData(entityKey int, entityValue string) (*CustomTag, error) + DeactivateImagePathReservationByImagePaths(tx *pg.Tx, imagePaths []string) error + DeactivateImagePathReservationByImagePathReservationIds(tx *pg.Tx, imagePathReservationIds []int) error + DisableCustomTag(entityKey int, entityValue string) error } type ImageTagRepositoryImpl struct { @@ -105,3 +109,27 @@ func (impl *ImageTagRepositoryImpl) FindByImagePath(tx *pg.Tx, path string) ([]* func (impl *ImageTagRepositoryImpl) InsertImagePath(tx *pg.Tx, reservation *ImagePathReservation) error { return tx.Insert(reservation) } + +func (impl *ImageTagRepositoryImpl) DeactivateImagePathReservationByImagePaths(tx *pg.Tx, imagePaths []string) error { + query := `UPDATE image_path_reservation set active=false where image_path in (?)` + _, err := tx.Exec(query, pg.In(imagePaths)) + if err != nil && err != pg.ErrNoRows { + return err + } + return nil +} + +func (impl *ImageTagRepositoryImpl) DeactivateImagePathReservationByImagePathReservationIds(tx *pg.Tx, imagePathReservationIds []int) error { + query := `UPDATE image_path_reservation set active=false where id in (?)` + _, err := tx.Exec(query, pg.In(imagePathReservationIds)) + if err != nil && err != pg.ErrNoRows { + return err + } + return nil +} + +func (impl *ImageTagRepositoryImpl) DisableCustomTag(entityKey int, entityValue string) error { + query := `update custom_tag set enabled = false where entity_key = ? and entity_value = ?` + _, err := impl.dbConnection.Exec(query, entityKey, entityValue) + return err +} diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index e115b34eb8..c98c749305 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -151,24 +151,25 @@ type CdWorkflowRunnerWithExtraFields struct { } type CdWorkflowRunner struct { - tableName struct{} `sql:"cd_workflow_runner" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - Name string `sql:"name"` - WorkflowType bean.WorkflowType `sql:"workflow_type"` //pre,post,deploy - ExecutorType WorkflowExecutorType `sql:"executor_type"` //awf, system - Status string `sql:"status"` - PodStatus string `sql:"pod_status"` - Message string `sql:"message"` - StartedOn time.Time `sql:"started_on"` - FinishedOn time.Time `sql:"finished_on"` - Namespace string `sql:"namespace"` - LogLocation string `sql:"log_file_path"` - TriggeredBy int32 `sql:"triggered_by"` - CdWorkflowId int `sql:"cd_workflow_id"` - PodName string `sql:"pod_name"` - BlobStorageEnabled bool `sql:"blob_storage_enabled,notnull"` - RefCdWorkflowRunnerId int `sql:"ref_cd_workflow_runner_id,notnull"` - CdWorkflow *CdWorkflow + tableName struct{} `sql:"cd_workflow_runner" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + Name string `sql:"name"` + WorkflowType bean.WorkflowType `sql:"workflow_type"` //pre,post,deploy + ExecutorType WorkflowExecutorType `sql:"executor_type"` //awf, system + Status string `sql:"status"` + PodStatus string `sql:"pod_status"` + Message string `sql:"message"` + StartedOn time.Time `sql:"started_on"` + FinishedOn time.Time `sql:"finished_on"` + Namespace string `sql:"namespace"` + LogLocation string `sql:"log_file_path"` + TriggeredBy int32 `sql:"triggered_by"` + CdWorkflowId int `sql:"cd_workflow_id"` + PodName string `sql:"pod_name"` + BlobStorageEnabled bool `sql:"blob_storage_enabled,notnull"` + RefCdWorkflowRunnerId int `sql:"ref_cd_workflow_runner_id,notnull"` + ImagePathReservationIds []int `sql:"image_path_reservation_ids" pg:",array,notnull"` + CdWorkflow *CdWorkflow sql.AuditLog } @@ -486,9 +487,15 @@ func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunnersWithTxn(wfrs []*CdWor return err } -func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunners(wfr []*CdWorkflowRunner) error { - _, err := impl.dbConnection.Model(&wfr).Update() - return err +func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunners(wfrs []*CdWorkflowRunner) error { + for _, wfr := range wfrs { + err := impl.dbConnection.Update(wfr) + if err != nil { + impl.logger.Errorw("error in updating wfr", "err", err) + return err + } + } + return nil } func (impl *CdWorkflowRepositoryImpl) FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error) { var wfr []*CdWorkflowRunner @@ -606,6 +613,36 @@ func (impl *CdWorkflowRepositoryImpl) FetchArtifactsByCdPipelineId(pipelineId in return wfrList, err } +func (impl *CdWorkflowRepositoryImpl) FetchArtifactsByCdPipelineIdV2(listingFilterOptions bean.ArtifactsListFilterOptions) ([]CdWorkflowRunner, int, error) { + var wfrList []CdWorkflowRunner + query := impl.dbConnection. + Model(&wfrList). + Column("cd_workflow_runner.*", "CdWorkflow", "CdWorkflow.Pipeline", "CdWorkflow.CiArtifact"). + Where("cd_workflow.pipeline_id = ?", listingFilterOptions.PipelineId). + Where("cd_workflow_runner.workflow_type = ?", listingFilterOptions.StageType). + Where("cd_workflow__ci_artifact.image LIKE ?", listingFilterOptions.SearchString) + + if len(listingFilterOptions.ExcludeArtifactIds) > 0 { + query = query.Where("cd_workflow__ci_artifact.id NOT IN (?)", pg.In(listingFilterOptions.ExcludeArtifactIds)) + } + totalCount, err := query.Count() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting Wfrs count and ci artifacts by pipelineId", "err", err, "pipelineId", listingFilterOptions.PipelineId) + return nil, totalCount, err + } + + query = query.Order("cd_workflow_runner.id DESC"). + Limit(listingFilterOptions.Limit). + Offset(listingFilterOptions.Offset) + + err = query.Select() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting Wfrs and ci artifacts by pipelineId", "err", err, "pipelineId", listingFilterOptions.PipelineId) + return nil, totalCount, err + } + return wfrList, totalCount, nil +} + func (impl *CdWorkflowRepositoryImpl) GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses(getPipelineDeployedWithinHours int) ([]*CdWorkflowRunner, error) { var wfrList []*CdWorkflowRunner err := impl.dbConnection. diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 1b91c202bf..d7a0ae473a 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -78,6 +78,7 @@ type CiWorkflow struct { ParentCiWorkFlowId int `sql:"parent_ci_workflow_id"` ExecutorType WorkflowExecutorType `sql:"executor_type"` //awf, system ImagePathReservationId int `sql:"image_path_reservation_id"` + ImagePathReservationIds []int `sql:"image_path_reservation_ids" pg:",array"` CiPipeline *CiPipeline } @@ -112,6 +113,7 @@ type WorkflowWithArtifact struct { ParentCiWorkflowId int `json:"parent_ci_workflow_id"` ExecutorType WorkflowExecutorType `json:"executor_type"` //awf, system ImagePathReservationId int `json:"image_path_reservation_id"` + ImagePathReservationIds []int `json:"image_path_reservation_ids" pg:",array"` } type GitCommit struct { diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 21c470f2ae..b0d62cb4fe 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -25,6 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" "time" ) @@ -121,6 +122,7 @@ type CiPipeline struct { LastTriggeredEnvId int `json:"lastTriggeredEnvId"` CustomTagObject *CustomTagData `json:"customTag,omitempty"` DefaultTag []string `json:"defaultTag,omitempty"` + EnableCustomTag bool `json:"enableCustomTag"` } type DockerConfigOverride struct { @@ -243,6 +245,7 @@ type CiMaterialPatchRequest struct { type CustomTagData struct { TagPattern string `json:"tagPattern"` CounterX int `json:"counterX"` + Enabled bool `json:"enabled"` } type CiMaterialValuePatchRequest struct { @@ -563,6 +566,9 @@ type CDPipelineConfigObject struct { SourceToNewPipelineId map[int]int `json:"sourceToNewPipelineId,omitempty"` RefPipelineId int `json:"refPipelineId,omitempty"` ExternalCiPipelineId int `json:"externalCiPipelineId,omitempty"` + CustomTagObject *CustomTagData `json:"customTag"` + CustomTagStage *repository.PipelineStageType `json:"customTagStage"` + EnableCustomTag bool `json:"enableCustomTag"` } type PreStageConfigMapSecretNames struct { @@ -733,6 +739,12 @@ type CiArtifactBean struct { ExternalCiPipelineId int `json:"-"` ParentCiArtifact int `json:"-"` CiWorkflowId int `json:"-"` + RegistryType string `json:"registryType"` + RegistryName string `json:"registryName"` + CiPipelineId int `json:"-"` + CredentialsSourceType string `json:"-"` + CredentialsSourceValue string `json:"-"` + } type CiArtifactResponse struct { diff --git a/pkg/dockerRegistry/DockerRegistryIpsConfigService.go b/pkg/dockerRegistry/DockerRegistryIpsConfigService.go index 564873b9ad..7e012c0cc3 100644 --- a/pkg/dockerRegistry/DockerRegistryIpsConfigService.go +++ b/pkg/dockerRegistry/DockerRegistryIpsConfigService.go @@ -20,6 +20,7 @@ package dockerRegistry import ( "encoding/json" "github.com/devtron-labs/common-lib/utils/k8s" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/cluster" @@ -34,7 +35,7 @@ import ( type DockerRegistryIpsConfigService interface { IsImagePullSecretAccessProvided(dockerRegistryId string, clusterId int, isVirtualEnv bool) (bool, error) - HandleImagePullSecretOnApplicationDeployment(environment *repository2.Environment, ciPipelineId int, valuesFileContent []byte) ([]byte, error) + HandleImagePullSecretOnApplicationDeployment(environment *repository2.Environment, artifact *repository3.CiArtifact, ciPipelineId int, valuesFileContent []byte) ([]byte, error) } type DockerRegistryIpsConfigServiceImpl struct { @@ -76,7 +77,7 @@ func (impl DockerRegistryIpsConfigServiceImpl) IsImagePullSecretAccessProvided(d return isAccessProvided, nil } -func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicationDeployment(environment *repository2.Environment, ciPipelineId int, valuesFileContent []byte) ([]byte, error) { +func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicationDeployment(environment *repository2.Environment, artifact *repository3.CiArtifact, ciPipelineId int, valuesFileContent []byte) ([]byte, error) { clusterId := environment.ClusterId impl.logger.Infow("handling ips if access given", "ciPipelineId", ciPipelineId, "clusterId", clusterId) @@ -85,7 +86,7 @@ func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicatio return valuesFileContent, nil } - dockerRegistryId, err := impl.getDockerRegistryIdForCiPipeline(ciPipelineId) + dockerRegistryId, err := impl.getDockerRegistryIdForCiPipeline(ciPipelineId, artifact) if err != nil { impl.logger.Errorw("error in getting docker registry", "dockerRegistryId", dockerRegistryId, "error", err) return valuesFileContent, err @@ -138,7 +139,7 @@ func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicatio return updatedValuesFileContent, nil } -func (impl DockerRegistryIpsConfigServiceImpl) getDockerRegistryIdForCiPipeline(ciPipelineId int) (*string, error) { +func (impl DockerRegistryIpsConfigServiceImpl) getDockerRegistryIdForCiPipeline(ciPipelineId int, artifact *repository3.CiArtifact) (*string, error) { ciPipeline, err := impl.ciPipelineRepository.FindById(ciPipelineId) if err != nil { impl.logger.Errorw("error in fetching ciPipeline", "ciPipelineId", ciPipelineId, "error", err) @@ -154,27 +155,35 @@ func (impl DockerRegistryIpsConfigServiceImpl) getDockerRegistryIdForCiPipeline( impl.logger.Warn("returning as ciPipeline.CiTemplate is found nil") return nil, nil } - - dockerRegistryId := ciPipeline.CiTemplate.DockerRegistryId - if dockerRegistryId != nil && len(*dockerRegistryId) == 0 { - impl.logger.Warn("returning as dockerRegistryId is found empty") - return nil, nil - } - - if ciPipeline.IsDockerConfigOverridden { - //set dockerRegistryId value with the DockerRegistryId of the overridden dockerRegistry - ciPipId := ciPipelineId - if ciPipeline.ParentCiPipeline != 0 { - ciPipId = ciPipeline.ParentCiPipeline + var dockerRegistryId string + if artifact.DataSource == repository3.POST_CI || artifact.DataSource == repository3.PRE_CD || artifact.DataSource == repository3.POST_CD { + // if image is generated by plugin at these stages + if artifact.CredentialsSourceType == repository3.GLOBAL_CONTAINER_REGISTRY { + dockerRegistryId = artifact.CredentialSourceValue } - ciTemplateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipId) - if err != nil { - impl.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "ciPipelineId", ciPipelineId, "error", err) - return nil, err + } else { + // if image is created by ci build + dockerRegistryId = *ciPipeline.CiTemplate.DockerRegistryId + if len(dockerRegistryId) == 0 { + impl.logger.Warn("returning as dockerRegistryId is found empty") + return nil, nil + } + + if ciPipeline.IsDockerConfigOverridden { + //set dockerRegistryId value with the DockerRegistryId of the overridden dockerRegistry + ciPipId := ciPipelineId + if ciPipeline.ParentCiPipeline != 0 { + ciPipId = ciPipeline.ParentCiPipeline + } + ciTemplateOverride, err := impl.ciTemplateOverrideRepository.FindByCiPipelineId(ciPipId) + if err != nil { + impl.logger.Errorw("error in getting ciTemplateOverride by ciPipelineId", "ciPipelineId", ciPipelineId, "error", err) + return nil, err + } + dockerRegistryId = ciTemplateOverride.DockerRegistryId } - dockerRegistryId = &ciTemplateOverride.DockerRegistryId } - return dockerRegistryId, nil + return &dockerRegistryId, nil } func (impl DockerRegistryIpsConfigServiceImpl) createOrUpdateDockerRegistryImagePullSecret(clusterId int, namespace string, ipsName string, dockerRegistryBean *repository.DockerArtifactStore) error { impl.logger.Infow("creating/updating ips", "ipsName", ipsName, "clusterId", clusterId) diff --git a/pkg/pipeline/AppArtifactManager.go b/pkg/pipeline/AppArtifactManager.go index 0049d5ce43..88eb0842a7 100644 --- a/pkg/pipeline/AppArtifactManager.go +++ b/pkg/pipeline/AppArtifactManager.go @@ -21,6 +21,7 @@ import ( "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/internal/sql/repository" + dockerArtifactStoreRegistry "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean2 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" @@ -56,6 +57,8 @@ type AppArtifactManagerImpl struct { ciWorkflowRepository pipelineConfig.CiWorkflowRepository pipelineStageService PipelineStageService cdPipelineConfigService CdPipelineConfigService + dockerArtifactRegistry dockerArtifactStoreRegistry.DockerArtifactStoreRepository + CiPipelineRepository pipelineConfig.CiPipelineRepository } func NewAppArtifactManagerImpl( @@ -66,7 +69,9 @@ func NewAppArtifactManagerImpl( ciArtifactRepository repository.CiArtifactRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, pipelineStageService PipelineStageService, - cdPipelineConfigService CdPipelineConfigService) *AppArtifactManagerImpl { + cdPipelineConfigService CdPipelineConfigService, + dockerArtifactRegistry dockerArtifactStoreRegistry.DockerArtifactStoreRepository, + CiPipelineRepository pipelineConfig.CiPipelineRepository) *AppArtifactManagerImpl { return &AppArtifactManagerImpl{ logger: logger, @@ -77,6 +82,8 @@ func NewAppArtifactManagerImpl( ciWorkflowRepository: ciWorkflowRepository, cdPipelineConfigService: cdPipelineConfigService, pipelineStageService: pipelineStageService, + dockerArtifactRegistry: dockerArtifactRegistry, + CiPipelineRepository: CiPipelineRepository, } } @@ -138,6 +145,9 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsForCdStage(pipelineId int, sta Latest: latest, Scanned: wfr.CdWorkflow.CiArtifact.Scanned, ScanEnabled: wfr.CdWorkflow.CiArtifact.ScanEnabled, + CiPipelineId: wfr.CdWorkflow.CiArtifact.PipelineId, + CredentialsSourceType: wfr.CdWorkflow.CiArtifact.CredentialsSourceType, + CredentialsSourceValue: wfr.CdWorkflow.CiArtifact.CredentialSourceValue, } if !parent { ciArtifact.Deployed = true @@ -177,12 +187,15 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsForCIParent(cdPipelineId int, impl.logger.Errorw("Error in parsing artifact material info", "err", err, "artifact", artifact) } ciArtifacts = append(ciArtifacts, bean2.CiArtifactBean{ - Id: artifact.Id, - Image: artifact.Image, - ImageDigest: artifact.ImageDigest, - MaterialInfo: mInfo, - ScanEnabled: artifact.ScanEnabled, - Scanned: artifact.Scanned, + Id: artifact.Id, + Image: artifact.Image, + ImageDigest: artifact.ImageDigest, + MaterialInfo: mInfo, + ScanEnabled: artifact.ScanEnabled, + Scanned: artifact.Scanned, + CiPipelineId: artifact.PipelineId, + CredentialsSourceType: artifact.CredentialsSourceType, + CredentialsSourceValue: artifact.CredentialSourceValue, }) } } @@ -300,6 +313,27 @@ func (impl *AppArtifactManagerImpl) FetchArtifactForRollbackV2(cdPipelineId, app if imageCommentResp := imageCommentsDataMap[deployedCiArtifacts[i].Id]; imageCommentResp != nil { deployedCiArtifacts[i].ImageComment = imageCommentResp } + var dockerRegistryId string + if deployedCiArtifacts[i].DataSource == repository.POST_CI || deployedCiArtifacts[i].DataSource == repository.PRE_CD || deployedCiArtifacts[i].DataSource == repository.POST_CD { + if deployedCiArtifacts[i].CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { + dockerRegistryId = deployedCiArtifacts[i].CredentialsSourceValue + } + } else { + ciPipeline, err := impl.CiPipelineRepository.FindById(deployedCiArtifacts[i].CiPipelineId) + if err != nil { + impl.logger.Errorw("error in fetching ciPipeline", "ciPipelineId", ciPipeline.Id, "error", err) + return deployedCiArtifactsResponse, err + } + dockerRegistryId = *ciPipeline.CiTemplate.DockerRegistryId + } + if len(dockerRegistryId) > 0 { + dockerArtifact, err := impl.dockerArtifactRegistry.FindOne(dockerRegistryId) + if err != nil { + impl.logger.Errorw("error in getting docker registry details", "err", err, "dockerArtifactStoreId", dockerRegistryId) + } + deployedCiArtifacts[i].RegistryType = string(dockerArtifact.RegistryType) + deployedCiArtifacts[i].RegistryName = dockerRegistryId + } } deployedCiArtifactsResponse.CdPipelineId = cdPipelineId @@ -357,14 +391,18 @@ func (impl *AppArtifactManagerImpl) BuildRollbackArtifactsList(artifactListingFi } userEmail := userEmails[ciArtifact.TriggeredBy] deployedCiArtifacts = append(deployedCiArtifacts, bean2.CiArtifactBean{ - Id: ciArtifact.Id, - Image: ciArtifact.Image, - MaterialInfo: mInfo, - DeployedTime: formatDate(ciArtifact.StartedOn, bean2.LayoutRFC3339), - WfrId: ciArtifact.CdWorkflowRunnerId, - DeployedBy: userEmail, - Scanned: ciArtifact.Scanned, - ScanEnabled: ciArtifact.ScanEnabled, + Id: ciArtifact.Id, + Image: ciArtifact.Image, + MaterialInfo: mInfo, + DeployedTime: formatDate(ciArtifact.StartedOn, bean2.LayoutRFC3339), + WfrId: ciArtifact.CdWorkflowRunnerId, + DeployedBy: userEmail, + Scanned: ciArtifact.Scanned, + ScanEnabled: ciArtifact.ScanEnabled, + CiPipelineId: ciArtifact.PipelineId, + CredentialsSourceType: ciArtifact.CredentialsSourceType, + CredentialsSourceValue: ciArtifact.CredentialSourceValue, + DataSource: ciArtifact.DataSource, }) artifactIds = append(artifactIds, ciArtifact.Id) } @@ -468,7 +506,27 @@ func (impl *AppArtifactManagerImpl) RetrieveArtifactsByCDPipeline(pipeline *pipe // if external webhook continue continue } - + var dockerRegistryId string + if artifact.PipelineId != 0 { + ciPipeline, err := impl.CiPipelineRepository.FindById(artifact.PipelineId) + if err != nil { + impl.logger.Errorw("error in fetching ciPipeline", "ciPipelineId", ciPipeline.Id, "error", err) + return nil, err + } + dockerRegistryId = *ciPipeline.CiTemplate.DockerRegistryId + } else { + if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { + dockerRegistryId = artifact.CredentialSourceValue + } + } + if len(dockerRegistryId) > 0 { + dockerArtifact, err := impl.dockerArtifactRegistry.FindOne(dockerRegistryId) + if err != nil { + impl.logger.Errorw("error in getting docker registry details", "err", err, "dockerArtifactStoreId", dockerRegistryId) + } + ciArtifacts[i].RegistryType = string(dockerArtifact.RegistryType) + ciArtifacts[i].RegistryName = dockerRegistryId + } var ciWorkflow *pipelineConfig.CiWorkflow if artifact.ParentCiArtifact != 0 { ciWorkflow, err = impl.ciWorkflowRepository.FindLastTriggeredWorkflowGitTriggersByArtifactId(artifact.ParentCiArtifact) @@ -610,6 +668,27 @@ func (impl *AppArtifactManagerImpl) setAdditionalDataInArtifacts(ciArtifacts []b if imageCommentResp := imageCommentsDataMap[ciArtifacts[i].Id]; imageCommentResp != nil { ciArtifacts[i].ImageComment = imageCommentResp } + var dockerRegistryId string + if ciArtifacts[i].DataSource == repository.POST_CI || ciArtifacts[i].DataSource == repository.PRE_CD || ciArtifacts[i].DataSource == repository.POST_CD { + if ciArtifacts[i].CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { + dockerRegistryId = ciArtifacts[i].CredentialsSourceValue + } + } else { + ciPipeline, err := impl.CiPipelineRepository.FindById(ciArtifacts[i].CiPipelineId) + if err != nil { + impl.logger.Errorw("error in fetching ciPipeline", "ciPipelineId", ciPipeline.Id, "error", err) + return nil, err + } + dockerRegistryId = *ciPipeline.CiTemplate.DockerRegistryId + } + if len(dockerRegistryId) > 0 { + dockerArtifact, err := impl.dockerArtifactRegistry.FindOne(dockerRegistryId) + if err != nil { + impl.logger.Errorw("error in getting docker registry details", "err", err, "dockerArtifactStoreId", dockerRegistryId) + } + ciArtifacts[i].RegistryType = string(dockerArtifact.RegistryType) + ciArtifacts[i].RegistryName = dockerRegistryId + } } return impl.setGitTriggerData(ciArtifacts) @@ -699,17 +778,20 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsList(listingFilterOpts *bean.A impl.logger.Errorw("Error in parsing artifact material info", "err", err, "artifact", currentRunningArtifact) } currentRunningArtifactBean = &bean2.CiArtifactBean{ - Id: currentRunningArtifact.Id, - Image: currentRunningArtifact.Image, - ImageDigest: currentRunningArtifact.ImageDigest, - MaterialInfo: mInfo, - ScanEnabled: currentRunningArtifact.ScanEnabled, - Scanned: currentRunningArtifact.Scanned, - Deployed: true, - DeployedTime: formatDate(latestWf[0].CdWorkflow.CreatedOn, bean2.LayoutRFC3339), - Latest: true, - CreatedTime: formatDate(currentRunningArtifact.CreatedOn, bean2.LayoutRFC3339), - DataSource: currentRunningArtifact.DataSource, + Id: currentRunningArtifact.Id, + Image: currentRunningArtifact.Image, + ImageDigest: currentRunningArtifact.ImageDigest, + MaterialInfo: mInfo, + ScanEnabled: currentRunningArtifact.ScanEnabled, + Scanned: currentRunningArtifact.Scanned, + Deployed: true, + DeployedTime: formatDate(latestWf[0].CdWorkflow.CreatedOn, bean2.LayoutRFC3339), + Latest: true, + CreatedTime: formatDate(currentRunningArtifact.CreatedOn, bean2.LayoutRFC3339), + DataSource: currentRunningArtifact.DataSource, + CiPipelineId: currentRunningArtifact.PipelineId, + CredentialsSourceType: currentRunningArtifact.CredentialsSourceType, + CredentialsSourceValue: currentRunningArtifact.CredentialSourceValue, } if currentRunningArtifact.WorkflowId != nil { currentRunningArtifactBean.CiWorkflowId = *currentRunningArtifact.WorkflowId @@ -723,6 +805,11 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsList(listingFilterOpts *bean.A return ciArtifacts, 0, "", totalCount, err } } else { + if listingFilterOpts.ParentStageType == WorklowTypePre { + listingFilterOpts.PluginStage = repository.PRE_CD + } else if listingFilterOpts.ParentStageType == WorklowTypePost { + listingFilterOpts.PluginStage = repository.POST_CD + } ciArtifacts, totalCount, err = impl.BuildArtifactsForCdStageV2(listingFilterOpts) if err != nil { impl.logger.Errorw("error in getting ci artifacts for ci/webhook type parent", "pipelineId", listingFilterOpts.PipelineId, "parentPipelineId", listingFilterOpts.ParentId, "parentStageType", listingFilterOpts.ParentStageType, "currentStageType", listingFilterOpts.StageType) @@ -743,12 +830,12 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsList(listingFilterOpts *bean.A } func (impl *AppArtifactManagerImpl) BuildArtifactsForCdStageV2(listingFilterOpts *bean.ArtifactsListFilterOptions) ([]*bean2.CiArtifactBean, int, error) { - cdWfrList, totalCount, err := impl.ciArtifactRepository.FindArtifactByListFilter(listingFilterOpts) + cdArtifacts, totalCount, err := impl.ciArtifactRepository.FindArtifactByListFilter(listingFilterOpts) if err != nil { impl.logger.Errorw("error in fetching cd workflow runners using filter", "filterOptions", listingFilterOpts, "err", err) return nil, totalCount, err } - ciArtifacts := make([]*bean2.CiArtifactBean, 0, len(cdWfrList)) + ciArtifacts := make([]*bean2.CiArtifactBean, 0, len(cdArtifacts)) //get artifact running on parent cd artifactRunningOnParentCd := 0 @@ -762,28 +849,31 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsForCdStageV2(listingFilterOpts artifactRunningOnParentCd = parentCdWfrList[0].CdWorkflow.CiArtifact.Id } - for _, wfr := range cdWfrList { - mInfo, err := parseMaterialInfo([]byte(wfr.MaterialInfo), wfr.DataSource) + for _, artifact := range cdArtifacts { + mInfo, err := parseMaterialInfo([]byte(artifact.MaterialInfo), artifact.DataSource) if err != nil { mInfo = []byte("[]") impl.logger.Errorw("Error in parsing artifact material info", "err", err) } ciArtifact := &bean2.CiArtifactBean{ - Id: wfr.Id, - Image: wfr.Image, - ImageDigest: wfr.ImageDigest, + Id: artifact.Id, + Image: artifact.Image, + ImageDigest: artifact.ImageDigest, MaterialInfo: mInfo, //TODO:LastSuccessfulTriggerOnParent - Scanned: wfr.Scanned, - ScanEnabled: wfr.ScanEnabled, - RunningOnParentCd: wfr.Id == artifactRunningOnParentCd, - ExternalCiPipelineId: wfr.ExternalCiPipelineId, - ParentCiArtifact: wfr.ParentCiArtifact, - CreatedTime: formatDate(wfr.CreatedOn, bean2.LayoutRFC3339), - DataSource: wfr.DataSource, + Scanned: artifact.Scanned, + ScanEnabled: artifact.ScanEnabled, + RunningOnParentCd: artifact.Id == artifactRunningOnParentCd, + ExternalCiPipelineId: artifact.ExternalCiPipelineId, + ParentCiArtifact: artifact.ParentCiArtifact, + CreatedTime: formatDate(artifact.CreatedOn, bean2.LayoutRFC3339), + DataSource: artifact.DataSource, + CiPipelineId: artifact.PipelineId, + CredentialsSourceType: artifact.CredentialsSourceType, + CredentialsSourceValue: artifact.CredentialSourceValue, } - if wfr.WorkflowId != nil { - ciArtifact.CiWorkflowId = *wfr.WorkflowId + if artifact.WorkflowId != nil { + ciArtifact.CiWorkflowId = *artifact.WorkflowId } ciArtifacts = append(ciArtifacts, ciArtifact) } @@ -808,18 +898,21 @@ func (impl *AppArtifactManagerImpl) BuildArtifactsForCIParentV2(listingFilterOpt impl.logger.Errorw("Error in parsing artifact material info", "err", err, "artifact", artifact) } ciArtifact := &bean2.CiArtifactBean{ - Id: artifact.Id, - Image: artifact.Image, - ImageDigest: artifact.ImageDigest, - MaterialInfo: mInfo, - ScanEnabled: artifact.ScanEnabled, - Scanned: artifact.Scanned, - Deployed: artifact.Deployed, - DeployedTime: formatDate(artifact.DeployedTime, bean2.LayoutRFC3339), - ExternalCiPipelineId: artifact.ExternalCiPipelineId, - ParentCiArtifact: artifact.ParentCiArtifact, - CreatedTime: formatDate(artifact.CreatedOn, bean2.LayoutRFC3339), - DataSource: artifact.DataSource, + Id: artifact.Id, + Image: artifact.Image, + ImageDigest: artifact.ImageDigest, + MaterialInfo: mInfo, + ScanEnabled: artifact.ScanEnabled, + Scanned: artifact.Scanned, + Deployed: artifact.Deployed, + DeployedTime: formatDate(artifact.DeployedTime, bean2.LayoutRFC3339), + ExternalCiPipelineId: artifact.ExternalCiPipelineId, + ParentCiArtifact: artifact.ParentCiArtifact, + CreatedTime: formatDate(artifact.CreatedOn, bean2.LayoutRFC3339), + CiPipelineId: artifact.PipelineId, + DataSource: artifact.DataSource, + CredentialsSourceType: artifact.CredentialsSourceType, + CredentialsSourceValue: artifact.CredentialSourceValue, } if artifact.WorkflowId != nil { ciArtifact.CiWorkflowId = *artifact.WorkflowId diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 933e20dc46..c67e2ec641 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -627,7 +627,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea ciPipeline.CustomTagObject = &bean.CustomTagData{ TagPattern: customTag.TagPattern, CounterX: customTag.AutoIncreasingNumber, + Enabled: customTag.Enabled, } + ciPipeline.EnableCustomTag = customTag.Enabled } if ciEnvMapping.Id > 0 { ciPipeline.EnvironmentId = ciEnvMapping.EnvironmentId @@ -765,6 +767,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi TagPattern: customTag.TagPattern, CounterX: customTag.AutoIncreasingNumber, } + ciPipeline.EnableCustomTag = customTag.Enabled } ciEnvMapping, err := impl.ciPipelineRepository.FindCiEnvMappingByCiPipelineId(pipelineId) if err != nil && err != pg.ErrNoRows { diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 62acb92a42..f7e7b6831e 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -85,6 +85,7 @@ type CdHandler interface { CheckAndSendArgoPipelineStatusSyncEventIfNeeded(pipelineId int, userId int32, isAppStoreApplication bool) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppDeploymentStatusForEnvironments(request resourceGroup2.ResourceGroupingRequest) ([]*pipelineConfig.AppDeploymentStatus, error) + DeactivateImageReservationPathsOnFailure(imagePathReservationIds []int) error } type CdHandlerImpl struct { @@ -123,9 +124,10 @@ type CdHandlerImpl struct { config *types.CdConfig clusterService cluster.ClusterService blobConfigStorageService BlobStorageConfigService + customTagService CustomTagService } -func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService) *CdHandlerImpl { +func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, @@ -161,6 +163,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c workflowService: workflowService, clusterService: clusterService, blobConfigStorageService: blobConfigStorageService, + customTagService: customTagService, } config, err := types.GetCdConfig() if err != nil { @@ -621,7 +624,13 @@ func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, userId int32) (int, impl.Logger.Error("cannot terminate wf runner", "err", err) return 0, err } - + if len(workflowRunner.ImagePathReservationIds) > 0 { + err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in deactivating image path reservation ids", "err", err) + return 0, err + } + } workflowRunner.Status = executors.WorkflowCancel workflowRunner.UpdatedOn = time.Now() workflowRunner.UpdatedBy = userId @@ -778,6 +787,7 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel var newCiArtifactIds []int for _, ciArtifact := range ciArtifacts { if ciArtifact.ParentCiArtifact > 0 && ciArtifact.WorkflowId == nil { + // parent ci artifact ID can be greater than zero when pipeline is linked or when image is copied at plugin level from some other image isLinked = true newCiArtifactIds = append(newCiArtifactIds, ciArtifact.ParentCiArtifact) parentCiArtifact[ciArtifact.Id] = ciArtifact.ParentCiArtifact @@ -1610,3 +1620,7 @@ func (impl *CdHandlerImpl) FetchAppDeploymentStatusForEnvironments(request resou return deploymentStatuses, err } + +func (impl *CdHandlerImpl) DeactivateImageReservationPathsOnFailure(imagePathReservationIds []int) error { + return impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) +} diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index f6c947c75d..bf12475b79 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -71,7 +71,7 @@ type CiCdPipelineOrchestrator interface { UpdateMaterial(updateMaterialRequest *bean.UpdateMaterialDTO) (*bean.UpdateMaterialDTO, error) CreateCiConf(createRequest *bean.CiConfigRequest, templateId int) (*bean.CiConfigRequest, error) CreateCDPipelines(pipelineRequest *bean.CDPipelineConfigObject, appId int, userId int32, tx *pg.Tx, appName string) (pipelineId int, err error) - UpdateCDPipeline(pipelineRequest *bean.CDPipelineConfigObject, userId int32, tx *pg.Tx) (err error) + UpdateCDPipeline(pipelineRequest *bean.CDPipelineConfigObject, userId int32, tx *pg.Tx) (pipeline *pipelineConfig.Pipeline, err error) DeleteCiPipeline(pipeline *pipelineConfig.CiPipeline, request *bean.CiPatchRequest, tx *pg.Tx) error DeleteCdPipeline(pipelineId int, userId int32, tx *pg.Tx) error PatchMaterialValue(createRequest *bean.CiPipeline, userId int32, oldPipeline *pipelineConfig.CiPipeline) (*bean.CiPipeline, error) @@ -331,14 +331,19 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. AuditLog: sql.AuditLog{UpdatedBy: userId, UpdatedOn: time.Now()}, } + if createRequest.EnableCustomTag && len(createRequest.CustomTagObject.TagPattern) == 0 { + return nil, errors.New("please input custom tag data if tag is enabled") + } + //If customTagObject has been passed, create or update the resource //Otherwise deleteIfExists - if createRequest.CustomTagObject != nil { + if len(createRequest.CustomTagObject.TagPattern) > 0 { customTag := bean4.CustomTag{ EntityKey: bean2.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipelineObject.Id), TagPattern: createRequest.CustomTagObject.TagPattern, AutoIncreasingNumber: createRequest.CustomTagObject.CounterX, + Enabled: createRequest.EnableCustomTag, } err = impl.customTagService.CreateOrUpdateCustomTag(&customTag) if err != nil { @@ -348,6 +353,7 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. customTag := bean4.CustomTag{ EntityKey: bean2.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipelineObject.Id), + Enabled: false, } err := impl.customTagService.DeleteCustomTagIfExists(customTag) if err != nil { @@ -770,12 +776,13 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf } //If customTagObejct has been passed, save it - if ciPipeline.CustomTagObject != nil { + if ciPipeline.CustomTagObject != nil && len(ciPipeline.CustomTagObject.TagPattern) != 0 { customTag := &bean4.CustomTag{ EntityKey: bean2.EntityTypeCiPipelineId, EntityValue: strconv.Itoa(ciPipeline.Id), TagPattern: ciPipeline.CustomTagObject.TagPattern, AutoIncreasingNumber: ciPipeline.CustomTagObject.CounterX, + Enabled: ciPipeline.EnableCustomTag, } err := impl.customTagService.CreateOrUpdateCustomTag(customTag) if err != nil { @@ -1522,14 +1529,14 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCDPipelines(pipelineRequest *bean return pipeline.Id, nil } -func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean.CDPipelineConfigObject, userId int32, tx *pg.Tx) (err error) { - pipeline, err := impl.pipelineRepository.FindById(pipelineRequest.Id) +func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean.CDPipelineConfigObject, userId int32, tx *pg.Tx) (pipeline *pipelineConfig.Pipeline, err error) { + pipeline, err = impl.pipelineRepository.FindById(pipelineRequest.Id) if err == pg.ErrNoRows { - return fmt.Errorf("no cd pipeline found") + return pipeline, fmt.Errorf("no cd pipeline found") } else if err != nil { - return err + return pipeline, err } else if pipeline.Id == 0 { - return fmt.Errorf("no cd pipeline found") + return pipeline, fmt.Errorf("no cd pipeline found") } preStageConfig := "" preTriggerType := pipelineConfig.TriggerType("") @@ -1554,13 +1561,13 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean. preStageConfigMapSecretNames, err := json.Marshal(&pipelineRequest.PreStageConfigMapSecretNames) if err != nil { impl.logger.Error(err) - return err + return pipeline, err } postStageConfigMapSecretNames, err := json.Marshal(&pipelineRequest.PostStageConfigMapSecretNames) if err != nil { impl.logger.Error(err) - return err + return pipeline, err } pipeline.TriggerType = pipelineRequest.TriggerType @@ -1577,20 +1584,20 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean. err = impl.pipelineRepository.Update(pipeline, tx) if err != nil { impl.logger.Errorw("error in updating cd pipeline", "err", err, "pipeline", pipeline) - return err + return pipeline, err } if pipeline.PreStageConfig != "" { err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, tx, repository4.PRE_CD_TYPE, false, 0, time.Time{}) if err != nil { impl.logger.Errorw("error in creating pre cd script entry", "err", err, "pipeline", pipeline) - return err + return pipeline, err } } if pipeline.PostStageConfig != "" { err = impl.prePostCdScriptHistoryService.CreatePrePostCdScriptHistory(pipeline, tx, repository4.POST_CD_TYPE, false, 0, time.Time{}) if err != nil { impl.logger.Errorw("error in creating post cd script entry", "err", err, "pipeline", pipeline) - return err + return pipeline, err } } @@ -1599,7 +1606,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean. err = impl.pipelineStageService.UpdatePipelineStage(pipelineRequest.PreDeployStage, repository5.PIPELINE_STAGE_TYPE_PRE_CD, pipelineRequest.Id, userId) if err != nil { impl.logger.Errorw("error in updating pre stage", "err", err, "preDeployStage", pipelineRequest.PreDeployStage, "cdPipelineId", pipelineRequest.Id) - return err + return pipeline, err } } if pipelineRequest.PostDeployStage != nil { @@ -1607,10 +1614,10 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateCDPipeline(pipelineRequest *bean. err = impl.pipelineStageService.UpdatePipelineStage(pipelineRequest.PostDeployStage, repository5.PIPELINE_STAGE_TYPE_POST_CD, pipelineRequest.Id, userId) if err != nil { impl.logger.Errorw("error in updating post stage", "err", err, "postDeployStage", pipelineRequest.PostDeployStage, "cdPipelineId", pipelineRequest.Id) - return err + return pipeline, err } } - return err + return pipeline, nil } func (impl CiCdPipelineOrchestratorImpl) DeleteCdPipeline(pipelineId int, userId int32, tx *pg.Tx) error { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index b972fcd71f..82d9faf391 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -623,6 +623,14 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) { impl.Logger.Errorw("error in marking image tag unreserved", "err", err) return 0, err } + imagePathReservationIds := workflow.ImagePathReservationIds + if len(imagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + } return workflow.Id, nil } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 3fbafddc8d..78efba7280 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + repository5 "github.com/devtron-labs/devtron/internal/sql/repository" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" @@ -30,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/user" @@ -77,6 +79,8 @@ type CiServiceImpl struct { customTagService CustomTagService config *types.CiConfig scopedVariableManager variables.ScopedVariableManager + pluginInputVariableParser PluginInputVariableParser + globalPluginService plugin.GlobalPluginService } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -89,6 +93,8 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService ciTemplateService CiTemplateService, appCrudOperationService app.AppCrudOperationService, envRepository repository1.EnvironmentRepository, appRepository appRepository.AppRepository, scopedVariableManager variables.ScopedVariableManager, customTagService CustomTagService, + pluginInputVariableParser PluginInputVariableParser, + globalPluginService plugin.GlobalPluginService, ) *CiServiceImpl { cis := &CiServiceImpl{ Logger: Logger, @@ -108,6 +114,8 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService appRepository: appRepository, scopedVariableManager: scopedVariableManager, customTagService: customTagService, + pluginInputVariableParser: pluginInputVariableParser, + globalPluginService: globalPluginService, } config, err := types.GetCiConfig() if err != nil { @@ -457,7 +465,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if err != nil && err != pg.ErrNoRows { return nil, err } - if customTag.Id != 0 { + if customTag.Id != 0 && customTag.Enabled == true { imagePathReservation, err := impl.customTagService.GenerateImagePath(bean2.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), pipeline.CiTemplate.DockerRegistry.RegistryURL, pipeline.CiTemplate.DockerRepository) if err != nil { if errors.Is(err, bean2.ErrImagePathInUse) { @@ -471,13 +479,30 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } return nil, err } - savedWf.ImagePathReservationId = imagePathReservation.Id + savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} //imagePath = docker.io/avd0/dashboard:fd23414b dockerImageTag = strings.Split(imagePathReservation.ImagePath, ":")[1] } else { dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) } + // copyContainerImage plugin specific logic + registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err := impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin( + preCiSteps, postCiSteps, dockerImageTag, customTag.Id, + fmt.Sprintf(bean2.ImagePathPattern, pipeline.CiTemplate.DockerRegistry.RegistryURL, pipeline.CiTemplate.DockerRepository, dockerImageTag), pipeline.CiTemplate.DockerRegistry.Id) + if err != nil { + impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") + savedWf.Status = pipelineConfig.WorkflowFailed + savedWf.Message = err.Error() + err1 := impl.ciWorkflowRepository.UpdateWorkFlow(savedWf) + if err1 != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") + } + return nil, err + } + + savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) + if ciWorkflowConfig.CiCacheBucket == "" { ciWorkflowConfig.CiCacheBucket = impl.config.DefaultCacheBucket } @@ -587,41 +612,44 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } workflowRequest := &types.WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", - CiProjectDetails: ciProjectDetails, - Namespace: ciWorkflowConfig.Namespace, - BlobStorageConfigured: savedWf.BlobStorageEnabled, - CiImage: ciWorkflowConfig.CiImage, - ActiveDeadlineSeconds: ciWorkflowConfig.CiTimeout, - WorkflowId: savedWf.Id, - TriggeredBy: savedWf.TriggeredBy, - CacheLimit: impl.config.CacheLimit, - ScanEnabled: pipeline.ScanEnabled, - CloudProvider: impl.config.CloudProvider, - DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), - DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), - PreCiSteps: preCiSteps, - PostCiSteps: postCiSteps, - RefPlugins: refPluginsData, - AppName: pipeline.App.AppName, - TriggerByAuthor: user.EmailId, - CiBuildConfig: ciBuildConfigBean, - CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, - IgnoreDockerCachePush: impl.config.IgnoreDockerCacheForCI, - IgnoreDockerCachePull: impl.config.IgnoreDockerCacheForCI, - CacheInvalidate: trigger.InvalidateCache, - ExtraEnvironmentVariables: trigger.ExtraEnvironmentVariables, - EnableBuildContext: impl.config.EnableBuildContext, - OrchestratorHost: impl.config.OrchestratorHost, - OrchestratorToken: impl.config.OrchestratorToken, - ImageRetryCount: impl.config.ImageRetryCount, - ImageRetryInterval: impl.config.ImageRetryInterval, - WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, - CiArtifactLastFetch: trigger.CiArtifactLastFetch, + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", + CiProjectDetails: ciProjectDetails, + Namespace: ciWorkflowConfig.Namespace, + BlobStorageConfigured: savedWf.BlobStorageEnabled, + CiImage: ciWorkflowConfig.CiImage, + ActiveDeadlineSeconds: ciWorkflowConfig.CiTimeout, + WorkflowId: savedWf.Id, + TriggeredBy: savedWf.TriggeredBy, + CacheLimit: impl.config.CacheLimit, + ScanEnabled: pipeline.ScanEnabled, + CloudProvider: impl.config.CloudProvider, + DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), + DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), + PreCiSteps: preCiSteps, + PostCiSteps: postCiSteps, + RefPlugins: refPluginsData, + AppName: pipeline.App.AppName, + TriggerByAuthor: user.EmailId, + CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, + IgnoreDockerCachePush: impl.config.IgnoreDockerCacheForCI, + IgnoreDockerCachePull: impl.config.IgnoreDockerCacheForCI, + CacheInvalidate: trigger.InvalidateCache, + ExtraEnvironmentVariables: trigger.ExtraEnvironmentVariables, + EnableBuildContext: impl.config.EnableBuildContext, + OrchestratorHost: impl.config.OrchestratorHost, + OrchestratorToken: impl.config.OrchestratorToken, + ImageRetryCount: impl.config.ImageRetryCount, + ImageRetryInterval: impl.config.ImageRetryInterval, + WorkflowExecutor: impl.config.GetWorkflowExecutorType(), + Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, + CiArtifactLastFetch: trigger.CiArtifactLastFetch, + RegistryDestinationImageMap: registryDestinationImageMap, + RegistryCredentialMap: registryCredentialMap, + PluginArtifactStage: pluginArtifactStage, } if dockerRegistry != nil { @@ -644,7 +672,9 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if ciWorkflowConfig.LogsBucket == "" { ciWorkflowConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() } - + if len(registryDestinationImageMap) > 0 { + workflowRequest.PushImageBeforePostCI = true + } switch workflowRequest.CloudProvider { case types.BLOB_STORAGE_S3: //No AccessKey is used for uploading artifacts, instead IAM based auth is used @@ -702,6 +732,63 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. return workflowRequest, nil } +func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*bean2.StepObject, postCiSteps []*bean2.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]plugin.RegistryCredentials, string, []int, error) { + var registryDestinationImageMap map[string][]string + var registryCredentialMap map[string]plugin.RegistryCredentials + var pluginArtifactStage string + var imagePathReservationIds []int + copyContainerImagePluginId, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, err + } + for _, step := range preCiSteps { + if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + return nil, nil, pluginArtifactStage, nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") + } + } + for _, step := range postCiSteps { + if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + registryDestinationImageMap, registryCredentialMap, err = impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) + if err != nil { + impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, err + } + pluginArtifactStage = repository5.POST_CI + } + } + for _, images := range registryDestinationImageMap { + for _, image := range images { + if image == buildImagePath { + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, + bean2.ErrImagePathInUse + } + } + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) + if err != nil { + return nil, nil, pluginArtifactStage, imagePathReservationIds, err + } + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil +} + +func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, registryImageMap map[string][]string) ([]int, error) { + var imagePathReservationIds []int + for _, images := range registryImageMap { + for _, image := range images { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + } + return imagePathReservationIds, nil +} + func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*bean2.StepObject { //before plugin support, few variables were set as env vars in ci-runner //these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload diff --git a/pkg/pipeline/CustomTagService.go b/pkg/pipeline/CustomTagService.go index 76cfc766ac..fb99f98485 100644 --- a/pkg/pipeline/CustomTagService.go +++ b/pkg/pipeline/CustomTagService.go @@ -19,6 +19,11 @@ type CustomTagService interface { GenerateImagePath(entityKey int, entityValue string, dockerRegistryURL string, dockerRepo string) (*repository.ImagePathReservation, error) DeleteCustomTagIfExists(tag bean.CustomTag) error DeactivateImagePathReservation(id int) error + GetCustomTag(entityKey int, entityValue string) (*repository.CustomTag, string, error) + ReserveImagePath(imagePath string, customTagId int) (*repository.ImagePathReservation, error) + DeactivateImagePathReservationByImagePath(imagePaths []string) error + DeactivateImagePathReservationByImageIds(imagePathReservationIds []int) error + DisableCustomTagIfExist(tag bean.CustomTag) error } type CustomTagServiceImpl struct { @@ -38,16 +43,24 @@ func (impl *CustomTagServiceImpl) DeactivateImagePathReservation(id int) error { } func (impl *CustomTagServiceImpl) CreateOrUpdateCustomTag(tag *bean.CustomTag) error { - if err := validateTagPattern(tag.TagPattern); err != nil { - return err + + if len(tag.TagPattern) == 0 && tag.Enabled { + return fmt.Errorf("tag pattern cannot be empty") } - customTagData := repository.CustomTag{ + if tag.Enabled { + if err := validateTagPattern(tag.TagPattern); err != nil { + return err + } + } + var customTagData repository.CustomTag + customTagData = repository.CustomTag{ EntityKey: tag.EntityKey, EntityValue: tag.EntityValue, TagPattern: strings.ReplaceAll(tag.TagPattern, bean2.IMAGE_TAG_VARIABLE_NAME_X, bean2.IMAGE_TAG_VARIABLE_NAME_x), AutoIncreasingNumber: tag.AutoIncreasingNumber, Metadata: tag.Metadata, Active: true, + Enabled: true, } oldTagObject, err := impl.customTagRepository.FetchCustomTagData(customTagData.EntityKey, customTagData.EntityValue) if err != nil && err != pg.ErrNoRows { @@ -160,18 +173,97 @@ func isValidDockerImageTag(tag string) bool { return re.MatchString(tag) } -func validateTag(imageTag string) error { - if len(imageTag) == 0 || len(imageTag) > 128 { - return fmt.Errorf("image tag should be of len 1-128 only, imageTag: %s", imageTag) +func (impl *CustomTagServiceImpl) GetCustomTag(entityKey int, entityValue string) (*repository.CustomTag, string, error) { + connection := impl.customTagRepository.GetConnection() + tx, err := connection.Begin() + customTagData, err := impl.customTagRepository.IncrementAndFetchByEntityKeyAndValue(tx, entityKey, entityValue) + if err != nil { + return nil, "", err + } + err = tx.Commit() + if err != nil { + impl.Logger.Errorw("Error in fetching custom tag", "err", err) + return customTagData, "", err + } + var dockerTag string + if customTagData != nil && len(customTagData.TagPattern) == 0 { + return customTagData, dockerTag, nil + } + dockerTag, err = validateAndConstructTag(customTagData) + if err != nil { + return nil, "", err + } + return customTagData, dockerTag, nil + +} + +func (impl *CustomTagServiceImpl) ReserveImagePath(imagePath string, customTagId int) (*repository.ImagePathReservation, error) { + connection := impl.customTagRepository.GetConnection() + tx, err := connection.Begin() + if err != nil { + return nil, err + } + imagePathReservations, err := impl.customTagRepository.FindByImagePath(tx, imagePath) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if len(imagePathReservations) > 0 { + return nil, bean2.ErrImagePathInUse + } + imagePathReservation := &repository.ImagePathReservation{ + ImagePath: imagePath, + CustomTagId: customTagId, + } + err = impl.customTagRepository.InsertImagePath(tx, imagePathReservation) + if err != nil { + return imagePathReservation, err + } + err = tx.Commit() + if err != nil { + impl.Logger.Errorw("Error in fetching custom tag", "err", err) + return imagePathReservation, err + } + return imagePathReservation, err +} + +func (impl *CustomTagServiceImpl) DeactivateImagePathReservationByImagePath(imagePaths []string) error { + connection := impl.customTagRepository.GetConnection() + tx, err := connection.Begin() + if err != nil { + return nil + } + err = impl.customTagRepository.DeactivateImagePathReservationByImagePaths(tx, imagePaths) + if err != nil { + impl.Logger.Errorw("error in marking image path unreserved") + return err + } + err = tx.Commit() + if err != nil { + impl.Logger.Errorw("Error in fetching custom tag", "err", err) + return err + } + return nil +} + +func (impl *CustomTagServiceImpl) DeactivateImagePathReservationByImageIds(imagePathReservationIds []int) error { + connection := impl.customTagRepository.GetConnection() + tx, err := connection.Begin() + if err != nil { + return nil } - allowedSymbols := ".abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRSTUVWXYZ-0987654321" - allowedCharSet := make(map[int32]struct{}) - for _, c := range allowedSymbols { - allowedCharSet[c] = struct{}{} + err = impl.customTagRepository.DeactivateImagePathReservationByImagePathReservationIds(tx, imagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in marking image path unreserved") + return err } - firstChar := imageTag[0:1] - if firstChar == "." || firstChar == "-" { - return fmt.Errorf("image tag can not start with a period or a hyphen, imageTag: %s", imageTag) + err = tx.Commit() + if err != nil { + impl.Logger.Errorw("Error in fetching custom tag", "err", err) + return err } return nil } + +func (impl *CustomTagServiceImpl) DisableCustomTagIfExist(tag bean.CustomTag) error { + return impl.customTagRepository.DisableCustomTag(tag.EntityKey, tag.EntityValue) +} diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index ec0bc77e72..a86cc898d2 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -56,6 +56,7 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/api/errors" "net/http" + "strconv" "strings" "time" ) @@ -140,8 +141,8 @@ type CdPipelineConfigServiceImpl struct { scopedVariableManager variables.ScopedVariableManager deploymentConfig *DeploymentServiceTypeConfig application application.ServiceClient - - devtronAppCMCSService DevtronAppCMCSService + customTagService CustomTagService + devtronAppCMCSService DevtronAppCMCSService } func NewCdPipelineConfigServiceImpl( @@ -173,8 +174,8 @@ func NewCdPipelineConfigServiceImpl( scopedVariableManager variables.ScopedVariableManager, deploymentConfig *DeploymentServiceTypeConfig, application application.ServiceClient, - - devtronAppCMCSService DevtronAppCMCSService) *CdPipelineConfigServiceImpl { + devtronAppCMCSService DevtronAppCMCSService, + customTagService CustomTagService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -205,6 +206,7 @@ func NewCdPipelineConfigServiceImpl( deploymentConfig: deploymentConfig, application: application, devtronAppCMCSService: devtronAppCMCSService, + customTagService: customTagService, } } @@ -261,6 +263,7 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi return nil, err } } + if dbPipeline.PostStageConfigMapSecretNames != "" { err = json.Unmarshal([]byte(dbPipeline.PostStageConfigMapSecretNames), &postStageConfigmapSecrets) if err != nil { @@ -272,6 +275,36 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi if err != nil { return nil, err } + + var customTag *bean.CustomTagData + var customTagStage repository5.PipelineStageType + var customTagEnabled bool + customTagPreCD, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypePreCD, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom Tag precd") + return nil, err + } + customTagPostCD, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypePostCD, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom Tag precd") + return nil, err + } + if customTagPreCD != nil && customTagPreCD.Id > 0 { + customTag = &bean.CustomTagData{TagPattern: customTagPreCD.TagPattern, + CounterX: customTagPreCD.AutoIncreasingNumber, + Enabled: customTagPreCD.Enabled, + } + customTagStage = repository5.PIPELINE_STAGE_TYPE_PRE_CD + customTagEnabled = customTagPreCD.Enabled + } else if customTagPostCD != nil && customTagPostCD.Id > 0 { + customTag = &bean.CustomTagData{TagPattern: customTagPostCD.TagPattern, + CounterX: customTagPostCD.AutoIncreasingNumber, + Enabled: customTagPostCD.Enabled, + } + customTagStage = repository5.PIPELINE_STAGE_TYPE_POST_CD + customTagEnabled = customTagPostCD.Enabled + } + cdPipeline = &bean.CDPipelineConfigObject{ Id: dbPipeline.Id, Name: dbPipeline.Name, @@ -293,6 +326,10 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi DeploymentAppType: dbPipeline.DeploymentAppType, DeploymentAppCreated: dbPipeline.DeploymentAppCreated, IsVirtualEnvironment: dbPipeline.Environment.IsVirtualEnvironment, + CustomTagObject: customTag, + CustomTagStage: &customTagStage, + EnableCustomTag: customTagEnabled, + AppId: dbPipeline.AppId, } var preDeployStage *bean3.PipelineStageDto var postDeployStage *bean3.PipelineStageDto @@ -382,10 +419,141 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest return nil, err } } + } + return pipelineCreateRequest, nil +} +func (impl *CdPipelineConfigServiceImpl) CDPipelineCustomTagDBOperations(pipeline *bean.CDPipelineConfigObject) error { + + if pipeline.EnableCustomTag && (pipeline.CustomTagObject != nil && len(pipeline.CustomTagObject.TagPattern) == 0) { + return fmt.Errorf("please provide custom tag data if tag is enabled") + } + if pipeline.CustomTagObject != nil && pipeline.CustomTagObject.CounterX < 0 { + return fmt.Errorf("value of {x} cannot be negative") } + if !pipeline.EnableCustomTag { + // disable custom tag if exist + err := impl.DisableCustomTag(pipeline) + if err != nil { + return err + } + return nil + } else { + err := impl.SaveOrUpdateCustomTagForCDPipeline(pipeline) + if err != nil { + impl.logger.Errorw("error in creating custom tag for pipeline stage", "err", err) + return err + } + } + if *pipeline.CustomTagStage == repository5.PIPELINE_STAGE_TYPE_POST_CD { + // delete entry for post stage if any + preCDStageName := repository5.PIPELINE_STAGE_TYPE_PRE_CD + err := impl.DeleteCustomTagByPipelineStageType(&preCDStageName, pipeline.Id) + if err != nil { + return err + } + } else if *pipeline.CustomTagStage == repository5.PIPELINE_STAGE_TYPE_PRE_CD { + postCdStageName := repository5.PIPELINE_STAGE_TYPE_POST_CD + err := impl.DeleteCustomTagByPipelineStageType(&postCdStageName, pipeline.Id) + if err != nil { + return err + } + } + return nil +} - return pipelineCreateRequest, nil +func (impl *CdPipelineConfigServiceImpl) DeleteCustomTag(pipeline *bean.CDPipelineConfigObject) error { + preStage := repository5.PIPELINE_STAGE_TYPE_PRE_CD + postStage := repository5.PIPELINE_STAGE_TYPE_POST_CD + err := impl.DeleteCustomTagByPipelineStageType(&preStage, pipeline.Id) + if err != nil { + return err + } + err = impl.DeleteCustomTagByPipelineStageType(&postStage, pipeline.Id) + if err != nil { + return err + } + return nil +} + +func (impl *CdPipelineConfigServiceImpl) DisableCustomTag(pipeline *bean.CDPipelineConfigObject) error { + preStage := repository5.PIPELINE_STAGE_TYPE_PRE_CD + postStage := repository5.PIPELINE_STAGE_TYPE_POST_CD + err := impl.DisableCustomTagByPipelineStageType(&preStage, pipeline.Id) + if err != nil { + return err + } + err = impl.DisableCustomTagByPipelineStageType(&postStage, pipeline.Id) + if err != nil { + return err + } + return nil +} + +func (impl *CdPipelineConfigServiceImpl) DeleteCustomTagByPipelineStageType(pipelineStageType *repository5.PipelineStageType, pipelineId int) error { + err := impl.customTagService.DeleteCustomTagIfExists( + bean2.CustomTag{EntityKey: getEntityTypeByPipelineStageType(*pipelineStageType), + EntityValue: fmt.Sprintf("%d", pipelineId), + }) + if err != nil { + impl.logger.Errorw("error in deleting custom tag for pre stage", "err", err, "pipeline-id", pipelineId) + return err + } + return nil +} + +func (impl *CdPipelineConfigServiceImpl) DisableCustomTagByPipelineStageType(pipelineStageType *repository5.PipelineStageType, pipelineId int) error { + err := impl.customTagService.DisableCustomTagIfExist( + bean2.CustomTag{EntityKey: getEntityTypeByPipelineStageType(*pipelineStageType), + EntityValue: fmt.Sprintf("%d", pipelineId), + }) + if err != nil { + impl.logger.Errorw("error in deleting custom tag for pre stage", "err", err, "pipeline-id", pipelineId) + return err + } + return nil +} + +func (impl *CdPipelineConfigServiceImpl) SaveOrUpdateCustomTagForCDPipeline(pipeline *bean.CDPipelineConfigObject) error { + customTag, err := impl.ParseCustomTagPatchRequest(pipeline) + if err != nil { + impl.logger.Errorw("err", err) + return err + } + err = impl.customTagService.CreateOrUpdateCustomTag(customTag) + if err != nil { + impl.logger.Errorw("error in creating custom tag", "err", err) + return err + } + return nil +} + +func (impl *CdPipelineConfigServiceImpl) ParseCustomTagPatchRequest(pipelineRequest *bean.CDPipelineConfigObject) (*bean2.CustomTag, error) { + entityType := getEntityTypeByPipelineStageType(*pipelineRequest.CustomTagStage) + if entityType == 0 { + return nil, fmt.Errorf("invalid stage for cd pipeline custom tag; pipelineStageType: %s ", string(*pipelineRequest.CustomTagStage)) + } + customTag := &bean2.CustomTag{ + EntityKey: entityType, + EntityValue: fmt.Sprintf("%d", pipelineRequest.Id), + TagPattern: pipelineRequest.CustomTagObject.TagPattern, + AutoIncreasingNumber: pipelineRequest.CustomTagObject.CounterX, + Metadata: "", + Enabled: pipelineRequest.EnableCustomTag, + } + return customTag, nil +} + +func getEntityTypeByPipelineStageType(pipelineStageType repository5.PipelineStageType) (customTagEntityType int) { + switch pipelineStageType { + case repository5.PIPELINE_STAGE_TYPE_PRE_CD: + customTagEntityType = bean3.EntityTypePreCD + case repository5.PIPELINE_STAGE_TYPE_POST_CD: + customTagEntityType = bean3.EntityTypePostCD + default: + customTagEntityType = bean3.EntityNull + } + return customTagEntityType } func (impl *CdPipelineConfigServiceImpl) PatchCdPipelines(cdPipelines *bean.CDPatchRequest, ctx context.Context) (*bean.CdPipelines, error) { @@ -578,6 +746,26 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf return deleteResponse, err } } + if cdPipelinePluginDeleteReq.PreDeployStage != nil { + tag := bean2.CustomTag{ + EntityKey: bean3.EntityTypePreCD, + EntityValue: strconv.Itoa(pipeline.Id), + } + err = impl.customTagService.DeleteCustomTagIfExists(tag) + if err != nil { + impl.logger.Errorw("error in deleting custom tag for pre-cd stage", "Err", err, "cd-pipeline-id", pipeline.Id) + } + } + if cdPipelinePluginDeleteReq.PostDeployStage != nil { + tag := bean2.CustomTag{ + EntityKey: bean3.EntityTypePostCD, + EntityValue: strconv.Itoa(pipeline.Id), + } + err = impl.customTagService.DeleteCustomTagIfExists(tag) + if err != nil { + impl.logger.Errorw("error in deleting custom tag for pre-cd stage", "Err", err, "cd-pipeline-id", pipeline.Id) + } + } //delete app from argo cd, if created if pipeline.DeploymentAppCreated == true { deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) @@ -778,6 +966,34 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesForApp(appId int) (cdPipe deploymentTemplate = item.Strategy } } + var customTag *bean.CustomTagData + var customTagStage repository5.PipelineStageType + var customTagEnabled bool + customTagPreCD, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypePreCD, strconv.Itoa(dbPipeline.Id)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom Tag precd") + return nil, err + } + customTagPostCD, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypePostCD, strconv.Itoa(dbPipeline.Id)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom Tag precd") + return nil, err + } + if customTagPreCD != nil && customTagPreCD.Id > 0 { + customTag = &bean.CustomTagData{TagPattern: customTagPreCD.TagPattern, + CounterX: customTagPreCD.AutoIncreasingNumber, + Enabled: customTagPreCD.Enabled, + } + customTagStage = repository5.PIPELINE_STAGE_TYPE_PRE_CD + customTagEnabled = customTagPreCD.Enabled + } else if customTagPostCD != nil && customTagPostCD.Id > 0 { + customTag = &bean.CustomTagData{TagPattern: customTagPostCD.TagPattern, + CounterX: customTagPostCD.AutoIncreasingNumber, + Enabled: customTagPostCD.Enabled, + } + customTagStage = repository5.PIPELINE_STAGE_TYPE_POST_CD + customTagEnabled = customTagPostCD.Enabled + } pipeline := &bean.CDPipelineConfigObject{ Id: dbPipeline.Id, Name: dbPipeline.Name, @@ -801,6 +1017,9 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesForApp(appId int) (cdPipe IsVirtualEnvironment: dbPipeline.IsVirtualEnvironment, PreDeployStage: dbPipeline.PreDeployStage, PostDeployStage: dbPipeline.PostDeployStage, + CustomTagObject: customTag, + CustomTagStage: &customTagStage, + EnableCustomTag: customTagEnabled, } pipelines = append(pipelines, pipeline) } @@ -890,6 +1109,29 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesByEnvironment(request res } for _, dbPipeline := range authorizedPipelines { + var customTag *bean.CustomTagData + var customTagStage repository5.PipelineStageType + customTagPreCD, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypePreCD, strconv.Itoa(dbPipeline.Id)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom Tag precd") + return nil, err + } + customTagPostCD, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(bean3.EntityTypePostCD, strconv.Itoa(dbPipeline.Id)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom Tag precd") + return nil, err + } + if customTagPreCD != nil && customTagPreCD.Id > 0 { + customTag = &bean.CustomTagData{TagPattern: customTagPreCD.TagPattern, + CounterX: customTagPreCD.AutoIncreasingNumber, + } + customTagStage = repository5.PIPELINE_STAGE_TYPE_PRE_CD + } else if customTagPostCD != nil && customTagPostCD.Id > 0 { + customTag = &bean.CustomTagData{TagPattern: customTagPostCD.TagPattern, + CounterX: customTagPostCD.AutoIncreasingNumber, + } + customTagStage = repository5.PIPELINE_STAGE_TYPE_POST_CD + } pipeline := &bean.CDPipelineConfigObject{ Id: dbPipeline.Id, Name: dbPipeline.Name, @@ -912,6 +1154,8 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesByEnvironment(request res IsVirtualEnvironment: dbPipeline.IsVirtualEnvironment, PreDeployStage: dbPipeline.PreDeployStage, PostDeployStage: dbPipeline.PostDeployStage, + CustomTagObject: customTag, + CustomTagStage: &customTagStage, } pipelines = append(pipelines, pipeline) } @@ -1505,7 +1749,11 @@ func (impl *CdPipelineConfigServiceImpl) createCdPipeline(ctx context.Context, a } } - + // save custom tag data + err = impl.CDPipelineCustomTagDBOperations(pipeline) + if err != nil { + return pipelineId, err + } err = tx.Commit() if err != nil { return 0, err @@ -1540,7 +1788,7 @@ func (impl *CdPipelineConfigServiceImpl) updateCdPipeline(ctx context.Context, p } // Rollback tx on error. defer tx.Rollback() - err = impl.ciCdPipelineOrchestrator.UpdateCDPipeline(pipeline, userID, tx) + dbPipelineObj, err := impl.ciCdPipelineOrchestrator.UpdateCDPipeline(pipeline, userID, tx) if err != nil { impl.logger.Errorw("error in updating pipeline") return err @@ -1622,6 +1870,13 @@ func (impl *CdPipelineConfigServiceImpl) updateCdPipeline(ctx context.Context, p } } } + // update custom tag data + pipeline.Id = dbPipelineObj.Id // pipeline object is request received from FE + err = impl.CDPipelineCustomTagDBOperations(pipeline) + if err != nil { + impl.logger.Errorw("error in updating custom tag data for pipeline", "err", err) + return err + } err = tx.Commit() if err != nil { return err diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 0cd2fbbd5e..1b41718880 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -250,7 +250,7 @@ type ConfigMapSecretsResponse struct { } func parseMaterialInfo(materialInfo json.RawMessage, source string) (json.RawMessage, error) { - if source != "GOCD" && source != "CI-RUNNER" && source != "EXTERNAL" { + if source != repository.GOCD && source != repository.CI_RUNNER && source != repository.WEBHOOK && source != repository.PRE_CD && source != repository.POST_CD && source != repository.POST_CI { return nil, fmt.Errorf("datasource: %s not supported", source) } var ciMaterials []repository.CiMaterialInfo diff --git a/pkg/pipeline/StageServiceUtil.go b/pkg/pipeline/StageServiceUtil.go index b3c5f9560c..39be12c9a0 100644 --- a/pkg/pipeline/StageServiceUtil.go +++ b/pkg/pipeline/StageServiceUtil.go @@ -7,12 +7,13 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/plugin/repository" "gopkg.in/yaml.v2" "strings" ) -var globalInputVariableList = []string{DOCKER_IMAGE, DEPLOYMENT_RELEASE_ID, DEPLOYMENT_UNIQUE_ID, DEVTRON_CD_TRIGGER_TIME, DEVTRON_CD_TRIGGERED_BY, CD_PIPELINE_ENV_NAME_KEY, CD_PIPELINE_CLUSTER_NAME_KEY, APP_NAME} +var globalInputVariableList = []string{plugin.DOCKER_IMAGE, plugin.DEPLOYMENT_RELEASE_ID, plugin.DEPLOYMENT_UNIQUE_ID, plugin.DEVTRON_CD_TRIGGER_TIME, plugin.DEVTRON_CD_TRIGGERED_BY, plugin.CD_PIPELINE_ENV_NAME_KEY, plugin.CD_PIPELINE_CLUSTER_NAME_KEY, plugin.APP_NAME} func ConvertStageYamlScriptsToPipelineStageSteps(cdPipeline *bean2.CDPipelineConfigObject) (*bean2.CDPipelineConfigObject, error) { if cdPipeline.PreDeployStage == nil && len(cdPipeline.PreStage.Config) > 0 { @@ -186,11 +187,11 @@ func constructGlobalInputVariablesUsedInScript(script string) []*bean.StepVariab VariableStepIndexInPlugin: 0, ReferenceVariableStage: "", } - if inputVariable == DEVTRON_CD_TRIGGER_TIME { - stepVariable.ReferenceVariableName = CD_TRIGGER_TIME + if inputVariable == plugin.DEVTRON_CD_TRIGGER_TIME { + stepVariable.ReferenceVariableName = plugin.CD_TRIGGER_TIME } - if inputVariable == DEVTRON_CD_TRIGGERED_BY { - stepVariable.ReferenceVariableName = CD_TRIGGERED_BY + if inputVariable == plugin.DEVTRON_CD_TRIGGERED_BY { + stepVariable.ReferenceVariableName = plugin.CD_TRIGGERED_BY } inputVariables = append(inputVariables, stepVariable) } diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index edd1de40ed..23f97c12c5 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -44,15 +44,17 @@ import ( ) type CiArtifactWebhookRequest struct { - Image string `json:"image"` - ImageDigest string `json:"imageDigest"` - MaterialInfo json.RawMessage `json:"materialInfo"` - DataSource string `json:"dataSource"` - PipelineName string `json:"pipelineName"` - WorkflowId *int `json:"workflowId"` - UserId int32 `json:"userId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - FailureReason string `json:"failureReason"` + Image string `json:"image"` + ImageDigest string `json:"imageDigest"` + MaterialInfo json.RawMessage `json:"materialInfo"` + DataSource string `json:"dataSource"` + PipelineName string `json:"pipelineName"` + WorkflowId *int `json:"workflowId"` + UserId int32 `json:"userId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + FailureReason string `json:"failureReason"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` //map of registry and array of images generated by Copy container image plugin + PluginArtifactStage string `json:"pluginArtifactStage"` // at which stage of CI artifact was generated by plugin ("pre_ci/post_ci") } type WebhookService interface { @@ -155,9 +157,11 @@ func (impl WebhookServiceImpl) HandleCiStepFailedEvent(ciPipelineId int, request } go func() { - err := impl.customTagService.DeactivateImagePathReservation(savedWorkflow.ImagePathReservationId) - if err != nil { - impl.logger.Errorw("unable to deactivate impage_path_reservation ", err) + if len(savedWorkflow.ImagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(savedWorkflow.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("unable to deactivate impage_path_reservation ", err) + } } }() @@ -209,7 +213,8 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C if !imagePushedAt.IsZero() { createdOn = *imagePushedAt } - artifact := &repository.CiArtifact{ + + buildArtifact := &repository.CiArtifact{ Image: request.Image, ImageDigest: request.ImageDigest, MaterialInfo: string(materialJson), @@ -232,14 +237,42 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } if pipeline.ScanEnabled || isScanPluginConfigured { - artifact.Scanned = true - artifact.ScanEnabled = true + buildArtifact.Scanned = true + buildArtifact.ScanEnabled = true } - if err = impl.ciArtifactRepository.Save(artifact); err != nil { + if err = impl.ciArtifactRepository.Save(buildArtifact); err != nil { impl.logger.Errorw("error in saving material", "err", err) return 0, err } + var pluginArtifacts []*repository.CiArtifact + for registry, artifacts := range request.PluginRegistryArtifactDetails { + for _, image := range artifacts { + pluginArtifact := &repository.CiArtifact{ + Image: image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.PluginArtifactStage, + ComponentId: pipeline.Id, + PipelineId: pipeline.Id, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: createdOn, UpdatedOn: updatedOn}, + CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, + CredentialSourceValue: registry, + ParentCiArtifact: buildArtifact.Id, + Scanned: buildArtifact.Scanned, + ScanEnabled: buildArtifact.ScanEnabled, + } + pluginArtifacts = append(pluginArtifacts, pluginArtifact) + } + } + if len(pluginArtifacts) > 0 { + err = impl.ciArtifactRepository.SaveAll(pluginArtifacts) + if err != nil { + impl.logger.Errorw("error while saving ci artifacts", "err", err) + return 0, err + } + } + childrenCi, err := impl.ciPipelineRepository.FindByParentCiPipelineId(ciPipelineId) if err != nil && !util2.IsErrNoRows(err) { impl.logger.Errorw("error while fetching childern ci ", "err", err) @@ -254,7 +287,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C MaterialInfo: string(materialJson), DataSource: request.DataSource, PipelineId: ci.Id, - ParentCiArtifact: artifact.Id, + ParentCiArtifact: buildArtifact.Id, ScanEnabled: ci.ScanEnabled, Scanned: false, IsArtifactUploaded: request.IsArtifactUploaded, @@ -274,9 +307,12 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } } - ciArtifactArr = append(ciArtifactArr, artifact) - go impl.WriteCISuccessEvent(request, pipeline, artifact) - + if len(pluginArtifacts) == 0 { + ciArtifactArr = append(ciArtifactArr, buildArtifact) + } else { + ciArtifactArr = append(ciArtifactArr, pluginArtifacts[0]) + } + go impl.WriteCISuccessEvent(request, pipeline, buildArtifact) isCiManual := true if request.UserId == 1 { impl.logger.Debugw("Trigger (auto) by system user", "userId", request.UserId) @@ -319,7 +355,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C i += batchSize } impl.logger.Debugw("Completed: auto trigger for children Stage/CD pipelines", "Time taken", time.Since(start).Seconds()) - return artifact.Id, err + return buildArtifact.Id, err } func (impl WebhookServiceImpl) HandleExternalCiWebhook(externalCiId int, request *CiArtifactWebhookRequest, auth func(token string, projectObject string, envObject string) bool) (id int, err error) { diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index f069e82436..cb5bae209a 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -42,6 +42,7 @@ import ( bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -93,7 +94,7 @@ type WorkflowDagExecutor interface { HandleWebhookExternalCiEvent(artifact *repository.CiArtifact, triggeredBy int32, externalCiId int, auth func(email string, projectObject string, envObject string) bool) (bool, error) HandlePreStageSuccessEvent(cdStageCompleteEvent CdStageCompleteEvent) error HandleDeploymentSuccessEvent(pipelineOverride *chartConfig.PipelineOverride) error - HandlePostStageSuccessEvent(cdWorkflowId int, cdPipelineId int, triggeredBy int32) error + HandlePostStageSuccessEvent(cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error Subscribe() error TriggerPostStage(cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32, refCdWorkflowRunnerId int) error TriggerPreStage(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, artifact *repository.CiArtifact, pipeline *pipelineConfig.Pipeline, triggeredBy int32, applyAuth bool, refCdWorkflowRunnerId int) error @@ -140,8 +141,11 @@ type WorkflowDagExecutorImpl struct { pipelineStageRepository repository4.PipelineStageRepository pipelineStageService PipelineStageService config *types.CdConfig + globalPluginService plugin.GlobalPluginService - scopedVariableManager variables.ScopedVariableCMCSManager + scopedVariableManager variables.ScopedVariableCMCSManager + variableSnapshotHistoryService variables.VariableSnapshotHistoryService + pluginInputVariableParser PluginInputVariableParser deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService configMapHistoryService history2.ConfigMapHistoryService @@ -175,6 +179,7 @@ type WorkflowDagExecutorImpl struct { gitFactory *util.GitFactory acdClient application2.ServiceClient argoClientWrapperService argocdServer.ArgoClientWrapperService + customTagService CustomTagService } const kedaAutoscaling = "kedaAutoscaling" @@ -185,41 +190,30 @@ const enabled = "enabled" const replicaCount = "replicaCount" const ( - CD_PIPELINE_ENV_NAME_KEY = "CD_PIPELINE_ENV_NAME" - CD_PIPELINE_CLUSTER_NAME_KEY = "CD_PIPELINE_CLUSTER_NAME" GIT_COMMIT_HASH_PREFIX = "GIT_COMMIT_HASH" GIT_SOURCE_TYPE_PREFIX = "GIT_SOURCE_TYPE" GIT_SOURCE_VALUE_PREFIX = "GIT_SOURCE_VALUE" - GIT_METADATA = "GIT_METADATA" GIT_SOURCE_COUNT = "GIT_SOURCE_COUNT" APP_LABEL_KEY_PREFIX = "APP_LABEL_KEY" APP_LABEL_VALUE_PREFIX = "APP_LABEL_VALUE" - APP_LABEL_METADATA = "APP_LABEL_METADATA" APP_LABEL_COUNT = "APP_LABEL_COUNT" CHILD_CD_ENV_NAME_PREFIX = "CHILD_CD_ENV_NAME" CHILD_CD_CLUSTER_NAME_PREFIX = "CHILD_CD_CLUSTER_NAME" - CHILD_CD_METADATA = "CHILD_CD_METADATA" CHILD_CD_COUNT = "CHILD_CD_COUNT" - DOCKER_IMAGE = "DOCKER_IMAGE" - DEPLOYMENT_RELEASE_ID = "DEPLOYMENT_RELEASE_ID" - DEPLOYMENT_UNIQUE_ID = "DEPLOYMENT_UNIQUE_ID" - CD_TRIGGERED_BY = "CD_TRIGGERED_BY" - CD_TRIGGER_TIME = "CD_TRIGGER_TIME" - APP_NAME = "APP_NAME" - DEVTRON_CD_TRIGGERED_BY = "DEVTRON_CD_TRIGGERED_BY" - DEVTRON_CD_TRIGGER_TIME = "DEVTRON_CD_TRIGGER_TIME" + DEVTRON_SYSTEM_USER_ID = 1 ) type CdStageCompleteEvent struct { - CiProjectDetails []bean3.CiProjectDetails `json:"ciProjectDetails"` - WorkflowId int `json:"workflowId"` - WorkflowRunnerId int `json:"workflowRunnerId"` - CdPipelineId int `json:"cdPipelineId"` - TriggeredBy int32 `json:"triggeredBy"` - StageYaml string `json:"stageYaml"` - ArtifactLocation string `json:"artifactLocation"` - PipelineName string `json:"pipelineName"` - CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` + CiProjectDetails []bean3.CiProjectDetails `json:"ciProjectDetails"` + WorkflowId int `json:"workflowId"` + WorkflowRunnerId int `json:"workflowRunnerId"` + CdPipelineId int `json:"cdPipelineId"` + TriggeredBy int32 `json:"triggeredBy"` + StageYaml string `json:"stageYaml"` + ArtifactLocation string `json:"artifactLocation"` + PipelineName string `json:"pipelineName"` + CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` + PluginRegistryArtifactDetails map[string][]string `json:"PluginRegistryArtifactDetails"` } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -247,6 +241,9 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciWorkflowRepository pipelineConfig.CiWorkflowRepository, appLabelRepository pipelineConfig.AppLabelRepository, gitSensorGrpcClient gitSensorClient.Client, pipelineStageService PipelineStageService, k8sCommonService k8s.K8sCommonService, + variableSnapshotHistoryService variables.VariableSnapshotHistoryService, + globalPluginService plugin.GlobalPluginService, + pluginInputVariableParser PluginInputVariableParser, scopedVariableManager variables.ScopedVariableCMCSManager, deploymentTemplateHistoryService history2.DeploymentTemplateHistoryService, @@ -281,6 +278,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi gitFactory *util.GitFactory, acdClient application2.ServiceClient, argoClientWrapperService argocdServer.ArgoClientWrapperService, + scopedVariableService variables.ScopedVariableService, + customTagService CustomTagService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -315,6 +314,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi k8sCommonService: k8sCommonService, pipelineStageService: pipelineStageService, scopedVariableManager: scopedVariableManager, + globalPluginService: globalPluginService, + pluginInputVariableParser: pluginInputVariableParser, deploymentTemplateHistoryService: deploymentTemplateHistoryService, configMapHistoryService: configMapHistoryService, @@ -348,6 +349,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi gitFactory: gitFactory, acdClient: acdClient, argoClientWrapperService: argoClientWrapperService, + customTagService: customTagService, } config, err := types.GetCdConfig() if err != nil { @@ -394,7 +396,7 @@ func (impl *WorkflowDagExecutorImpl) Subscribe() error { } } else if wf.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { impl.logger.Debugw("received post stage success event for workflow runner ", "wfId", strconv.Itoa(wf.Id)) - err = impl.HandlePostStageSuccessEvent(wf.CdWorkflowId, cdStageCompleteEvent.CdPipelineId, cdStageCompleteEvent.TriggeredBy) + err = impl.HandlePostStageSuccessEvent(wf.CdWorkflowId, cdStageCompleteEvent.CdPipelineId, cdStageCompleteEvent.TriggeredBy, cdStageCompleteEvent.PluginRegistryArtifactDetails) if err != nil { impl.logger.Errorw("deployment success event error", "err", err) return @@ -413,7 +415,14 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(artifact *repository.C //1. get cd pipelines //2. get config //3. trigger wf/ deployment - pipelines, err := impl.pipelineRepository.FindByParentCiPipelineId(artifact.PipelineId) + var pipelineID int + if artifact.DataSource == repository.POST_CI { + pipelineID = artifact.ComponentId + } else { + // TODO: need to migrate artifact.PipelineId for dataSource="CI_RUNNER" also to component_id + pipelineID = artifact.PipelineId + } + pipelines, err := impl.pipelineRepository.FindByParentCiPipelineId(pipelineID) if err != nil { impl.logger.Errorw("error in fetching cd pipeline", "pipelineId", artifact.PipelineId, "err", err) return err @@ -565,10 +574,18 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(cdStageCompleteE if err != nil { return err } + ciArtifact, err := impl.ciArtifactRepository.Get(cdStageCompleteEvent.CiArtifactDTO.Id) + if err != nil { + return err + } + PreCDArtifacts, err := impl.SavePluginArtifacts(ciArtifact, cdStageCompleteEvent.PluginRegistryArtifactDetails, pipeline.Id, repository.PRE_CD, cdStageCompleteEvent.TriggeredBy) + if err != nil { + impl.logger.Errorw("error in saving plugin artifacts", "err", err) + return err + } if pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { - ciArtifact, err := impl.ciArtifactRepository.Get(cdStageCompleteEvent.CiArtifactDTO.Id) - if err != nil { - return err + if len(PreCDArtifacts) > 0 { + ciArtifact = PreCDArtifacts[0] // deployment will be trigger with artifact copied by plugin } cdWorkflow, err := impl.cdWorkflowRepository.FindById(cdStageCompleteEvent.WorkflowId) if err != nil { @@ -588,12 +605,63 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(cdStageCompleteE return nil } +func (impl *WorkflowDagExecutorImpl) SavePluginArtifacts(ciArtifact *repository.CiArtifact, pluginArtifactsDetail map[string][]string, pipelineId int, stage string, triggerdBy int32) ([]*repository.CiArtifact, error) { + + saveArtifacts, err := impl.ciArtifactRepository.GetArtifactsByDataSourceAndComponentId(stage, pipelineId) + if err != nil { + return nil, err + } + PipelineArtifacts := make(map[string]bool) + for _, artifact := range saveArtifacts { + PipelineArtifacts[artifact.Image] = true + } + var parentCiArtifactId int + if ciArtifact.ParentCiArtifact > 0 { + parentCiArtifactId = ciArtifact.ParentCiArtifact + } else { + parentCiArtifactId = ciArtifact.Id + } + var CDArtifacts []*repository.CiArtifact + for registry, artifacts := range pluginArtifactsDetail { + // artifacts are list of images + for _, artifact := range artifacts { + _, artifactAlreadySaved := PipelineArtifacts[artifact] + if artifactAlreadySaved { + continue + } + pluginArtifact := &repository.CiArtifact{ + Image: artifact, + ImageDigest: ciArtifact.ImageDigest, + MaterialInfo: ciArtifact.MaterialInfo, + DataSource: stage, + ComponentId: pipelineId, + CredentialsSourceType: repository.GLOBAL_CONTAINER_REGISTRY, + CredentialSourceValue: registry, + AuditLog: sql.AuditLog{ + CreatedOn: time.Now(), + CreatedBy: triggerdBy, + UpdatedOn: time.Now(), + UpdatedBy: triggerdBy, + }, + ParentCiArtifact: parentCiArtifactId, + } + CDArtifacts = append(CDArtifacts, pluginArtifact) + } + } + err = impl.ciArtifactRepository.SaveAll(CDArtifacts) + if err != nil { + impl.logger.Errorw("Error in saving artifacts metadata generated by plugin") + return CDArtifacts, err + } + return CDArtifacts, nil +} + func (impl *WorkflowDagExecutorImpl) TriggerPreStage(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, artifact *repository.CiArtifact, pipeline *pipelineConfig.Pipeline, triggeredBy int32, applyAuth bool, refCdWorkflowRunnerId int) error { //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() //in case of pre stage manual trigger auth is already applied - if applyAuth { + if applyAuth && triggeredBy != 1 { user, err := impl.user.GetById(artifact.UpdatedBy) if err != nil { impl.logger.Errorw("error in fetching user for auto pipeline", "UpdatedBy", artifact.UpdatedBy) @@ -681,13 +749,24 @@ func (impl *WorkflowDagExecutorImpl) TriggerPreStage(ctx context.Context, cdWf * return err } cdStageWorkflowRequest.StageType = types.PRE + // handling copyContainerImage plugin specific logic + imagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.PRE, artifact) + if err != nil { + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = err.Error() + _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + return err + } else { + runner.ImagePathReservationIds = imagePathReservationIds + _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + } + _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowService.SubmitWorkflow") cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() - err = impl.sendPreStageNotification(ctx, cdWf, pipeline) if err != nil { return err @@ -703,6 +782,93 @@ func (impl *WorkflowDagExecutorImpl) TriggerPreStage(ctx context.Context, cdWf * return nil } +func (impl *WorkflowDagExecutorImpl) SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { + copyContainerImagePluginId, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) + var imagePathReservationIds []int + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return imagePathReservationIds, err + } + for _, step := range cdStageWorkflowRequest.PrePostDeploySteps { + if copyContainerImagePluginId != 0 && step.RefPluginId == copyContainerImagePluginId { + var pipelineStageEntityType int + if pipelineStage == types.PRE { + pipelineStageEntityType = bean3.EntityTypePreCD + } else { + pipelineStageEntityType = bean3.EntityTypePostCD + } + customTagId := -1 + var DockerImageTag string + + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineStageEntityType, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom tag data", "err", err) + return imagePathReservationIds, err + } + + if !customTag.Enabled { + DockerImageTag = "" + } else { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + customTagDbObject, customDockerImageTag, err := impl.customTagService.GetCustomTag(pipelineStageEntityType, strconv.Itoa(pipelineId)) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching custom tag by entity key and value for CD", "err", err) + return imagePathReservationIds, err + } + if customTagDbObject != nil && customTagDbObject.Id > 0 { + customTagId = customTagDbObject.Id + } + DockerImageTag = customDockerImageTag + } + + var sourceDockerRegistryId string + if artifact.DataSource == repository.PRE_CD || artifact.DataSource == repository.POST_CD || artifact.DataSource == repository.POST_CI { + if artifact.CredentialsSourceType == repository.GLOBAL_CONTAINER_REGISTRY { + sourceDockerRegistryId = artifact.CredentialSourceValue + } + } else { + sourceDockerRegistryId = cdStageWorkflowRequest.DockerRegistryId + } + registryDestinationImageMap, registryCredentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, DockerImageTag, cdStageWorkflowRequest.CiArtifactDTO.Image, sourceDockerRegistryId) + if err != nil { + impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return imagePathReservationIds, err + } + var destinationImages []string + for _, images := range registryDestinationImageMap { + for _, image := range images { + destinationImages = append(destinationImages, image) + } + } + // fetch already saved artifacts to check if they are already present + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(destinationImages) + if err != nil { + impl.logger.Errorw("error in fetching artifacts by image path", "err", err) + return imagePathReservationIds, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return imagePathReservationIds, bean3.ErrImagePathInUse + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, registryDestinationImageMap) + if err != nil { + impl.logger.Errorw("error in reserving image", "err", err) + return imagePathReservationIds, err + } + cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap + cdStageWorkflowRequest.RegistryCredentialMap = registryCredentialMap + var pluginArtifactStage string + if pipelineStage == types.PRE { + pluginArtifactStage = repository.PRE_CD + } else { + pluginArtifactStage = repository.POST_CD + } + cdStageWorkflowRequest.PluginArtifactStage = pluginArtifactStage + } + } + return imagePathReservationIds, nil +} + func (impl *WorkflowDagExecutorImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean.CD_WORKFLOW_TYPE_PRE) if err != nil { @@ -801,6 +967,16 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE + // handling plugin specific logic + + pluginImagePathReservationIds, err := impl.SetCopyContainerImagePluginDataInWorkflowRequest(cdStageWorkflowRequest, pipeline.Id, types.POST, cdWf.CiArtifact) + if err != nil { + runner.Status = pipelineConfig.WorkflowFailed + runner.Message = err.Error() + _ = impl.cdWorkflowRepository.UpdateWorkFlowRunner(runner) + return err + } + _, err = impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) if err != nil { impl.logger.Errorw("error in submitting workflow", "err", err, "cdStageWorkflowRequest", cdStageWorkflowRequest, "pipeline", pipeline, "env", env) @@ -812,6 +988,11 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor impl.logger.Errorw("error in getting wfr by workflowId and runnerType", "err", err, "wfId", cdWf.Id) return err } + wfr.ImagePathReservationIds = pluginImagePathReservationIds + err = impl.cdWorkflowRepository.UpdateWorkFlowRunner(&wfr) + if err != nil { + impl.logger.Error("error in updating image path reservation ids in cd workflow runner", "err", "err") + } event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, &pipeline.EnvironmentId, util2.CD) impl.logger.Debugw("event Cd Post Trigger", "event", event) @@ -828,6 +1009,24 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor } return nil } + +func (impl *WorkflowDagExecutorImpl) ReserveImagesGeneratedAtPlugin(customTagId int, registryImageMap map[string][]string) ([]int, error) { + var imagePathReservationIds []int + for _, images := range registryImageMap { + for _, image := range images { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + if imagePathReservationData != nil { + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + } + } + return imagePathReservationIds, nil +} + func (impl *WorkflowDagExecutorImpl) buildArtifactLocationForS3(cdWorkflowConfig *pipelineConfig.CdWorkflowConfig, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := cdWorkflowConfig.CdArtifactLocationFormat if cdArtifactLocationFormat == "" { @@ -1107,9 +1306,9 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor extraEnvVariables := make(map[string]string) if env != nil { - extraEnvVariables[CD_PIPELINE_ENV_NAME_KEY] = env.Name + extraEnvVariables[plugin.CD_PIPELINE_ENV_NAME_KEY] = env.Name if env.Cluster != nil { - extraEnvVariables[CD_PIPELINE_CLUSTER_NAME_KEY] = env.Cluster.ClusterName + extraEnvVariables[plugin.CD_PIPELINE_CLUSTER_NAME_KEY] = env.Cluster.ClusterName } } ciWf, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(artifact.Id) @@ -1162,7 +1361,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor impl.logger.Errorw("err while marshaling git metdata", "err", err) return nil, err } - extraEnvVariables[GIT_METADATA] = string(gitMetadata) + extraEnvVariables[plugin.GIT_METADATA] = string(gitMetadata) extraEnvVariables[GIT_SOURCE_COUNT] = strconv.Itoa(len(ciWf.GitTriggers)) } @@ -1193,7 +1392,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor impl.logger.Errorw("err while marshaling childCdEnvVariables", "err", err) return nil, err } - extraEnvVariables[CHILD_CD_METADATA] = string(childCdEnvVariablesMetadata) + extraEnvVariables[plugin.CHILD_CD_METADATA] = string(childCdEnvVariablesMetadata) extraEnvVariables[CHILD_CD_COUNT] = strconv.Itoa(len(childPipelines)) } @@ -1247,7 +1446,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor impl.logger.Errorw("err while marshaling appLabelEnvVariables", "err", err) return nil, err } - extraEnvVariables[APP_LABEL_METADATA] = string(appLabelEnvVariablesMetadata) + extraEnvVariables[plugin.APP_LABEL_METADATA] = string(appLabelEnvVariablesMetadata) } } @@ -1375,7 +1574,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(pipelineOverri } else { // to trigger next pre/cd, if any // finding children cd by pipeline id - err = impl.HandlePostStageSuccessEvent(cdWorkflow.Id, pipelineOverride.PipelineId, 1) + err = impl.HandlePostStageSuccessEvent(cdWorkflow.Id, pipelineOverride.PipelineId, 1, nil) if err != nil { impl.logger.Errorw("error in triggering children cd after successful deployment event", "parentCdPipelineId", pipelineOverride.PipelineId) return err @@ -1384,7 +1583,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(pipelineOverri return nil } -func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(cdWorkflowId int, cdPipelineId int, triggeredBy int32) error { +func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(cdWorkflowId int, cdPipelineId int, triggeredBy int32, pluginRegistryImageDetails map[string][]string) error { // finding children cd by pipeline id cdPipelinesMapping, err := impl.appWorkflowRepository.FindWFCDMappingByParentCDPipelineId(cdPipelineId) if err != nil { @@ -1396,6 +1595,16 @@ func (impl *WorkflowDagExecutorImpl) HandlePostStageSuccessEvent(cdWorkflowId in impl.logger.Errorw("error in finding artifact by cd workflow id", "err", err, "cdWorkflowId", cdWorkflowId) return err } + if len(pluginRegistryImageDetails) > 0 { + PostCDArtifacts, err := impl.SavePluginArtifacts(ciArtifact, pluginRegistryImageDetails, cdPipelineId, repository.POST_CD, triggeredBy) + if err != nil { + impl.logger.Errorw("error in saving plugin artifacts", "err", err) + return err + } + if len(PostCDArtifacts) > 0 { + ciArtifact = PostCDArtifacts[0] + } + } //TODO : confirm about this logic used for applyAuth applyAuth := false if triggeredBy != 1 { @@ -1440,7 +1649,8 @@ func (impl *WorkflowDagExecutorImpl) TriggerDeployment(cdWf *pipelineConfig.CdWo //setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() - if cdWf == nil { + if cdWf == nil || (cdWf != nil && cdWf.CiArtifactId != artifact.Id) { + // cdWf != nil && cdWf.CiArtifactId != artifact.Id for auto trigger case when deployment is triggered with image generated by plugin cdWf = &pipelineConfig.CdWorkflow{ CiArtifactId: artifact.Id, PipelineId: pipeline.Id, @@ -1660,7 +1870,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne timelines = append(timelines, timeline) } - err = impl.cdWorkflowRepository.UpdateWorkFlowRunnersWithTxn(previousNonTerminalRunners, tx) + err = impl.cdWorkflowRepository.UpdateWorkFlowRunners(previousNonTerminalRunners) if err != nil { impl.logger.Errorw("error updating cd wf runner status", "err", err, "previousNonTerminalRunners", previousNonTerminalRunners) return err @@ -2716,7 +2926,7 @@ func (impl *WorkflowDagExecutorImpl) GetValuesOverrideForTrigger(overrideRequest _, span = otel.Tracer("orchestrator").Start(ctx, "dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment") // handle image pull secret if access given - mergedValues, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, pipeline.CiPipelineId, mergedValues) + mergedValues, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, artifact, pipeline.CiPipelineId, mergedValues) valuesOverrideResponse.MergedValues = string(mergedValues) span.End() if err != nil { @@ -3568,7 +3778,7 @@ func (impl *WorkflowDagExecutorImpl) mergeAndSave(envOverride *chartConfig.EnvCo _, span := otel.Tracer("orchestrator").Start(ctx, "dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment") // handle image pull secret if access given - merged, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, pipeline.CiPipelineId, merged) + merged, err = impl.dockerRegistryIpsConfigService.HandleImagePullSecretOnApplicationDeployment(envOverride.Environment, artifact, pipeline.CiPipelineId, merged) span.End() if err != nil { return 0, 0, "", err diff --git a/pkg/pipeline/bean/CustomTagService.go b/pkg/pipeline/bean/CustomTagService.go index b823de3aed..13ac0da763 100644 --- a/pkg/pipeline/bean/CustomTagService.go +++ b/pkg/pipeline/bean/CustomTagService.go @@ -5,6 +5,8 @@ import "fmt" const ( EntityNull = iota EntityTypeCiPipelineId + EntityTypePreCD + EntityTypePostCD ) const ( diff --git a/pkg/pipeline/pipelineStageVariableParser.go b/pkg/pipeline/pipelineStageVariableParser.go new file mode 100644 index 0000000000..b938e384ce --- /dev/null +++ b/pkg/pipeline/pipelineStageVariableParser.go @@ -0,0 +1,196 @@ +package pipeline + +import ( + "errors" + "fmt" + dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/plugin" + "github.com/go-pg/pg" + errors1 "github.com/juju/errors" + "go.uber.org/zap" + "strings" +) + +type copyContainerImagePluginInputVariable = string +type RefPluginName = string + +const ( + COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" + EMPTY_STRING = " " +) + +const ( + DESTINATION_INFO copyContainerImagePluginInputVariable = "DESTINATION_INFO" + SOURCE_REGISTRY_CREDENTIALS_KEY = "SOURCE_REGISTRY_CREDENTIAL" +) + +type PluginInputVariableParser interface { + HandleCopyContainerImagePluginInputVariables(inputVariables []*bean.VariableObject, dockerImageTag string, pluginTriggerImage string, sourceImageDockerRegistry string) (registryDestinationImageMap map[string][]string, registryCredentials map[string]plugin.RegistryCredentials, err error) +} + +type PluginInputVariableParserImpl struct { + logger *zap.SugaredLogger + dockerRegistryConfig DockerRegistryConfig + customTagService CustomTagService +} + +func NewPluginInputVariableParserImpl( + logger *zap.SugaredLogger, + dockerRegistryConfig DockerRegistryConfig, + customTagService CustomTagService, +) *PluginInputVariableParserImpl { + return &PluginInputVariableParserImpl{ + logger: logger, + dockerRegistryConfig: dockerRegistryConfig, + customTagService: customTagService, + } +} + +func (impl *PluginInputVariableParserImpl) HandleCopyContainerImagePluginInputVariables(inputVariables []*bean.VariableObject, + dockerImageTag string, + pluginTriggerImage string, + sourceImageDockerRegistry string) (registryDestinationImageMap map[string][]string, registryCredentials map[string]plugin.RegistryCredentials, err error) { + + var DestinationInfo string + for _, ipVariable := range inputVariables { + if ipVariable.Name == DESTINATION_INFO { + DestinationInfo = ipVariable.Value + } + } + + if len(pluginTriggerImage) == 0 { + return nil, nil, errors.New("no image provided during trigger time") + } + + if len(DestinationInfo) == 0 { + return nil, nil, errors.New("destination info now") + } + + if len(dockerImageTag) == 0 { + // case when custom tag is not configured - source image tag will be taken as docker image tag + pluginTriggerImageSplit := strings.Split(pluginTriggerImage, ":") + dockerImageTag = pluginTriggerImageSplit[len(pluginTriggerImageSplit)-1] + } + + registryRepoMapping := impl.getRegistryRepoMapping(DestinationInfo) + registryCredentials, err = impl.getRegistryDetails(registryRepoMapping, sourceImageDockerRegistry) + if err != nil { + return nil, nil, err + } + registryDestinationImageMap = impl.getRegistryDestinationImageMapping(registryRepoMapping, dockerImageTag, registryCredentials) + + err = impl.createEcrRepoIfRequired(registryCredentials, registryRepoMapping) + if err != nil { + impl.logger.Errorw("error in creating ecr repo", "err", err) + return registryDestinationImageMap, registryCredentials, err + } + + return registryDestinationImageMap, registryCredentials, nil +} + +func (impl *PluginInputVariableParserImpl) getRegistryRepoMapping(destinationInfo string) map[string][]string { + /* + creating map with registry as key and list of repositories in that registry where we need to copy image + destinationInfo format (each registry detail is separated by new line) : + | + | + */ + destinationRegistryRepositoryMap := make(map[string][]string) + destinationRegistryRepoDetails := strings.Split(destinationInfo, "\n") + for _, detail := range destinationRegistryRepoDetails { + registryRepoSplit := strings.Split(detail, "|") + registryName := strings.Trim(registryRepoSplit[0], EMPTY_STRING) + repositoryValuesSplit := strings.Split(registryRepoSplit[1], ",") + var repositories []string + for _, repositoryName := range repositoryValuesSplit { + repositoryName = strings.Trim(repositoryName, EMPTY_STRING) + repositories = append(repositories, repositoryName) + } + destinationRegistryRepositoryMap[registryName] = repositories + } + return destinationRegistryRepositoryMap +} + +func (impl *PluginInputVariableParserImpl) getRegistryDetails(destinationRegistryRepositoryMap map[string][]string, sourceRegistry string) (map[string]plugin.RegistryCredentials, error) { + registryCredentialsMap := make(map[string]plugin.RegistryCredentials) + //saving source registry credentials + sourceRegistry = strings.Trim(sourceRegistry, " ") + sourceRegistryCredentials, err := impl.getPluginRegistryCredentialsByRegistryName(sourceRegistry) + if err != nil { + return nil, err + } + registryCredentialsMap[SOURCE_REGISTRY_CREDENTIALS_KEY] = *sourceRegistryCredentials + + // saving destination registry credentials; destinationRegistryRepositoryMap -> map[registryName]= [, ] + for registry, _ := range destinationRegistryRepositoryMap { + destinationRegistryCredential, err := impl.getPluginRegistryCredentialsByRegistryName(registry) + if err != nil { + return nil, err + } + registryCredentialsMap[registry] = *destinationRegistryCredential + } + return registryCredentialsMap, nil +} + +func (impl *PluginInputVariableParserImpl) getPluginRegistryCredentialsByRegistryName(registryName string) (*plugin.RegistryCredentials, error) { + registryCredentials, err := impl.dockerRegistryConfig.FetchOneDockerAccount(registryName) + if err != nil { + impl.logger.Errorw("error in fetching registry details by registry name", "err", err) + if err == pg.ErrNoRows { + return nil, fmt.Errorf("invalid registry name: registry details not found in global container registries") + } + return nil, err + } + return &plugin.RegistryCredentials{ + RegistryType: string(registryCredentials.RegistryType), + RegistryURL: registryCredentials.RegistryURL, + Username: registryCredentials.Username, + Password: registryCredentials.Password, + AWSRegion: registryCredentials.AWSRegion, + AWSSecretAccessKey: registryCredentials.AWSSecretAccessKey, + AWSAccessKeyId: registryCredentials.AWSAccessKeyId, + }, nil +} + +func (impl *PluginInputVariableParserImpl) getRegistryDestinationImageMapping( + registryRepoMapping map[string][]string, + dockerImageTag string, + registryCredentials map[string]plugin.RegistryCredentials) map[string][]string { + + // creating map with registry as key and list of destination images in that registry + registryDestinationImageMapping := make(map[string][]string) + for registry, destinationRepositories := range registryRepoMapping { + registryCredential := registryCredentials[registry] + var destinationImages []string + for _, repo := range destinationRepositories { + destinationImage := fmt.Sprintf("%s/%s:%s", registryCredential.RegistryURL, repo, dockerImageTag) + destinationImages = append(destinationImages, destinationImage) + } + registryDestinationImageMapping[registry] = destinationImages + } + + return registryDestinationImageMapping +} + +func (impl *PluginInputVariableParserImpl) createEcrRepoIfRequired(registryCredentials map[string]plugin.RegistryCredentials, registryRepoMapping map[string][]string) error { + for registry, registryCredential := range registryCredentials { + if registryCredential.RegistryType == dockerRegistryRepository.REGISTRYTYPE_ECR { + repositories := registryRepoMapping[registry] + for _, dockerRepo := range repositories { + err := util.CreateEcrRepo(dockerRepo, registryCredential.AWSRegion, registryCredential.AWSAccessKeyId, registryCredential.AWSSecretAccessKey) + if err != nil { + if errors1.IsAlreadyExists(err) { + impl.logger.Warnw("this repo already exists!!, skipping repo creation", "repo", dockerRepo) + } else { + impl.logger.Errorw("ecr repo creation failed, it might be due to authorization or any other external "+ + "dependency. please create repo manually before triggering ci", "repo", dockerRepo, "err", err) + return err + } + } + } + } + } + return nil +} diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 2c3facfaa7..597e59330e 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -29,6 +29,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -103,29 +104,33 @@ type WorkflowRequest struct { ImageRetryCount int `json:"imageRetryCount"` ImageRetryInterval int `json:"imageRetryInterval"` // Data from CD Workflow service - WorkflowRunnerId int `json:"workflowRunnerId"` - CdPipelineId int `json:"cdPipelineId"` - StageYaml string `json:"stageYaml"` - ArtifactLocation string `json:"artifactLocation"` - CiArtifactDTO CiArtifactDTO `json:"ciArtifactDTO"` - CdImage string `json:"cdImage"` - StageType string `json:"stageType"` - CdCacheLocation string `json:"cdCacheLocation"` - CdCacheRegion string `json:"cdCacheRegion"` - WorkflowPrefixForLog string `json:"workflowPrefixForLog"` - DeploymentTriggeredBy string `json:"deploymentTriggeredBy,omitempty"` - DeploymentTriggerTime time.Time `json:"deploymentTriggerTime,omitempty"` - DeploymentReleaseCounter int `json:"deploymentReleaseCounter,omitempty"` - WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` - PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` - CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` - CiPipelineType string `json:"ciPipelineType"` - UseExternalClusterBlob bool `json:"useExternalClusterBlob"` - Type bean.WorkflowPipelineType - Pipeline *pipelineConfig.Pipeline - Env *repository.Environment - AppLabels map[string]string - Scope resourceQualifiers.Scope + WorkflowRunnerId int `json:"workflowRunnerId"` + CdPipelineId int `json:"cdPipelineId"` + StageYaml string `json:"stageYaml"` + ArtifactLocation string `json:"artifactLocation"` + CiArtifactDTO CiArtifactDTO `json:"ciArtifactDTO"` + CdImage string `json:"cdImage"` + StageType string `json:"stageType"` + CdCacheLocation string `json:"cdCacheLocation"` + CdCacheRegion string `json:"cdCacheRegion"` + WorkflowPrefixForLog string `json:"workflowPrefixForLog"` + DeploymentTriggeredBy string `json:"deploymentTriggeredBy,omitempty"` + DeploymentTriggerTime time.Time `json:"deploymentTriggerTime,omitempty"` + DeploymentReleaseCounter int `json:"deploymentReleaseCounter,omitempty"` + WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` + PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` + CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` + CiPipelineType string `json:"ciPipelineType"` + UseExternalClusterBlob bool `json:"useExternalClusterBlob"` + RegistryDestinationImageMap map[string][]string `json:"registryDestinationImageMap"` + RegistryCredentialMap map[string]plugin.RegistryCredentials `json:"registryCredentialMap"` + PluginArtifactStage string `json:"pluginArtifactStage"` + PushImageBeforePostCI bool `json:"pushImageBeforePostCI"` + Type bean.WorkflowPipelineType + Pipeline *pipelineConfig.Pipeline + Env *repository.Environment + AppLabels map[string]string + Scope resourceQualifiers.Scope } func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { diff --git a/pkg/plugin/GlobalPluginService.go b/pkg/plugin/GlobalPluginService.go index 957d7d81e0..3698c8d793 100644 --- a/pkg/plugin/GlobalPluginService.go +++ b/pkg/plugin/GlobalPluginService.go @@ -3,7 +3,6 @@ package plugin import ( "errors" "fmt" - "github.com/devtron-labs/devtron/pkg/pipeline" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/sql" @@ -21,10 +20,27 @@ type GlobalVariable struct { Type string `json:"stageType"` } +const ( + DOCKER_IMAGE = "DOCKER_IMAGE" + DEPLOYMENT_RELEASE_ID = "DEPLOYMENT_RELEASE_ID" + DEPLOYMENT_UNIQUE_ID = "DEPLOYMENT_UNIQUE_ID" + CD_TRIGGERED_BY = "CD_TRIGGERED_BY" + CD_TRIGGER_TIME = "CD_TRIGGER_TIME" + APP_NAME = "APP_NAME" + DEVTRON_CD_TRIGGERED_BY = "DEVTRON_CD_TRIGGERED_BY" + DEVTRON_CD_TRIGGER_TIME = "DEVTRON_CD_TRIGGER_TIME" + CD_PIPELINE_ENV_NAME_KEY = "CD_PIPELINE_ENV_NAME" + CD_PIPELINE_CLUSTER_NAME_KEY = "CD_PIPELINE_CLUSTER_NAME" + GIT_METADATA = "GIT_METADATA" + CHILD_CD_METADATA = "CHILD_CD_METADATA" + APP_LABEL_METADATA = "APP_LABEL_METADATA" +) + type GlobalPluginService interface { GetAllGlobalVariables() ([]*GlobalVariable, error) ListAllPlugins(stageTypeReq string) ([]*PluginListComponentDto, error) GetPluginDetailById(pluginId int) (*PluginDetailDto, error) + GetRefPluginIdByRefPluginName(pluginName string) (refPluginId int, err error) PatchPlugin(pluginDto *PluginMetadataDto, userId int32) (*PluginMetadataDto, error) GetDetailedPluginInfoByPluginId(pluginId int) (*PluginMetadataDto, error) GetAllDetailedPluginInfo() ([]*PluginMetadataDto, error) @@ -90,67 +106,67 @@ func (impl *GlobalPluginServiceImpl) GetAllGlobalVariables() ([]*GlobalVariable, Type: "ci", }, { - Name: pipeline.CD_PIPELINE_ENV_NAME_KEY, + Name: CD_PIPELINE_ENV_NAME_KEY, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "The name of the environment for which this deployment pipeline is configured.", Type: "cd", }, { - Name: pipeline.CD_PIPELINE_CLUSTER_NAME_KEY, + Name: CD_PIPELINE_CLUSTER_NAME_KEY, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "The name of the cluster to which the environment belongs for which this deployment pipeline is configured.", Type: "cd", }, { - Name: pipeline.DOCKER_IMAGE, + Name: DOCKER_IMAGE, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "Complete image name(repository+registry+tag).", Type: "cd", }, { - Name: pipeline.APP_NAME, + Name: APP_NAME, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "The name of the app this pipeline resides in.", Type: "cd", }, { - Name: pipeline.DEPLOYMENT_RELEASE_ID, + Name: DEPLOYMENT_RELEASE_ID, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "Auto-incremented counter for deployment triggers.", Type: "post-cd", }, { - Name: pipeline.DEPLOYMENT_UNIQUE_ID, + Name: DEPLOYMENT_UNIQUE_ID, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "Auto-incremented counter for deployment triggers. Counter is shared between Pre/Post/Deployment stages.", Type: "cd", }, { - Name: pipeline.CD_TRIGGERED_BY, + Name: CD_TRIGGERED_BY, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "Email-Id/Name of the user who triggered the deployment pipeline.", Type: "post-cd", }, { - Name: pipeline.CD_TRIGGER_TIME, + Name: CD_TRIGGER_TIME, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "Time when the deployment pipeline was triggered.", Type: "post-cd", }, { - Name: pipeline.GIT_METADATA, + Name: GIT_METADATA, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "GIT_METADATA consists of GIT_COMMIT_HASH, GIT_SOURCE_TYPE, GIT_SOURCE_VALUE.", Type: "cd", }, { - Name: pipeline.APP_LABEL_METADATA, + Name: APP_LABEL_METADATA, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "APP_LABEL_METADATA consists of APP_LABEL_KEY, APP_LABEL_VALUE. APP_LABEL_METADATA will only be available if workflow has External CI.", Type: "cd", }, { - Name: pipeline.CHILD_CD_METADATA, + Name: CHILD_CD_METADATA, Format: string(repository.PLUGIN_VARIABLE_FORMAT_TYPE_STRING), Description: "CHILD_CD_METADATA consists of CHILD_CD_ENV_NAME, CHILD_CD_CLUSTER_NAME. CHILD_CD_METADATA will only be available if this CD pipeline has a Child CD pipeline.", Type: "cd", @@ -322,6 +338,18 @@ func getVariableDto(pluginVariable *repository.PluginStepVariable) *PluginVariab } } +func (impl *GlobalPluginServiceImpl) GetRefPluginIdByRefPluginName(pluginName string) (refPluginId int, err error) { + pluginMetadata, err := impl.globalPluginRepository.GetPluginByName(pluginName) + if err != nil { + impl.logger.Errorw("error in fetching plugin metadata by name", "err", err) + return 0, err + } + if pluginMetadata == nil { + return 0, nil + } + return pluginMetadata[0].Id, nil +} + func (impl *GlobalPluginServiceImpl) PatchPlugin(pluginDto *PluginMetadataDto, userId int32) (*PluginMetadataDto, error) { switch pluginDto.Action { diff --git a/pkg/plugin/bean.go b/pkg/plugin/bean.go index e319b7b1b4..6a8bcda866 100644 --- a/pkg/plugin/bean.go +++ b/pkg/plugin/bean.go @@ -117,3 +117,13 @@ type ScriptPathArgPortMapping struct { PortOnContainer int `json:"portOnContainer"` ScriptId int `json:"scriptId"` } + +type RegistryCredentials struct { + RegistryType string `json:"registryType" validate:"required"` + RegistryURL string `json:"registryURL"` + Username string `json:"username"` + Password string `json:"password"` + AWSAccessKeyId string `json:"awsAccessKeyId,omitempty"` + AWSSecretAccessKey string `json:"awsSecretAccessKey,omitempty"` + AWSRegion string `json:"awsRegion,omitempty"` +} diff --git a/scripts/sql/190_copy_container_images.down.sql b/scripts/sql/190_copy_container_images.down.sql new file mode 100644 index 0000000000..afaaf2bc6a --- /dev/null +++ b/scripts/sql/190_copy_container_images.down.sql @@ -0,0 +1,15 @@ +DELETE FROM plugin_step_variable WHERE plugin_step_id =(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Copy container image' and ps."index"=1 and ps.deleted=false); +DELETE FROM plugin_step WHERE plugin_id=(SELECT id FROM plugin_metadata WHERE name='Copy container image'); +DELETE FROM plugin_stage_mapping WHERE plugin_id =(SELECT id FROM plugin_metadata WHERE name='Copy container image'); +DELETE FROM pipeline_stage_step_variable WHERE pipeline_stage_step_id in (SELECT id FROM pipeline_stage_step where ref_plugin_id =(SELECT id from plugin_metadata WHERE name ='Copy container image')); +DELETE FROM pipeline_stage_step where ref_plugin_id in (SELECT id from plugin_metadata WHERE name ='Copy container image'); +DELETE FROM plugin_metadata WHERE name ='Copy container image'; + + +ALTER TABLE custom_tag DROP COLUMN enabled; +ALTER TABLE ci_artifact DROP COLUMN credentials_source_type ; +ALTER TABLE ci_artifact DROP COLUMN credentials_source_value ; +ALTER TABLE ci_artifact DROP COLUMN component_id; +ALTER TABLE ci_workflow DROP COLUMN image_path_reservation_ids; +ALTER TABLE cd_workflow_runner DROP COLUMN image_path_reservation_ids; +ALTER TABLE image_path_reservation DROP CONSTRAINT image_path_reservation_custom_tag_id_fkey; \ No newline at end of file diff --git a/scripts/sql/190_copy_container_images.up.sql b/scripts/sql/190_copy_container_images.up.sql new file mode 100644 index 0000000000..c41f18e536 --- /dev/null +++ b/scripts/sql/190_copy_container_images.up.sql @@ -0,0 +1,49 @@ + +-- copy container images plugin migration script start + +INSERT INTO "plugin_metadata" ("id", "name", "description","type","icon","deleted", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_metadata'), 'Copy container image','Copy container images from the source repository to a desired repository','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/ic-plugin-copy-container-image.png','f', 'now()', 1, 'now()', 1); + +INSERT INTO "plugin_tag_relation" ("id", "tag_id", "plugin_id", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_tag_relation'),(SELECT id FROM plugin_tag WHERE name='Image source') , (SELECT id FROM plugin_metadata WHERE name='Copy container image'),'now()', 1, 'now()', 1); + +INSERT INTO "plugin_stage_mapping" ("plugin_id","stage_type","created_on", "created_by", "updated_on", "updated_by") +VALUES ((SELECT id FROM plugin_metadata WHERE name='Copy container image'),0,'now()', 1, 'now()', 1); + +INSERT INTO "plugin_pipeline_script" ("id","type","mount_directory_from_host","container_image_path","deleted","created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_pipeline_script'),'CONTAINER_IMAGE','t','quay.io/devtron/copy-container-images:7285439d-567-19519','f','now()',1,'now()',1); + +INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='Copy container image'),'Step 1','Step 1 - Copy container images','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1); + +INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value","variable_type", "value_type", "variable_step_index", "deleted", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Copy container image' and ps."index"=1 and ps.deleted=false), 'DESTINATION_INFO','STRING', + 'In case of CI, build image will be copied to registry and repository provided in DESTINATION_INFO. In case of PRE-CD/POST-CD, Image used to trigger stage will be copied in DESTINATION_INFO + Format: + | ,', true,false,'INPUT','NEW',1 ,'f','now()', 1, 'now()', 1); + +INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value","variable_type", "value_type", "variable_step_index",reference_variable_name, "deleted", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Copy container image' and ps."index"=1 and ps.deleted=false), 'DOCKER_IMAGE','STRING','',false,true,'INPUT','GLOBAL',1 ,'DOCKER_IMAGE','f','now()', 1, 'now()', 1); + +INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value","variable_type", "value_type", "variable_step_index",reference_variable_name, "deleted", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Copy container image' and ps."index"=1 and ps.deleted=false), 'REGISTRY_DESTINATION_IMAGE_MAP','STRING','map of registry name and images needed to be copied in that images',false,true,'INPUT','GLOBAL',1 ,'REGISTRY_DESTINATION_IMAGE_MAP','f','now()', 1, 'now()', 1); + +INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value","variable_type", "value_type", "variable_step_index",reference_variable_name, "deleted", "created_on", "created_by", "updated_on", "updated_by") +VALUES (nextval('id_seq_plugin_step_variable'), (SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Copy container image' and ps."index"=1 and ps.deleted=false), 'REGISTRY_CREDENTIALS','STRING','',false,true,'INPUT','GLOBAL',1 ,'REGISTRY_CREDENTIALS','f','now()', 1, 'now()', 1); + +-- copy container images plugin migration script ends + +-- requiered db changes for above scipt + +ALTER TABLE custom_tag ADD COLUMN enabled boolean default false; +ALTER TABLE ci_artifact ADD COLUMN credentials_source_type VARCHAR(50); +ALTER TABLE ci_artifact ADD COLUMN credentials_source_value VARCHAR(50); +ALTER TABLE ci_artifact ADD COLUMN component_id integer; + +ALTER TABLE ci_workflow ADD COLUMN image_path_reservation_ids integer[]; + +UPDATE ci_workflow set image_path_reservation_ids=ARRAY["image_path_reservation_id"] where image_path_reservation_id is not NULL; + +ALTER TABLE cd_workflow_runner ADD COLUMN image_path_reservation_ids integer[]; + +ALTER TABLE image_path_reservation DROP CONSTRAINT image_path_reservation_custom_tag_id_fkey; \ No newline at end of file diff --git a/wire_gen.go b/wire_gen.go index e35ab767d2..b49cdd848c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -457,7 +457,12 @@ func InitializeApp() (*App, error) { pipelineStageRepositoryImpl := repository11.NewPipelineStageRepository(sugaredLogger, db) globalPluginRepositoryImpl := repository12.NewGlobalPluginRepository(sugaredLogger, db) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl) - workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl) + globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) + dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) + imageTagRepositoryImpl := repository.NewImageTagRepository(db, sugaredLogger) + customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) + pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + workflowDagExecutorImpl := pipeline.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, pubSubClientServiceImpl, appServiceImpl, workflowServiceImpl, ciArtifactRepositoryImpl, ciPipelineRepositoryImpl, materialRepositoryImpl, pipelineOverrideRepositoryImpl, userServiceImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, enforcerImpl, enforcerUtilImpl, tokenCache, acdAuthConfig, eventSimpleFactoryImpl, eventRESTClientImpl, cvePolicyRepositoryImpl, imageScanResultRepositoryImpl, appWorkflowRepositoryImpl, prePostCdScriptHistoryServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineServiceImpl, ciTemplateRepositoryImpl, ciWorkflowRepositoryImpl, appLabelRepositoryImpl, clientImpl, pipelineStageServiceImpl, k8sCommonServiceImpl, variableSnapshotHistoryServiceImpl, globalPluginServiceImpl, pluginInputVariableParserImpl, scopedVariableCMCSManagerImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, appCrudOperationServiceImpl, pipelineConfigRepositoryImpl, dockerRegistryIpsConfigServiceImpl, chartRepositoryImpl, chartTemplateServiceImpl, pipelineStrategyHistoryRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, argoK8sClientImpl, configMapRepositoryImpl, configMapHistoryRepositoryImpl, refChartDir, helmAppServiceImpl, helmAppClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, dbMigrationConfigRepositoryImpl, mergeUtil, gitOpsConfigRepositoryImpl, gitFactory, applicationServiceClientImpl, argoClientWrapperServiceImpl, scopedVariableServiceImpl, customTagServiceImpl) deploymentGroupAppRepositoryImpl := repository.NewDeploymentGroupAppRepositoryImpl(sugaredLogger, db) deploymentGroupServiceImpl := deploymentGroup.NewDeploymentGroupServiceImpl(appRepositoryImpl, sugaredLogger, pipelineRepositoryImpl, ciPipelineRepositoryImpl, deploymentGroupRepositoryImpl, environmentRepositoryImpl, deploymentGroupAppRepositoryImpl, ciArtifactRepositoryImpl, appWorkflowRepositoryImpl, workflowDagExecutorImpl) deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, envLevelAppMetricsRepositoryImpl, appLevelMetricsRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, chartRefRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -474,8 +479,6 @@ func InitializeApp() (*App, error) { ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) - imageTagRepositoryImpl := repository.NewImageTagRepository(db, sugaredLogger) - customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appListingRepositoryImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { @@ -497,15 +500,15 @@ func InitializeApp() (*App, error) { return nil, err } devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartDeploymentServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, devtronAppCMCSServiceImpl) - appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, chartDeploymentServiceImpl, chartTemplateServiceImpl, propertiesConfigServiceImpl, appLevelMetricsRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, pipelineDeploymentServiceTypeConfig, applicationServiceClientImpl, devtronAppCMCSServiceImpl, customTagServiceImpl) + appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, workflowDagExecutorImpl, appServiceImpl, chartTemplateServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sUtil) if err != nil { return nil, err @@ -513,12 +516,11 @@ func InitializeApp() (*App, error) { blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sUtil, ciCdConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sUtil, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) - dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -526,7 +528,7 @@ func InitializeApp() (*App, error) { imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) - pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl) + pipelineConfigRestHandlerImpl := app3.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, deploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl) appWorkflowRestHandlerImpl := restHandler.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) webhookEventDataRepositoryImpl := repository.NewWebhookEventDataRepositoryImpl(db) webhookEventDataConfigImpl := pipeline.NewWebhookEventDataConfigImpl(sugaredLogger, webhookEventDataRepositoryImpl) @@ -755,7 +757,6 @@ func InitializeApp() (*App, error) { externalLinkServiceImpl := externalLink.NewExternalLinkServiceImpl(sugaredLogger, externalLinkMonitoringToolRepositoryImpl, externalLinkIdentifierMappingRepositoryImpl, externalLinkRepositoryImpl) externalLinkRestHandlerImpl := externalLink2.NewExternalLinkRestHandlerImpl(sugaredLogger, externalLinkServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl) externalLinkRouterImpl := externalLink2.NewExternalLinkRouterImpl(externalLinkRestHandlerImpl) - globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) globalPluginRestHandlerImpl := restHandler.NewGlobalPluginRestHandler(sugaredLogger, globalPluginServiceImpl, enforcerUtilImpl, enforcerImpl, pipelineBuilderImpl, userServiceImpl) globalPluginRouterImpl := router.NewGlobalPluginRouter(sugaredLogger, globalPluginRestHandlerImpl) moduleRestHandlerImpl := module2.NewModuleRestHandlerImpl(sugaredLogger, moduleServiceImpl, userServiceImpl, enforcerImpl, validate)