diff --git a/Wire.go b/Wire.go index 00bf2a8909..b7ef63e6b0 100644 --- a/Wire.go +++ b/Wire.go @@ -572,6 +572,9 @@ func InitializeApp() (*App, error) { pipeline.NewCdHandlerImpl, wire.Bind(new(pipeline.CdHandler), new(*pipeline.CdHandlerImpl)), + pipeline.NewBlobStorageConfigServiceImpl, + wire.Bind(new(pipeline.BlobStorageConfigService), new(*pipeline.BlobStorageConfigServiceImpl)), + pipeline.NewWorkflowDagExecutorImpl, wire.Bind(new(pipeline.WorkflowDagExecutor), new(*pipeline.WorkflowDagExecutorImpl)), appClone.NewAppCloneServiceImpl, diff --git a/go.mod b/go.mod index e7e5df8f94..79e2bd138a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.31 - github.com/devtron-labs/common-lib v0.0.3 + github.com/devtron-labs/common-lib v0.0.4 github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index 7cd7f50f86..8315b81d4e 100644 --- a/go.sum +++ b/go.sum @@ -293,8 +293,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.31 h1:CEMLek3JnMuH9ULsC6BHNJr+NiyGzBd4lgdSxH2IGnc= github.com/devtron-labs/authenticator v0.4.31/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.3 h1:9I1czMTU6+IeNpIPBydVibTRu8H9JQmsNi7JXTEgI/8= -github.com/devtron-labs/common-lib v0.0.3/go.mod h1:x6OdUIo2z9kxXtBfz7fJEfD4s8kiAtEmlApozOf7ECM= +github.com/devtron-labs/common-lib v0.0.4 h1:kKdhQnzy9cK+fDdcbh2JmjLGfzRF67VqtsbTHKyK47Q= +github.com/devtron-labs/common-lib v0.0.4/go.mod h1:x6OdUIo2z9kxXtBfz7fJEfD4s8kiAtEmlApozOf7ECM= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 9acaaa578d..2812aef458 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -100,6 +100,9 @@ const ( WorkflowSucceeded = "Succeeded" WorkflowTimedOut = "TimedOut" WorkflowUnableToFetchState = "UnableToFetch" + WorkflowTypeDeploy = "DEPLOY" + WorkflowTypePre = "PRE" + WorkflowTypePost = "POST" ) func (a WorkflowStatus) String() string { @@ -166,6 +169,16 @@ type CdWorkflowRunner struct { sql.AuditLog } +func (c *CdWorkflowRunner) IsExternalRun() bool { + var isExtCluster bool + if c.WorkflowType == WorkflowTypePre { + isExtCluster = c.CdWorkflow.Pipeline.RunPreStageInEnv + } else if c.WorkflowType == WorkflowTypePost { + isExtCluster = c.CdWorkflow.Pipeline.RunPostStageInEnv + } + return isExtCluster +} + type CiPipelineMaterialResponse struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId"` diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 171cd872d4..9b1477aa24 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -72,9 +72,13 @@ type CiWorkflow struct { CiBuildType string `sql:"ci_build_type"` EnvironmentId int `sql:"environment_id"` ImagePathReservationId int `sql:"image_path_reservation_id"` - ReferenceCiWorkflowId int `sql:"ref_ci_workflow_id"` - ParentCiWorkFlowId int `sql:"parent_ci_workflow_id"` - CiPipeline *CiPipeline + ReferenceCiWorkflowId int `sql:"ref_ci_workflow_id"` + ParentCiWorkFlowId int `sql:"parent_ci_workflow_id"` + CiPipeline *CiPipeline +} + +func (r *CiWorkflow) IsExternalRunInJobType() bool { + return r.EnvironmentId != 0 } type WorkflowWithArtifact struct { @@ -101,8 +105,8 @@ type WorkflowWithArtifact struct { EnvironmentId int `json:"environmentId"` EnvironmentName string `json:"environmentName"` ImagePathReservationId int `json:"image_path_reservation_id"` - RefCiWorkflowId int `json:"referenceCiWorkflowId"` - ParentCiWorkflowId int `json:"parent_ci_workflow_id"` + RefCiWorkflowId int `json:"referenceCiWorkflowId"` + ParentCiWorkflowId int `json:"parent_ci_workflow_id"` } type GitCommit struct { diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index b7aab98214..992b377518 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -180,6 +180,9 @@ type ClusterService interface { ConnectClustersInBatch(clusters []*ClusterBean, clusterExistInDb bool) ConvertClusterBeanToCluster(clusterBean *ClusterBean, userId int32) *repository.Cluster ConvertClusterBeanObjectToCluster(bean *ClusterBean) *v1alpha1.Cluster + + GetClusterConfigByClusterId(clusterId int) (*k8s.ClusterConfig, error) + GetClusterConfigByEnvId(envId int) (*k8s.ClusterConfig, error) } type ClusterServiceImpl struct { @@ -1095,3 +1098,32 @@ func (impl ClusterServiceImpl) ConvertClusterBeanObjectToCluster(bean *ClusterBe } return cl } + +func (impl ClusterServiceImpl) GetClusterConfigByClusterId(clusterId int) (*k8s.ClusterConfig, error) { + clusterBean, err := impl.FindById(clusterId) + if err != nil { + impl.logger.Errorw("error in getting clusterBean by cluster id", "err", err, "clusterId", clusterId) + return nil, err + } + rq := *clusterBean + clusterConfig, err := rq.GetClusterConfig() + if err != nil { + impl.logger.Errorw("error in getting cluster config", "err", err, "clusterId", clusterBean.Id) + return nil, err + } + return clusterConfig, nil +} + +func (impl ClusterServiceImpl) GetClusterConfigByEnvId(envId int) (*k8s.ClusterConfig, error) { + envBean, err := impl.environmentService.FindById(envId) + if err != nil { + impl.logger.Errorw("error in getting envBean by envId", "err", err, "envId", envId) + return nil, err + } + clusterConfig, err := impl.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.logger.Errorw("error in getting cluster config by env id", "err", err, "envId", envId) + return nil, err + } + return clusterConfig, nil +} diff --git a/pkg/generateManifest/DeployementTemplateService_test.go b/pkg/generateManifest/DeployementTemplateService_test.go index ad97dcc5b6..3552db8fce 100644 --- a/pkg/generateManifest/DeployementTemplateService_test.go +++ b/pkg/generateManifest/DeployementTemplateService_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" client2 "github.com/devtron-labs/authenticator/client" + "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/api/helm-app" mocks4 "github.com/devtron-labs/devtron/api/helm-app/mocks" @@ -16,7 +17,6 @@ import ( "github.com/devtron-labs/devtron/pkg/chart/mocks" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" mocks5 "github.com/devtron-labs/devtron/pkg/chartRepo/repository/mocks" - "github.com/devtron-labs/devtron/util/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "testing" diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go new file mode 100644 index 0000000000..c758ec1ac4 --- /dev/null +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -0,0 +1,99 @@ +package pipeline + +import ( + "context" + "fmt" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils/k8s" + bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "go.uber.org/zap" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strconv" +) + +type BlobStorageConfigService interface { + FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) +} +type BlobStorageConfigServiceImpl struct { + Logger *zap.SugaredLogger + k8sUtil *k8s.K8sUtil + ciCdConfig *types.CiCdConfig +} + +func NewBlobStorageConfigServiceImpl(Logger *zap.SugaredLogger, k8sUtil *k8s.K8sUtil, ciCdConfig *types.CiCdConfig) *BlobStorageConfigServiceImpl { + return &BlobStorageConfigServiceImpl{ + Logger: Logger, + k8sUtil: k8sUtil, + ciCdConfig: ciCdConfig, + } +} + +func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig *k8s.ClusterConfig, namespace string) (*bean2.CmBlobStorageConfig, *bean2.SecretBlobStorageConfig, error) { + cmConfig := &bean2.CmBlobStorageConfig{} + secretConfig := &bean2.SecretBlobStorageConfig{} + _, _, kubeClient, err := impl.k8sUtil.GetK8sConfigAndClients(clusterConfig) + if err != nil { + impl.Logger.Errorw("FetchCmAndSecretBlobConfigFromExternalCluster, error in getting kubeClient by cluster config", "err", err) + return cmConfig, secretConfig, err + } + cv1 := kubeClient.CoreV1() + ctx := context.Background() + opts := v12.GetOptions{} + cmName := impl.ciCdConfig.ExtBlobStorageCmName + secretName := impl.ciCdConfig.ExtBlobStorageSecretName + cm, err := cv1.ConfigMaps(namespace).Get(ctx, cmName, opts) + if err != nil { + impl.Logger.Errorw("error in getting config map in external cluster", "err", err, "blobStorageCmName", impl.ciCdConfig.ExtBlobStorageCmName, "clusterName", clusterConfig.ClusterName) + return cmConfig, secretConfig, err + } + secret, err := cv1.Secrets(namespace).Get(ctx, secretName, opts) + if err != nil { + impl.Logger.Errorw("error in getting secret in external cluster", "err", err, "blobStorageSecretName", impl.ciCdConfig.ExtBlobStorageSecretName, "clusterName", clusterConfig.ClusterName) + return cmConfig, secretConfig, err + } + //for IAM configured in S3 in external cluster, get logs/artifact will not work + if cm.Data != nil && secret.Data != nil { + err = cmConfig.SetCmBlobStorageConfig(cm.Data) + if err != nil { + fmt.Println("error marshalling external blob storage cm data to struct:", err) + return cmConfig, secretConfig, err + } + err = secretConfig.SetSecretBlobStorageConfig(secret.Data) + if err != nil { + fmt.Println("error marshalling external blob storage secret data to struct:", err) + return cmConfig, secretConfig, err + } + } + if cm.Data == nil { + fmt.Println("Data field not found in config map") + } + if secret.Data == nil { + fmt.Println("Data field not found in secret") + } + impl.Logger.Infow("fetching cm and secret from external cluster cloud provider", "ext cluster config: ", cmConfig) + return cmConfig, secretConfig, nil +} + +func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { + request.StorageType = cmConfig.CloudProvider + + request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + request.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + request.AwsS3BaseConfig.Passkey = types.DecodeSecretKey(secretConfig.S3SecretKey) + isInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + request.AwsS3BaseConfig.IsInSecure = isInSecure + request.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + request.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + request.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned + + request.AzureBlobBaseConfig.AccountName = cmConfig.AzureAccountName + request.AzureBlobBaseConfig.AccountKey = types.DecodeSecretKey(secretConfig.AzureAccountKey) + request.AzureBlobBaseConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + request.GcpBlobBaseConfig.CredentialFileJsonData = types.DecodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + request.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + return request +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index d858a4a18b..ad6c41bdd7 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -120,9 +120,11 @@ type CdHandlerImpl struct { k8sUtil *k8s.K8sUtil workflowService WorkflowService config *types.CdConfig + clusterService cluster.ClusterService + blobConfigStorageService BlobStorageConfigService } -func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService) *CdHandlerImpl { +func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, @@ -156,6 +158,8 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, workflowService: workflowService, + clusterService: clusterService, + blobConfigStorageService: blobConfigStorageService, } config, err := types.GetCdConfig() if err != nil { @@ -914,7 +918,7 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC return nil, nil, errors.New("logs-not-stored-in-repository") } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == WorkflowCancel { impl.Logger.Debugw("pod is not live ", "err", err) - return impl.getLogsFromRepository(pipelineId, cdWorkflow) + return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) } impl.Logger.Errorw("err on fetch workflow logs", "err", err) return nil, nil, err @@ -923,8 +927,8 @@ func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineC return logReader, cleanUp, err } -func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs") +func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) cdConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(pipelineId) if err != nil && !util.IsErrNoRows(err) { @@ -965,6 +969,19 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } + useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &cdLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + impl.Logger.Infow("s3 log req ", "req", cdLogRequest) oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) if err != nil { @@ -974,6 +991,10 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip logReader := bufio.NewReader(oldLogsStream) return logReader, cleanUp, err } +func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { + //TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db + return isExternalRun && !useBlobStorageConfigInCdWorkflow +} func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) @@ -1066,7 +1087,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - + useExternalBlobStorage := isExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) if !wfr.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") } @@ -1116,6 +1137,21 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i AwsS3BaseConfig: awsS3BaseConfig, GcpBlobBaseConfig: gcpBlobBaseConfig, } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) + return nil, err + } + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } _, numBytes, err := blobStorageService.Get(request) if err != nil { impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index e8779b7db3..0441255a1f 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -110,9 +110,11 @@ type CiHandlerImpl struct { appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig k8sCommonService k8s2.K8sCommonService + clusterService cluster.ClusterService + blobConfigStorageService BlobStorageConfigService } -func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sUtil, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository3.EnvironmentRepository, imageTaggingService ImageTaggingService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, k8sCommonService k8s2.K8sCommonService) *CiHandlerImpl { +func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sUtil, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository3.EnvironmentRepository, imageTaggingService ImageTaggingService, k8sCommonService k8s2.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, ciService: ciService, @@ -136,6 +138,8 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline customTagService: customTagService, appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, + clusterService: clusterService, + blobConfigStorageService: blobConfigStorageService, } config, err := types.GetCiConfig() if err != nil { @@ -747,7 +751,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC return nil, nil, errors.New("logs-not-stored-in-repository") } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == WorkflowCancel { impl.Logger.Errorw("err", "err", err) - return impl.getLogsFromRepository(pipelineId, ciWorkflow) + return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt) } impl.Logger.Errorw("err", "err", err) return nil, nil, err @@ -756,7 +760,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC return logReader, cleanUp, err } -func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { +func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { impl.Logger.Debug("getting historic logs") ciConfig, err := impl.ciWorkflowRepository.FindConfigByPipelineId(pipelineId) @@ -801,6 +805,18 @@ func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pip CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } + useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) if err != nil { impl.Logger.Errorw("err", "err", err) @@ -816,7 +832,7 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) return nil, err } - + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) if !ciWorkflow.BlobStorageEnabled { return nil, errors.New("logs-not-stored-in-repository") } @@ -873,6 +889,21 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i AwsS3BaseConfig: awsS3BaseConfig, GcpBlobBaseConfig: gcpBlobBaseConfig, } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByEnvId(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByEnvId, error in fetching clusterConfig by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } _, numBytes, err := blobStorageService.Get(request) if err != nil { impl.Logger.Errorw("error occurred while downloading file", "request", request) @@ -902,7 +933,6 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, return nil, err } } - if ciConfig.LogsBucket == "" { ciConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() } @@ -932,6 +962,23 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, }, } + useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByEnvId(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByEnvId, error in fetching clusterConfig by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + //from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) logs, err := ioutil.ReadFile(logsFile.Name()) if err != nil { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index f34785f4f5..88a01df693 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -626,6 +626,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. Type: bean2.CI_WORKFLOW_PIPELINE_TYPE, CiArtifactLastFetch: trigger.CiArtifactLastFetch, } + if dockerRegistry != nil { workflowRequest.DockerRegistryId = dockerRegistry.Id diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index a7927b0a50..6ed3d4feb6 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -237,8 +237,31 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *type workflowSecrets = append(workflowSecrets, *secret) } } + + //internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from + //in-cluster configured blob storage, if USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW = false and isExt = true + if workflowRequest.UseExternalClusterBlob { + workflowConfigMaps, workflowSecrets = impl.addExtBlobStorageCmCsInResponse(workflowConfigMaps, workflowSecrets) + } + return workflowConfigMaps, workflowSecrets, nil } +func (impl *WorkflowServiceImpl) addExtBlobStorageCmCsInResponse(workflowConfigMaps []bean.ConfigSecretMap, workflowSecrets []bean.ConfigSecretMap) ([]bean.ConfigSecretMap, []bean.ConfigSecretMap) { + blobDetailsConfigMap := bean.ConfigSecretMap{ + Name: impl.ciCdConfig.ExtBlobStorageCmName, + Type: "environment", + External: true, + } + workflowConfigMaps = append(workflowConfigMaps, blobDetailsConfigMap) + + blobDetailsSecret := bean.ConfigSecretMap{ + Name: impl.ciCdConfig.ExtBlobStorageSecretName, + Type: "environment", + External: true, + } + workflowSecrets = append(workflowSecrets, blobDetailsSecret) + return workflowConfigMaps, workflowSecrets +} func (impl *WorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *types.WorkflowRequest, workflowTemplate *bean3.WorkflowTemplate) { workflowTemplate.BlobStorageConfigured = workflowRequest.BlobStorageConfigured && (workflowRequest.CheckBlobStorageConfig(impl.ciCdConfig) || !workflowRequest.IsExtRun) diff --git a/pkg/pipeline/bean/CloudHelperBean.go b/pkg/pipeline/bean/CloudHelperBean.go new file mode 100644 index 0000000000..879c2afa75 --- /dev/null +++ b/pkg/pipeline/bean/CloudHelperBean.go @@ -0,0 +1,65 @@ +package bean + +import ( + "encoding/json" + "fmt" + "github.com/devtron-labs/common-lib/blob-storage" +) + +type CmBlobStorageConfig struct { + //AWS credentials + CloudProvider blob_storage.BlobStorageType `json:"BLOB_STORAGE_PROVIDER"` + S3AccessKey string `json:"BLOB_STORAGE_S3_ACCESS_KEY"` + S3Endpoint string `json:"BLOB_STORAGE_S3_ENDPOINT"` + S3EndpointInsecure string `json:"BLOB_STORAGE_S3_ENDPOINT_INSECURE"` + S3BucketVersioned string `json:"BLOB_STORAGE_S3_BUCKET_VERSIONED"` + CdDefaultBuildLogsBucket string `json:"DEFAULT_BUILD_LOGS_BUCKET" ` + CdDefaultCdLogsBucketRegion string `json:"DEFAULT_CD_LOGS_BUCKET_REGION" ` + DefaultCacheBucket string `json:"DEFAULT_CACHE_BUCKET"` + DefaultCacheBucketRegion string `json:"DEFAULT_CACHE_BUCKET_REGION"` + + //Azure credentials + AzureAccountName string `json:"AZURE_ACCOUNT_NAME"` + AzureGatewayUrl string `json:"AZURE_GATEWAY_URL"` + AzureGatewayConnectionInsecure string `json:"AZURE_GATEWAY_CONNECTION_INSECURE"` + AzureBlobContainerCiLog string `json:"AZURE_BLOB_CONTAINER_CI_LOG"` + AzureBlobContainerCiCache string `json:"AZURE_BLOB_CONTAINER_CI_CACHE"` +} + +func (c *CmBlobStorageConfig) SetCmBlobStorageConfig(cm map[string]string) error { + cmDataJson, err := json.Marshal(cm) + if err != nil { + fmt.Println("error marshalling external blob storage cm data to json:", err) + return err + } + err = json.Unmarshal(cmDataJson, &c) + if err != nil { + fmt.Println("error unmarshalling external blob storage cm json to struct:", err) + return err + } + return nil +} + +type SecretBlobStorageConfig struct { + //aws + S3SecretKey string `json:"BLOB_STORAGE_S3_SECRET_KEY"` + //gcp + GcpBlobStorageCredentialJson string `json:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` + //azure + AzureAccountKey string `json:"AZURE_ACCOUNT_KEY"` +} + +// input secret data contains encoded bytes +func (s *SecretBlobStorageConfig) SetSecretBlobStorageConfig(secret map[string][]byte) error { + cmDataJson, err := json.Marshal(secret) + if err != nil { + fmt.Println("error marshalling external blob storage secret data to json:", err) + return err + } + err = json.Unmarshal(cmDataJson, &s) + if err != nil { + fmt.Println("error unmarshalling external blob storage secret json to struct:", err) + return err + } + return nil +} diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 41a4f8ea3f..f17f5fbb65 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -1,6 +1,7 @@ package types import ( + "encoding/base64" "encoding/json" "flag" "fmt" @@ -13,6 +14,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "os/user" "path/filepath" + "strconv" "strings" "time" ) @@ -117,6 +119,8 @@ type CiCdConfig struct { BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"` InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"` BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:""` //provenance is set to false if this flag is not set + ExtBlobStorageCmName string `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"` + ExtBlobStorageSecretName string `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` } type CiConfig struct { @@ -534,3 +538,31 @@ type BuildLogRequest struct { GcpBlobBaseConfig *blob_storage.GcpBlobBaseConfig MinioEndpoint string } + +func (r *BuildLogRequest) SetBuildLogRequest(cmConfig *bean.CmBlobStorageConfig, secretConfig *bean.SecretBlobStorageConfig) { + r.CloudProvider = cmConfig.CloudProvider + r.AzureBlobConfig.AccountName = cmConfig.AzureAccountName + r.AzureBlobConfig.AccountKey = DecodeSecretKey(secretConfig.AzureAccountKey) + r.AzureBlobConfig.BlobContainerName = cmConfig.AzureBlobContainerCiLog + + r.GcpBlobBaseConfig.CredentialFileJsonData = DecodeSecretKey(secretConfig.GcpBlobStorageCredentialJson) + r.GcpBlobBaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + + r.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey + r.AwsS3BaseConfig.EndpointUrl = cmConfig.S3Endpoint + r.AwsS3BaseConfig.Passkey = DecodeSecretKey(secretConfig.S3SecretKey) + isEndpointInSecure, _ := strconv.ParseBool(cmConfig.S3EndpointInsecure) + r.AwsS3BaseConfig.IsInSecure = isEndpointInSecure + r.AwsS3BaseConfig.BucketName = cmConfig.CdDefaultBuildLogsBucket + r.AwsS3BaseConfig.Region = cmConfig.CdDefaultCdLogsBucketRegion + s3BucketVersioned, _ := strconv.ParseBool(cmConfig.S3BucketVersioned) + r.AwsS3BaseConfig.VersioningEnabled = s3BucketVersioned +} + +func DecodeSecretKey(secretKey string) string { + decodedKey, err := base64.StdEncoding.DecodeString(secretKey) + if err != nil { + fmt.Println("error decoding base64 key:", err) + } + return string(decodedKey) +} diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 78eab9d0c7..6520d3cb74 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -118,6 +118,7 @@ type WorkflowRequest struct { PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` CiPipelineType string `json:"ciPipelineType"` + UseExternalClusterBlob bool `json:"useExternalClusterBlob"` Type bean.WorkflowPipelineType Pipeline *pipelineConfig.Pipeline Env *repository.Environment @@ -150,6 +151,10 @@ func (workflowRequest *WorkflowRequest) CheckBlobStorageConfig(config *CiCdConfi } +func (workflowRequest *WorkflowRequest) updateUseExternalClusterBlob(config *CiCdConfig) { + workflowRequest.UseExternalClusterBlob = !workflowRequest.CheckBlobStorageConfig(config) && workflowRequest.IsExtRun +} + func (workflowRequest *WorkflowRequest) GetWorkflowTemplate(workflowJson []byte, config *CiCdConfig) bean.WorkflowTemplate { ttl := int32(config.BuildLogTTLValue) @@ -184,6 +189,7 @@ func (workflowRequest *WorkflowRequest) GetBlobStorageLogsKey(config *CiCdConfig func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]byte, error) { workflowRequest.updateBlobStorageLogsKey(config) workflowRequest.updateExternalRunMetadata() + workflowRequest.updateUseExternalClusterBlob(config) workflowJson, err := workflowRequest.getWorkflowJson() if err != nil { return nil, err @@ -218,10 +224,6 @@ func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdCon workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { containerEnvVariables = []v1.EnvVar{{Name: "IMAGE_SCANNER_ENDPOINT", Value: config.ImageScannerEndpoint}} } - if config.CloudProvider == BLOB_STORAGE_S3 && config.BlobStorageS3AccessKey != "" { - miniCred := []v1.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: config.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: config.BlobStorageS3SecretKey}} - containerEnvVariables = append(containerEnvVariables, miniCred...) - } eventEnv := v1.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)} inAppLoggingEnv := v1.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(workflowRequest.InAppLoggingEnabled)} containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv) diff --git a/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go b/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go index 6c41651ddf..a574f8e0e8 100644 --- a/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go +++ b/vendor/github.com/devtron-labs/common-lib/blob-storage/AwsS3Blob.go @@ -29,7 +29,9 @@ func (impl *AwsS3Blob) UploadBlob(request *BlobStorageRequest, err error) error if s3BaseConfig.Region != "" { cmdArgs = append(cmdArgs, "--region", s3BaseConfig.Region) } + command := exec.Command("aws", cmdArgs...) + setAWSEnvironmentVariables(s3BaseConfig, command) err = utils.RunCommand(command) return err } diff --git a/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go new file mode 100644 index 0000000000..70f7782641 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/blob-storage/BlobUtils.go @@ -0,0 +1,16 @@ +package blob_storage + +import ( + "fmt" + "os" + "os/exec" +) + +func setAWSEnvironmentVariables(s3Config *AwsS3BaseConfig, command *exec.Cmd) { + if s3Config.AccessKey != "" && s3Config.Passkey != "" { + command.Env = append(os.Environ(), + fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", s3Config.AccessKey), + fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", s3Config.Passkey), + ) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5b4d496363..1b3b37b6dc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -349,7 +349,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.3 +# github.com/devtron-labs/common-lib v0.0.4 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib diff --git a/wire_gen.go b/wire_gen.go index 63f82c1512..da459064b0 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -501,14 +501,15 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sUtil, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, k8sCommonServiceImpl) + blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sUtil, ciCdConfig) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sUtil, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl) gitRegistryConfigImpl := pipeline.NewGitRegistryConfigImpl(sugaredLogger, gitProviderRepositoryImpl, clientImpl) dockerRegistryConfigImpl := pipeline.NewDockerRegistryConfigImpl(sugaredLogger, helmAppServiceImpl, dockerArtifactStoreRepositoryImpl, dockerRegistryIpsConfigRepositoryImpl, ociRegistryConfigRepositoryImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository.NewLinkoutsRepositoryImpl(sugaredLogger, db) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl) deploymentEventHandlerImpl := app2.NewDeploymentEventHandlerImpl(sugaredLogger, appListingServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, enforcerUtilImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sUtil, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, appWorkflowRepositoryImpl) deploymentTemplateRepositoryImpl := repository.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger)