From 0f6c0a472022f3f562b1ba4c8d216c7619651174 Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Sun, 29 Oct 2023 18:36:52 -0400 Subject: [PATCH 1/7] Revert "Revert "refactored few type objects"" This reverts commit 926a98677790afae9af543a188f2f362485b7bc4. --- Wire.go | 3 +- api/restHandler/PubSubClientRestHandler.go | 6 +- pkg/pipeline/ArgoWorkflowExecutor_test.go | 5 +- pkg/pipeline/BuildPipelineConfigService.go | 17 +- pkg/pipeline/CdHandler.go | 17 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 7 +- pkg/pipeline/CiCdPipelineOrchestrator_test.go | 3 +- pkg/pipeline/CiHandler.go | 11 +- pkg/pipeline/CiService.go | 27 +- pkg/pipeline/SystemWorkflowExecutor_test.go | 3 +- pkg/pipeline/WebhookService.go | 5 +- pkg/pipeline/WorkflowDagExecutor.go | 32 +- pkg/pipeline/WorkflowService.go | 34 +- pkg/pipeline/WorkflowServiceIT_test.go | 15 +- pkg/pipeline/WorkflowService_test.go | 5 +- pkg/pipeline/WorkflowUtils.go | 521 ----------------- pkg/pipeline/{ => types}/CiCdConfig.go | 12 +- pkg/pipeline/types/Workflow.go | 551 ++++++++++++++++++ pkg/security/policyService.go | 6 +- wire_gen.go | 3 +- 20 files changed, 659 insertions(+), 624 deletions(-) rename pkg/pipeline/{ => types}/CiCdConfig.go (98%) create mode 100644 pkg/pipeline/types/Workflow.go diff --git a/Wire.go b/Wire.go index 7cfd730487..00bf2a8909 100644 --- a/Wire.go +++ b/Wire.go @@ -112,6 +112,7 @@ import ( history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" repository6 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" @@ -441,7 +442,7 @@ func InitializeApp() (*App, error) { repository.NewNotificationSettingsRepositoryImpl, wire.Bind(new(repository.NotificationSettingsRepository), new(*repository.NotificationSettingsRepositoryImpl)), util.IntValidator, - pipeline.GetCiCdConfig, + types.GetCiCdConfig, pipeline.NewWorkflowServiceImpl, wire.Bind(new(pipeline.WorkflowService), new(*pipeline.WorkflowServiceImpl)), diff --git a/api/restHandler/PubSubClientRestHandler.go b/api/restHandler/PubSubClientRestHandler.go index 67356aa02e..c474f1d83a 100644 --- a/api/restHandler/PubSubClientRestHandler.go +++ b/api/restHandler/PubSubClientRestHandler.go @@ -19,12 +19,12 @@ package restHandler import ( "encoding/json" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "net/http" "strings" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/restHandler/common" - "github.com/devtron-labs/devtron/pkg/pipeline" "go.uber.org/zap" ) @@ -35,7 +35,7 @@ type PubSubClientRestHandler interface { type PubSubClientRestHandlerImpl struct { pubsubClient *pubsub.PubSubClientServiceImpl logger *zap.SugaredLogger - cdConfig *pipeline.CiCdConfig + cdConfig *types.CiCdConfig } type PublishRequest struct { @@ -43,7 +43,7 @@ type PublishRequest struct { Payload json.RawMessage `json:"payload"` } -func NewPubSubClientRestHandlerImpl(pubsubClient *pubsub.PubSubClientServiceImpl, logger *zap.SugaredLogger, cdConfig *pipeline.CiCdConfig) *PubSubClientRestHandlerImpl { +func NewPubSubClientRestHandlerImpl(pubsubClient *pubsub.PubSubClientServiceImpl, logger *zap.SugaredLogger, cdConfig *types.CiCdConfig) *PubSubClientRestHandlerImpl { return &PubSubClientRestHandlerImpl{ pubsubClient: pubsubClient, logger: logger, diff --git a/pkg/pipeline/ArgoWorkflowExecutor_test.go b/pkg/pipeline/ArgoWorkflowExecutor_test.go index 2ef36b827b..8afe3d7573 100644 --- a/pkg/pipeline/ArgoWorkflowExecutor_test.go +++ b/pkg/pipeline/ArgoWorkflowExecutor_test.go @@ -9,6 +9,7 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/stretchr/testify/assert" v12 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,7 +28,7 @@ func TestExecuteWorkflow(t *testing.T) { t.SkipNow() logger, loggerErr := util.NewSugardLogger() assert.Nil(t, loggerErr) - cdConfig, err := GetCiCdConfig() + cdConfig, err := types.GetCiCdConfig() assert.Nil(t, err) workflowExecutorImpl := NewArgoWorkflowExecutorImpl(logger) @@ -260,7 +261,7 @@ func getGcpBlobStorage() *blob_storage.GcpBlobConfig { } } -func getBaseWorkflowTemplate(cdConfig *CiCdConfig) bean.WorkflowTemplate { +func getBaseWorkflowTemplate(cdConfig *types.CiCdConfig) bean.WorkflowTemplate { workflowTemplate := bean.WorkflowTemplate{} workflowTemplate.WfControllerInstanceID = "random-controller-id" diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 4219f677ef..84cf0a8934 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" + "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/sql" util2 "github.com/devtron-labs/devtron/util" @@ -109,7 +110,7 @@ type CiPipelineConfigServiceImpl struct { ciTemplateService CiTemplateService materialRepo pipelineConfig.MaterialRepository ciPipelineRepository pipelineConfig.CiPipelineRepository - ciConfig *CiCdConfig + ciConfig *types.CiCdConfig attributesService attributes.AttributesService ciWorkflowRepository pipelineConfig.CiWorkflowRepository appWorkflowRepository appWorkflow.AppWorkflowRepository @@ -137,7 +138,7 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, ciPipelineRepository pipelineConfig.CiPipelineRepository, ecrConfig *EcrConfig, appWorkflowRepository appWorkflow.AppWorkflowRepository, - ciConfig *CiCdConfig, + ciConfig *types.CiCdConfig, attributesService attributes.AttributesService, pipelineStageService PipelineStageService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, @@ -543,7 +544,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea return nil, err } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } } //map of ciPipelineId and their templateOverrideConfig @@ -705,7 +706,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi return nil, err } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } } @@ -933,7 +934,7 @@ func (impl *CiPipelineConfigServiceImpl) GetExternalCi(appId int) (ciConfig []*b return nil, err } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } externalCiConfigs := make([]*bean.ExternalCiConfig, 0) @@ -1065,7 +1066,7 @@ func (impl *CiPipelineConfigServiceImpl) GetExternalCiById(appId int, externalCi return nil, err } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } appWorkflowMappings, err := impl.appWorkflowRepository.FindWFCDMappingByExternalCiId(externalCiPipeline.Id) @@ -1604,7 +1605,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso return nil, err } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } } @@ -1891,7 +1892,7 @@ func (impl *CiPipelineConfigServiceImpl) GetExternalCiByEnvironment(request reso return nil, err } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } var externalCiPipelineIds []int diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ad22659614..4077f8bddd 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -43,6 +43,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/user" @@ -118,7 +119,7 @@ type CdHandlerImpl struct { imageTaggingService ImageTaggingService k8sUtil *k8s.K8sUtil workflowService WorkflowService - config *CdConfig + config *types.CdConfig } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository2.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, helmAppService client.HelmAppService, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, workflowDagExecutor WorkflowDagExecutor, appListingService app.AppListingService, appListingRepository repository.AppListingRepository, pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, application application.ServiceClient, argoUserService argo.ArgoUserService, deploymentEventHandler app.DeploymentEventHandler, eventClient client2.EventClient, pipelineStatusTimelineResourcesService status.PipelineStatusTimelineResourcesService, pipelineStatusSyncDetailService status.PipelineStatusSyncDetailService, pipelineStatusTimelineService status.PipelineStatusTimelineService, appService app.AppService, appStatusService app_status.AppStatusService, enforcerUtil rbac.EnforcerUtil, installedAppRepository repository3.InstalledAppRepository, installedAppVersionHistoryRepository repository3.InstalledAppVersionHistoryRepository, appRepository app2.AppRepository, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sUtil, workflowService WorkflowService) *CdHandlerImpl { @@ -156,7 +157,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, c k8sUtil: k8sUtil, workflowService: workflowService, } - config, err := GetCdConfig() + config, err := types.GetCdConfig() if err != nil { return nil } @@ -596,9 +597,9 @@ func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, userId int32) (int, return 0, err } var isExtCluster bool - if workflowRunner.WorkflowType == PRE { + if workflowRunner.WorkflowType == types.PRE { isExtCluster = pipeline.RunPreStageInEnv - } else if workflowRunner.WorkflowType == POST { + } else if workflowRunner.WorkflowType == types.POST { isExtCluster = pipeline.RunPostStageInEnv } var restConfig *rest.Config @@ -893,9 +894,9 @@ func (impl *CdHandlerImpl) GetRunningWorkflowLogs(environmentId int, pipelineId return nil, nil, err } var isExtCluster bool - if cdWorkflow.WorkflowType == PRE { + if cdWorkflow.WorkflowType == types.PRE { isExtCluster = pipeline.RunPreStageInEnv - } else if cdWorkflow.WorkflowType == POST { + } else if cdWorkflow.WorkflowType == types.POST { isExtCluster = pipeline.RunPostStageInEnv } return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) @@ -945,7 +946,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme CloudProvider: impl.config.CloudProvider, AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == BLOB_STORAGE_AZURE, + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountName: impl.config.AzureAccountName, BlobContainerName: impl.config.AzureBlobContainerCiLog, AccountKey: impl.config.AzureAccountKey, @@ -1094,7 +1095,7 @@ func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(pipelineId int, buildId i VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, } azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == BLOB_STORAGE_AZURE, + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountKey: impl.config.AzureAccountKey, AccountName: impl.config.AzureAccountName, BlobContainerName: impl.config.AzureBlobContainerCiLog, diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index aee7eedeee..f9c16a0518 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -39,6 +39,7 @@ import ( history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/user" bean3 "github.com/devtron-labs/devtron/pkg/user/bean" @@ -96,7 +97,7 @@ type CiCdPipelineOrchestratorImpl struct { ciPipelineRepository pipelineConfig.CiPipelineRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository GitSensorClient gitSensor.Client - ciConfig *CiCdConfig + ciConfig *types.CiCdConfig appWorkflowRepository appWorkflow.AppWorkflowRepository envRepository repository2.EnvironmentRepository attributesService attributes.AttributesService @@ -124,7 +125,7 @@ func NewCiCdPipelineOrchestrator( pipelineRepository pipelineConfig.PipelineRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - GitSensorClient gitSensor.Client, ciConfig *CiCdConfig, + GitSensorClient gitSensor.Client, ciConfig *types.CiCdConfig, appWorkflowRepository appWorkflow.AppWorkflowRepository, envRepository repository2.EnvironmentRepository, attributesService attributes.AttributesService, @@ -994,7 +995,7 @@ func (impl CiCdPipelineOrchestratorImpl) generateExternalCiPayload(ciPipeline *b return nil } if hostUrl != nil { - impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, ExternalCiWebhookPath) + impl.ciConfig.ExternalCiWebhookUrl = fmt.Sprintf("%s/%s", hostUrl.Value, types.ExternalCiWebhookPath) } } accessKey := keyPrefix + "." + apiKey diff --git a/pkg/pipeline/CiCdPipelineOrchestrator_test.go b/pkg/pipeline/CiCdPipelineOrchestrator_test.go index 146f7a3c23..70f32b9f48 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator_test.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator_test.go @@ -17,6 +17,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/user" "github.com/stretchr/testify/assert" "log" @@ -135,7 +136,7 @@ func InitClusterNoteService() { ciPipelineHistoryRepository := repository4.NewCiPipelineHistoryRepositoryImpl(conn, logger) ciPipelineMaterialRepository := pipelineConfig.NewCiPipelineMaterialRepositoryImpl(conn, logger) GitSensorClient, err := gitSensor.NewGitSensorClient(logger, &gitSensor.ClientConfig{}) - ciConfig := &CiCdConfig{} + ciConfig := &types.CiCdConfig{} appWorkflowRepository := appWorkflow.NewAppWorkflowRepositoryImpl(logger, conn) envRepository := repository3.NewEnvironmentRepositoryImpl(conn, logger, nil) attributesService := attributes.NewAttributesServiceImpl(logger, nil) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 385036df79..e2ee7ceaca 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -34,6 +34,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/cluster/repository" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" "io/ioutil" @@ -108,7 +109,7 @@ type CiHandlerImpl struct { imageTaggingService ImageTaggingService customTagService CustomTagService appWorkflowRepository appWorkflow.AppWorkflowRepository - config *CiConfig + config *types.CiConfig k8sCommonService k8s2.K8sCommonService } @@ -137,7 +138,7 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, } - config, err := GetCiConfig() + config, err := types.GetCiConfig() if err != nil { return nil } @@ -862,7 +863,7 @@ func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pip LogsFilePath: logsFilePath, CloudProvider: impl.config.CloudProvider, AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == BLOB_STORAGE_AZURE, + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountName: impl.config.AzureAccountName, BlobContainerName: impl.config.AzureBlobContainerCiLog, AccountKey: impl.config.AzureAccountKey, @@ -921,7 +922,7 @@ func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId i ciConfig.CiCacheRegion = impl.config.DefaultCacheBucketRegion } azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == BLOB_STORAGE_AZURE, + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountName: impl.config.AzureAccountName, BlobContainerName: impl.config.AzureBlobContainerCiLog, AccountKey: impl.config.AzureAccountKey, @@ -993,7 +994,7 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, LogsFilePath: ciWorkflow.LogLocation, CloudProvider: impl.config.CloudProvider, AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == BLOB_STORAGE_AZURE, + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountName: impl.config.AzureAccountName, BlobContainerName: impl.config.AzureBlobContainerCiLog, AccountKey: impl.config.AzureAccountKey, diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index c3dacf2cb6..2e90e9ac74 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -29,6 +29,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/user" @@ -60,7 +61,7 @@ type CiServiceImpl struct { workflowService WorkflowService ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository - ciConfig *CiConfig + ciConfig *types.CiConfig eventClient client.EventClient eventFactory client.EventFactory mergeUtil *util.MergeUtil @@ -74,7 +75,7 @@ type CiServiceImpl struct { appRepository appRepository.AppRepository customTagService CustomTagService variableSnapshotHistoryService variables.VariableSnapshotHistoryService - config *CiConfig + config *types.CiConfig } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -107,7 +108,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService variableSnapshotHistoryService: variableSnapshotHistoryService, customTagService: customTagService, } - config, err := GetCiConfig() + config, err := types.GetCiConfig() if err != nil { return nil } @@ -260,7 +261,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { return savedCiWf.Id, err } -func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *WorkflowRequest) error { +func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { ciBuildConfig := workflowRequest.CiBuildConfig if ciBuildConfig != nil { if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { @@ -302,7 +303,7 @@ func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipel return nil, isJob, nil } -func (impl *CiServiceImpl) WriteCITriggerEvent(trigger Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *WorkflowRequest) { +func (impl *CiServiceImpl) WriteCITriggerEvent(trigger Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) material := &client.MaterialTriggerInfo{} @@ -354,7 +355,7 @@ func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, return ciWorkflow, nil } -func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *WorkflowRequest) error { +func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { _, err := impl.workflowService.SubmitWorkflow(workflowRequest) if err != nil { impl.Logger.Errorw("workflow error", "err", err) @@ -385,7 +386,7 @@ func (impl *CiServiceImpl) buildDefaultArtifactLocation(ciWorkflowConfig *pipeli func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, ciPipelineScripts []*pipelineConfig.CiPipelineScript, - preCiSteps []*bean2.StepObject, postCiSteps []*bean2.StepObject, refPluginsData []*bean2.RefPluginObject) (*WorkflowRequest, error) { + preCiSteps []*bean2.StepObject, postCiSteps []*bean2.StepObject, refPluginsData []*bean2.RefPluginObject) (*types.WorkflowRequest, error) { var ciProjectDetails []bean2.CiProjectDetails commitHashes := trigger.CommitHashes for _, ciMaterial := range ciMaterials { @@ -588,7 +589,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode } - workflowRequest := &WorkflowRequest{ + workflowRequest := &types.WorkflowRequest{ WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, PipelineName: pipeline.Name, PipelineId: pipeline.Id, @@ -647,7 +648,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } switch workflowRequest.CloudProvider { - case BLOB_STORAGE_S3: + case types.BLOB_STORAGE_S3: //No AccessKey is used for uploading artifacts, instead IAM based auth is used workflowRequest.CiCacheRegion = ciWorkflowConfig.CiCacheRegion workflowRequest.CiCacheLocation = ciWorkflowConfig.CiCacheBucket @@ -667,7 +668,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, } - case BLOB_STORAGE_GCP: + case types.BLOB_STORAGE_GCP: workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, CacheBucketName: ciWorkflowConfig.CiCacheBucket, @@ -676,9 +677,9 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(ciWorkflowConfig, savedWf) workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - case BLOB_STORAGE_AZURE: + case types.BLOB_STORAGE_AZURE: workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ - Enabled: impl.config.CloudProvider == BLOB_STORAGE_AZURE, + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, AccountName: impl.config.AzureAccountName, BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, AccountKey: impl.config.AzureAccountKey, @@ -787,7 +788,7 @@ func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.Git return dockerImageTag } -func (impl *CiServiceImpl) updateCiWorkflow(request *WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { +func (impl *CiServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { ciBuildConfig := request.CiBuildConfig ciBuildType := string(ciBuildConfig.CiBuildType) savedWf.CiBuildType = ciBuildType diff --git a/pkg/pipeline/SystemWorkflowExecutor_test.go b/pkg/pipeline/SystemWorkflowExecutor_test.go index 8025ab21e5..c5a3f2b51a 100644 --- a/pkg/pipeline/SystemWorkflowExecutor_test.go +++ b/pkg/pipeline/SystemWorkflowExecutor_test.go @@ -8,6 +8,7 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/stretchr/testify/assert" v1 "k8s.io/api/batch/v1" v12 "k8s.io/api/core/v1" @@ -24,7 +25,7 @@ func TestSystemWorkflowExecute(t *testing.T) { t.SkipNow() logger, loggerErr := util.NewSugardLogger() assert.Nil(t, loggerErr) - cdConfig, err := GetCiCdConfig() + cdConfig, err := types.GetCiCdConfig() assert.Nil(t, err) runtimeConfig, err := client.GetRuntimeConfig() assert.Nil(t, err) diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index 5e1b6bc18e..b3b320537e 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" + types2 "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" @@ -61,7 +62,7 @@ type WebhookService interface { type WebhookServiceImpl struct { ciArtifactRepository repository.CiArtifactRepository - ciConfig *CiConfig + ciConfig *types2.CiConfig logger *zap.SugaredLogger ciPipelineRepository pipelineConfig.CiPipelineRepository ciWorkflowRepository pipelineConfig.CiWorkflowRepository @@ -94,7 +95,7 @@ func NewWebhookServiceImpl( ciHandler: ciHandler, customTagService: customTagService, } - config, err := GetCiConfig() + config, err := types2.GetCiConfig() if err != nil { return nil } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 1a66caafe5..8aa7c48ed0 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -41,6 +41,7 @@ import ( "github.com/devtron-labs/devtron/pkg/k8s" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -138,7 +139,7 @@ type WorkflowDagExecutorImpl struct { k8sCommonService k8s.K8sCommonService pipelineStageRepository repository4.PipelineStageRepository pipelineStageService PipelineStageService - config *CdConfig + config *types.CdConfig variableSnapshotHistoryService variables.VariableSnapshotHistoryService @@ -212,17 +213,6 @@ const ( DEVTRON_CD_TRIGGER_TIME = "DEVTRON_CD_TRIGGER_TIME" ) -type CiArtifactDTO struct { - Id int `json:"id"` - PipelineId int `json:"pipelineId"` //id of the ci pipeline from which this webhook was triggered - Image string `json:"image"` - ImageDigest string `json:"imageDigest"` - MaterialInfo string `json:"materialInfo"` //git material metadata json array string - DataSource string `json:"dataSource"` - WorkflowId *int `json:"workflowId"` - ciArtifactRepository repository.CiArtifactRepository -} - type CdStageCompleteEvent struct { CiProjectDetails []bean3.CiProjectDetails `json:"ciProjectDetails"` WorkflowId int `json:"workflowId"` @@ -384,7 +374,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi argoClientWrapperService: argoClientWrapperService, scopedVariableService: scopedVariableService, } - config, err := GetCdConfig() + config, err := types.GetCdConfig() if err != nil { return nil } @@ -715,7 +705,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPreStage(ctx context.Context, cdWf * if err != nil { return err } - cdStageWorkflowRequest.StageType = PRE + cdStageWorkflowRequest.StageType = types.PRE _, span = otel.Tracer("orchestrator").Start(ctx, "cdWorkflowService.SubmitWorkflow") cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env @@ -832,7 +822,7 @@ func (impl *WorkflowDagExecutorImpl) TriggerPostStage(cdWf *pipelineConfig.CdWor impl.logger.Errorw("error in building wfRequest", "err", err, "runner", runner, "cdWf", cdWf, "pipeline", pipeline) return err } - cdStageWorkflowRequest.StageType = POST + cdStageWorkflowRequest.StageType = types.POST cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = bean3.CD_WORKFLOW_PIPELINE_TYPE @@ -925,7 +915,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*bean3.StepObject, extraEnvVa } } } -func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32) (*WorkflowRequest, error) { +func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32) (*types.WorkflowRequest, error) { cdWorkflowConfig, err := impl.cdWorkflowRepository.FindConfigByPipelineId(cdPipeline.Id) if err != nil && !util.IsErrNoRows(err) { return nil, err @@ -1111,7 +1101,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor } } - cdStageWorkflowRequest := &WorkflowRequest{ + cdStageWorkflowRequest := &types.WorkflowRequest{ EnvironmentId: cdPipeline.EnvironmentId, AppId: cdPipeline.AppId, WorkflowId: cdWf.Id, @@ -1125,7 +1115,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor CiProjectDetails: ciProjectDetails, Namespace: runner.Namespace, ActiveDeadlineSeconds: impl.config.GetDefaultTimeout(), - CiArtifactDTO: CiArtifactDTO{ + CiArtifactDTO: types.CiArtifactDTO{ Id: artifact.Id, PipelineId: artifact.PipelineId, Image: artifact.Image, @@ -1306,7 +1296,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor } cdStageWorkflowRequest.BlobStorageConfigured = runner.BlobStorageEnabled switch cdStageWorkflowRequest.CloudProvider { - case BLOB_STORAGE_S3: + case types.BLOB_STORAGE_S3: //No AccessKey is used for uploading artifacts, instead IAM based auth is used cdStageWorkflowRequest.CdCacheRegion = cdWorkflowConfig.CdCacheRegion cdStageWorkflowRequest.CdCacheLocation = cdWorkflowConfig.CdCacheBucket @@ -1326,7 +1316,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, } - case BLOB_STORAGE_GCP: + case types.BLOB_STORAGE_GCP: cdStageWorkflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, ArtifactBucketName: impl.config.GetDefaultBuildLogsBucket(), @@ -1334,7 +1324,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor } cdStageWorkflowRequest.ArtifactLocation = impl.buildDefaultArtifactLocation(cdWorkflowConfig, cdWf, runner) cdStageWorkflowRequest.CiArtifactFileName = cdStageWorkflowRequest.ArtifactLocation - case BLOB_STORAGE_AZURE: + case types.BLOB_STORAGE_AZURE: cdStageWorkflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ Enabled: true, AccountName: impl.config.AzureAccountName, diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 5ced308973..a7927b0a50 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/repository" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "go.uber.org/zap" v12 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,7 +43,7 @@ import ( // TODO: move isCi/isJob to workflowRequest type WorkflowService interface { - SubmitWorkflow(workflowRequest *WorkflowRequest) (*unstructured.UnstructuredList, error) + SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, error) //DeleteWorkflow(wfName string, namespace string) error GetWorkflow(name string, namespace string, isExt bool, environment *repository.Environment) (*v1alpha1.Workflow, error) //ListAllWorkflows(namespace string) (*v1alpha1.WorkflowList, error) @@ -53,7 +54,7 @@ type WorkflowService interface { type WorkflowServiceImpl struct { Logger *zap.SugaredLogger config *rest.Config - ciCdConfig *CiCdConfig + ciCdConfig *types.CiCdConfig appService app.AppService envRepository repository.EnvironmentRepository globalCMCSService GlobalCMCSService @@ -65,7 +66,7 @@ type WorkflowServiceImpl struct { // TODO: Move to bean -func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository.EnvironmentRepository, ciCdConfig *CiCdConfig, +func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository.EnvironmentRepository, ciCdConfig *types.CiCdConfig, appService app.AppService, globalCMCSService GlobalCMCSService, argoWorkflowExecutor ArgoWorkflowExecutor, k8sUtil *k8s.K8sUtil, systemWorkflowExecutor SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService) (*WorkflowServiceImpl, error) { @@ -89,20 +90,13 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository. } const ( - BLOB_STORAGE_AZURE = "AZURE" - BLOB_STORAGE_S3 = "S3" - BLOB_STORAGE_GCP = "GCP" - BLOB_STORAGE_MINIO = "MINIO" CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector" - CI_NODE_PVC_ALL_ENV = "devtron.ai/ci-pvc-all" - CI_NODE_PVC_PIPELINE_PREFIX = "devtron.ai/ci-pvc" - PRE = "PRE" - POST = "POST" - preCdStage = "preCD" - postCdStage = "postCD" + + preCdStage = "preCD" + postCdStage = "postCD" ) -func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest) (*unstructured.UnstructuredList, error) { +func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, error) { workflowTemplate, err := impl.createWorkflowTemplate(workflowRequest) if err != nil { return nil, err @@ -115,7 +109,7 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest return createdWf, err } -func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *WorkflowRequest) (bean3.WorkflowTemplate, error) { +func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.WorkflowRequest) (bean3.WorkflowTemplate, error) { workflowJson, err := workflowRequest.GetWorkflowJson(impl.ciCdConfig) if err != nil { impl.Logger.Errorw("error occurred while getting workflow json", "err", err) @@ -164,7 +158,7 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *Workflo return workflowTemplate, nil } -func (impl *WorkflowServiceImpl) getClusterConfig(workflowRequest *WorkflowRequest) (*rest.Config, error) { +func (impl *WorkflowServiceImpl) getClusterConfig(workflowRequest *types.WorkflowRequest) (*rest.Config, error) { env := workflowRequest.Env if workflowRequest.IsExtRun { configMap := env.Cluster.Config @@ -186,7 +180,7 @@ func (impl *WorkflowServiceImpl) getClusterConfig(workflowRequest *WorkflowReque } -func (impl *WorkflowServiceImpl) appendGlobalCMCS(workflowRequest *WorkflowRequest) ([]bean.ConfigSecretMap, []bean.ConfigSecretMap, error) { +func (impl *WorkflowServiceImpl) appendGlobalCMCS(workflowRequest *types.WorkflowRequest) ([]bean.ConfigSecretMap, []bean.ConfigSecretMap, error) { var workflowConfigMaps []bean.ConfigSecretMap var workflowSecrets []bean.ConfigSecretMap if !workflowRequest.IsExtRun { @@ -208,7 +202,7 @@ func (impl *WorkflowServiceImpl) appendGlobalCMCS(workflowRequest *WorkflowReque return workflowConfigMaps, workflowSecrets, nil } -func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *WorkflowRequest, workflowConfigMaps []bean.ConfigSecretMap, workflowSecrets []bean.ConfigSecretMap) ([]bean.ConfigSecretMap, []bean.ConfigSecretMap, error) { +func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *types.WorkflowRequest, workflowConfigMaps []bean.ConfigSecretMap, workflowSecrets []bean.ConfigSecretMap) ([]bean.ConfigSecretMap, []bean.ConfigSecretMap, error) { pipelineLevelConfigMaps, pipelineLevelSecrets, err := workflowRequest.GetConfiguredCmCs() if err != nil { @@ -246,7 +240,7 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *Work return workflowConfigMaps, workflowSecrets, nil } -func (impl *WorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *WorkflowRequest, workflowTemplate *bean3.WorkflowTemplate) { +func (impl *WorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *types.WorkflowRequest, workflowTemplate *bean3.WorkflowTemplate) { workflowTemplate.BlobStorageConfigured = workflowRequest.BlobStorageConfigured && (workflowRequest.CheckBlobStorageConfig(impl.ciCdConfig) || !workflowRequest.IsExtRun) workflowTemplate.BlobStorageS3Config = workflowRequest.BlobStorageS3Config workflowTemplate.AzureBlobConfig = workflowRequest.AzureBlobConfig @@ -254,7 +248,7 @@ func (impl *WorkflowServiceImpl) updateBlobStorageConfig(workflowRequest *Workfl workflowTemplate.CloudStorageKey = workflowRequest.BlobStorageLogsKey } -func (impl *WorkflowServiceImpl) getAppLabelNodeSelector(workflowRequest *WorkflowRequest) map[string]string { +func (impl *WorkflowServiceImpl) getAppLabelNodeSelector(workflowRequest *types.WorkflowRequest) map[string]string { // node selector if val, ok := workflowRequest.AppLabels[CI_NODE_SELECTOR_APP_LABEL_KEY]; ok && !(workflowRequest.CheckForJob() && workflowRequest.IsExtRun) { var nodeSelectors map[string]string diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/pipeline/WorkflowServiceIT_test.go index 8a0a5f6f8b..f43f18d45d 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/pipeline/WorkflowServiceIT_test.go @@ -18,6 +18,7 @@ import ( k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/k8s/informer" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/stretchr/testify/assert" v12 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -40,7 +41,7 @@ var cmManifest4 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\ func getWorkflowServiceImpl(t *testing.T) *WorkflowServiceImpl { logger, dbConnection := getDbConnAndLoggerService(t) - ciCdConfig, _ := GetCiCdConfig() + ciCdConfig, _ := types.GetCiCdConfig() newGlobalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(logger, dbConnection) globalCMCSServiceImpl := NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) newEnvConfigOverrideRepository := chartConfig.NewEnvConfigOverrideRepository(dbConnection) @@ -67,7 +68,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { t.Run("Verify submit workflow with S3 archive logs", func(t *testing.T) { - workflowRequest := WorkflowRequest{ + workflowRequest := types.WorkflowRequest{ WorkflowNamePrefix: "1-ci", PipelineName: "ci-1-sslm", PipelineId: 2, @@ -217,7 +218,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { t.Run("Verify submit workflow without archive logs", func(t *testing.T) { - workflowRequest := WorkflowRequest{ + workflowRequest := types.WorkflowRequest{ WorkflowNamePrefix: "1-ci", PipelineName: "ci-1-sslm", PipelineId: 2, @@ -359,7 +360,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { t.Run("Verify submit workflow for Jobs", func(t *testing.T) { - workflowRequest := WorkflowRequest{ + workflowRequest := types.WorkflowRequest{ WorkflowNamePrefix: "20-pipeline-2", PipelineName: "pipeline", PipelineId: 1, @@ -556,7 +557,7 @@ func TestWorkflowServiceImpl_SubmitWorkflow(t *testing.T) { t.Run("Verify submit workflow for External Jobs", func(t *testing.T) { - workflowRequest := WorkflowRequest{ + workflowRequest := types.WorkflowRequest{ WorkflowNamePrefix: "7-pipeline-1", PipelineName: "pipeline", PipelineId: 1, @@ -855,7 +856,7 @@ func verifyTemplateResource(t *testing.T, resource *v1alpha1.ResourceTemplate, a //assert.Equal(t, manifest, resource.Manifest) } -func verifyMetadata(t *testing.T, workflowRequest WorkflowRequest, createdWf *v1alpha1.Workflow) { +func verifyMetadata(t *testing.T, workflowRequest types.WorkflowRequest, createdWf *v1alpha1.Workflow) { assert.Equal(t, map[string]string{"devtron.ai/workflow-purpose": "ci"}, createdWf.ObjectMeta.Labels) assert.Equal(t, workflowRequest.WorkflowNamePrefix+"-", createdWf.ObjectMeta.GenerateName) assert.Equal(t, workflowRequest.Namespace, createdWf.ObjectMeta.Namespace) @@ -915,7 +916,7 @@ func verifyResourceLimitAndRequest(t *testing.T, template v1alpha1.Template, wor assert.Equal(t, resourceRequest["memory"], resource.MustParse(workflowServiceImpl.ciCdConfig.CiReqMem)) } -func verifyS3BlobStorage(t *testing.T, template v1alpha1.Template, workflowServiceImpl *WorkflowServiceImpl, workflowRequest WorkflowRequest) { +func verifyS3BlobStorage(t *testing.T, template v1alpha1.Template, workflowServiceImpl *WorkflowServiceImpl, workflowRequest types.WorkflowRequest) { assert.Equal(t, true, *template.ArchiveLocation.ArchiveLogs) assert.Equal(t, workflowServiceImpl.ciCdConfig.CiDefaultBuildLogsKeyPrefix+"/"+workflowRequest.WorkflowNamePrefix, template.ArchiveLocation.S3.Key) diff --git a/pkg/pipeline/WorkflowService_test.go b/pkg/pipeline/WorkflowService_test.go index 31306dab92..218226f764 100644 --- a/pkg/pipeline/WorkflowService_test.go +++ b/pkg/pipeline/WorkflowService_test.go @@ -5,6 +5,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" bean3 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/stretchr/testify/assert" v12 "k8s.io/api/core/v1" "log" @@ -14,11 +15,11 @@ import ( func Test_getConfigMapsAndSecrets(t *testing.T) { t.SkipNow() type args struct { - workflowRequest *WorkflowRequest + workflowRequest *types.WorkflowRequest existingConfigMap *bean3.ConfigMapJson existingSecrets *bean3.ConfigSecretJson } - workflowRequest := &WorkflowRequest{ + workflowRequest := &types.WorkflowRequest{ WorkflowId: 123, } existingConfigMap := &bean3.ConfigMapJson{ diff --git a/pkg/pipeline/WorkflowUtils.go b/pkg/pipeline/WorkflowUtils.go index c020464564..51adce3d85 100644 --- a/pkg/pipeline/WorkflowUtils.go +++ b/pkg/pipeline/WorkflowUtils.go @@ -2,26 +2,17 @@ package pipeline import ( "encoding/json" - "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" - "github.com/argoproj/argo-workflows/v3/workflow/common" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - bean3 "github.com/devtron-labs/devtron/pkg/bean" - repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/util" v12 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "strconv" - "strings" - "time" ) var ArgoWorkflowOwnerRef = v1.OwnerReference{APIVersion: "argoproj.io/v1alpha1", Kind: "Workflow", Name: "{{workflow.name}}", UID: "{{workflow.uid}}", BlockOwnerDeletion: &[]bool{true}[0]} @@ -77,45 +68,6 @@ func extractVolumesFromConfigSecretMaps(isCm bool, configSecretMaps []bean2.Conf return volumes } -func UpdateContainerEnvsFromCmCs(workflowMainContainer *v12.Container, configMaps []bean2.ConfigSecretMap, secrets []bean2.ConfigSecretMap) { - for _, configMap := range configMaps { - updateContainerEnvs(true, workflowMainContainer, configMap) - } - - for _, secret := range secrets { - updateContainerEnvs(false, workflowMainContainer, secret) - } -} - -func updateContainerEnvs(isCM bool, workflowMainContainer *v12.Container, configSecretMap bean2.ConfigSecretMap) { - if configSecretMap.Type == repository.VOLUME_CONFIG { - workflowMainContainer.VolumeMounts = append(workflowMainContainer.VolumeMounts, v12.VolumeMount{ - Name: configSecretMap.Name + "-vol", - MountPath: configSecretMap.MountPath, - }) - } else if configSecretMap.Type == repository.ENVIRONMENT_CONFIG { - var envFrom v12.EnvFromSource - if isCM { - envFrom = v12.EnvFromSource{ - ConfigMapRef: &v12.ConfigMapEnvSource{ - LocalObjectReference: v12.LocalObjectReference{ - Name: configSecretMap.Name, - }, - }, - } - } else { - envFrom = v12.EnvFromSource{ - SecretRef: &v12.SecretEnvSource{ - LocalObjectReference: v12.LocalObjectReference{ - Name: configSecretMap.Name, - }, - }, - } - } - workflowMainContainer.EnvFrom = append(workflowMainContainer.EnvFrom, envFrom) - } -} - func GetConfigMapJson(configMapSecretDto ConfigMapSecretDto) (string, error) { configMapBody := GetConfigMapBody(configMapSecretDto) configMapJson, err := json.Marshal(configMapBody) @@ -285,481 +237,8 @@ func GetClientInstance(config *rest.Config, namespace string) (v1alpha12.Workflo return wfClient, nil } -type WorkflowRequest struct { - WorkflowNamePrefix string `json:"workflowNamePrefix"` - PipelineName string `json:"pipelineName"` - PipelineId int `json:"pipelineId"` - DockerImageTag string `json:"dockerImageTag"` - DockerRegistryId string `json:"dockerRegistryId"` - DockerRegistryType string `json:"dockerRegistryType"` - DockerRegistryURL string `json:"dockerRegistryURL"` - DockerConnection string `json:"dockerConnection"` - DockerCert string `json:"dockerCert"` - DockerRepository string `json:"dockerRepository"` - CheckoutPath string `json:"checkoutPath"` - DockerUsername string `json:"dockerUsername"` - DockerPassword string `json:"dockerPassword"` - AwsRegion string `json:"awsRegion"` - AccessKey string `json:"accessKey"` - SecretKey string `json:"secretKey"` - CiCacheLocation string `json:"ciCacheLocation"` - CiCacheRegion string `json:"ciCacheRegion"` - CiCacheFileName string `json:"ciCacheFileName"` - CiProjectDetails []bean.CiProjectDetails `json:"ciProjectDetails"` - ContainerResources bean.ContainerResources `json:"containerResources"` - ActiveDeadlineSeconds int64 `json:"activeDeadlineSeconds"` - CiImage string `json:"ciImage"` - Namespace string `json:"namespace"` - WorkflowId int `json:"workflowId"` - TriggeredBy int32 `json:"triggeredBy"` - CacheLimit int64 `json:"cacheLimit"` - BeforeDockerBuildScripts []*bean3.CiScript `json:"beforeDockerBuildScripts"` - AfterDockerBuildScripts []*bean3.CiScript `json:"afterDockerBuildScripts"` - CiArtifactLocation string `json:"ciArtifactLocation"` - CiArtifactBucket string `json:"ciArtifactBucket"` - CiArtifactFileName string `json:"ciArtifactFileName"` - CiArtifactRegion string `json:"ciArtifactRegion"` - ScanEnabled bool `json:"scanEnabled"` - CloudProvider blob_storage.BlobStorageType `json:"cloudProvider"` - BlobStorageConfigured bool `json:"blobStorageConfigured"` - BlobStorageS3Config *blob_storage.BlobStorageS3Config `json:"blobStorageS3Config"` - AzureBlobConfig *blob_storage.AzureBlobConfig `json:"azureBlobConfig"` - GcpBlobConfig *blob_storage.GcpBlobConfig `json:"gcpBlobConfig"` - BlobStorageLogsKey string `json:"blobStorageLogsKey"` - InAppLoggingEnabled bool `json:"inAppLoggingEnabled"` - DefaultAddressPoolBaseCidr string `json:"defaultAddressPoolBaseCidr"` - DefaultAddressPoolSize int `json:"defaultAddressPoolSize"` - PreCiSteps []*bean.StepObject `json:"preCiSteps"` - PostCiSteps []*bean.StepObject `json:"postCiSteps"` - RefPlugins []*bean.RefPluginObject `json:"refPlugins"` - AppName string `json:"appName"` - TriggerByAuthor string `json:"triggerByAuthor"` - CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig"` - CiBuildDockerMtuValue int `json:"ciBuildDockerMtuValue"` - IgnoreDockerCachePush bool `json:"ignoreDockerCachePush"` - IgnoreDockerCachePull bool `json:"ignoreDockerCachePull"` - CacheInvalidate bool `json:"cacheInvalidate"` - IsPvcMounted bool `json:"IsPvcMounted"` - ExtraEnvironmentVariables map[string]string `json:"extraEnvironmentVariables"` - EnableBuildContext bool `json:"enableBuildContext"` - AppId int `json:"appId"` - EnvironmentId int `json:"environmentId"` - OrchestratorHost string `json:"orchestratorHost"` - OrchestratorToken string `json:"orchestratorToken"` - IsExtRun bool `json:"isExtRun"` - ImageRetryCount int `json:"imageRetryCount"` - ImageRetryInterval int `json:"imageRetryInterval"` - // Data from CD Workflow service - WorkflowRunnerId int `json:"workflowRunnerId"` - CdPipelineId int `json:"cdPipelineId"` - StageYaml string `json:"stageYaml"` - ArtifactLocation string `json:"artifactLocation"` - CiArtifactDTO CiArtifactDTO `json:"ciArtifactDTO"` - CdImage string `json:"cdImage"` - StageType string `json:"stageType"` - CdCacheLocation string `json:"cdCacheLocation"` - CdCacheRegion string `json:"cdCacheRegion"` - WorkflowPrefixForLog string `json:"workflowPrefixForLog"` - DeploymentTriggeredBy string `json:"deploymentTriggeredBy,omitempty"` - DeploymentTriggerTime time.Time `json:"deploymentTriggerTime,omitempty"` - DeploymentReleaseCounter int `json:"deploymentReleaseCounter,omitempty"` - WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` - PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` - CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` - Type bean.WorkflowPipelineType - Pipeline *pipelineConfig.Pipeline - Env *repository2.Environment - AppLabels map[string]string -} - -type CiCdTriggerEvent struct { - Type string `json:"type"` - CommonWorkflowRequest *WorkflowRequest `json:"commonWorkflowRequest"` -} - -func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { - pipeline := workflowRequest.Pipeline - env := workflowRequest.Env - // Check for external in case of PRE-/POST-CD - if (workflowRequest.StageType == PRE && pipeline.RunPreStageInEnv) || (workflowRequest.StageType == POST && pipeline.RunPostStageInEnv) { - workflowRequest.IsExtRun = true - } - // Check for external in case of JOB - if env != nil && env.Id != 0 && workflowRequest.CheckForJob() { - workflowRequest.EnvironmentId = env.Id - workflowRequest.IsExtRun = true - } -} - -func (workflowRequest *WorkflowRequest) CheckBlobStorageConfig(config *CiCdConfig) bool { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return config.UseBlobStorageConfigInCiWorkflow - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return config.UseBlobStorageConfigInCdWorkflow - default: - return false - } - -} - -func (workflowRequest *WorkflowRequest) GetWorkflowTemplate(workflowJson []byte, config *CiCdConfig) bean.WorkflowTemplate { - - ttl := int32(config.BuildLogTTLValue) - workflowTemplate := bean.WorkflowTemplate{} - workflowTemplate.TTLValue = &ttl - workflowTemplate.WorkflowId = workflowRequest.WorkflowId - workflowTemplate.WorkflowRequestJson = string(workflowJson) - workflowTemplate.RefPlugins = workflowRequest.RefPlugins - workflowTemplate.ActiveDeadlineSeconds = &workflowRequest.ActiveDeadlineSeconds - workflowTemplate.Namespace = workflowRequest.Namespace - workflowTemplate.WorkflowNamePrefix = workflowRequest.WorkflowNamePrefix - if workflowRequest.Type == bean.CD_WORKFLOW_PIPELINE_TYPE { - workflowTemplate.WorkflowRunnerId = workflowRequest.WorkflowRunnerId - workflowTemplate.PrePostDeploySteps = workflowRequest.PrePostDeploySteps - } - return workflowTemplate -} - -func (workflowRequest *WorkflowRequest) checkConfigType(config *CiCdConfig) { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - config.Type = CiConfigType - case bean.CD_WORKFLOW_PIPELINE_TYPE: - config.Type = CdConfigType - } -} - -func (workflowRequest *WorkflowRequest) GetBlobStorageLogsKey(config *CiCdConfig) string { - return fmt.Sprintf("%s/%s", config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowPrefixForLog) -} - -func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]byte, error) { - workflowRequest.updateBlobStorageLogsKey(config) - workflowRequest.updateExternalRunMetadata() - workflowJson, err := workflowRequest.getWorkflowJson() - if err != nil { - return nil, err - } - return workflowJson, err -} - -func (workflowRequest *WorkflowRequest) GetEventTypeForWorkflowRequest() string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return bean.CiStage - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return bean.CdStage - default: - return "" - } -} - -func (workflowRequest *WorkflowRequest) GetWorkflowTypeForWorkflowRequest() string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return bean.CI_WORKFLOW_NAME - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return bean.CD_WORKFLOW_NAME - default: - return "" - } -} - -func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdConfig, workflowJson []byte) (containerEnvVariables []v12.EnvVar) { - if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE || - workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { - containerEnvVariables = []v12.EnvVar{{Name: "IMAGE_SCANNER_ENDPOINT", Value: config.ImageScannerEndpoint}} - } - if config.CloudProvider == BLOB_STORAGE_S3 && config.BlobStorageS3AccessKey != "" { - miniCred := []v12.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: config.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: config.BlobStorageS3SecretKey}} - containerEnvVariables = append(containerEnvVariables, miniCred...) - } - eventEnv := v12.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)} - inAppLoggingEnv := v12.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(workflowRequest.InAppLoggingEnabled)} - containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv) - return containerEnvVariables -} - -func (workflowRequest *WorkflowRequest) getPVCForWorkflowRequest() string { - var pvc string - workflowRequestType := workflowRequest.Type - if workflowRequestType == bean.CI_WORKFLOW_PIPELINE_TYPE || - workflowRequestType == bean.JOB_WORKFLOW_PIPELINE_TYPE { - pvc = workflowRequest.AppLabels[strings.ToLower(fmt.Sprintf("%s-%s", CI_NODE_PVC_PIPELINE_PREFIX, workflowRequest.PipelineName))] - if len(pvc) == 0 { - pvc = workflowRequest.AppLabels[CI_NODE_PVC_ALL_ENV] - } - if len(pvc) != 0 { - workflowRequest.IsPvcMounted = true - workflowRequest.IgnoreDockerCachePush = true - workflowRequest.IgnoreDockerCachePull = true - } - } else { - //pvc not supported for other then ci and job currently - } - return pvc -} - -func (workflowRequest *WorkflowRequest) getDefaultBuildLogsKeyPrefix(config *CiCdConfig) string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return config.CiDefaultBuildLogsKeyPrefix - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return config.CdDefaultBuildLogsKeyPrefix - default: - return "" - } -} - -func (workflowRequest *WorkflowRequest) getBlobStorageLogsPrefix() string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return workflowRequest.WorkflowNamePrefix - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return workflowRequest.WorkflowPrefixForLog - default: - return "" - } -} - -func (workflowRequest *WorkflowRequest) updateBlobStorageLogsKey(config *CiCdConfig) { - workflowRequest.BlobStorageLogsKey = fmt.Sprintf("%s/%s", workflowRequest.getDefaultBuildLogsKeyPrefix(config), workflowRequest.getBlobStorageLogsPrefix()) - workflowRequest.InAppLoggingEnabled = config.InAppLoggingEnabled || (workflowRequest.WorkflowExecutor == pipelineConfig.WORKFLOW_EXECUTOR_TYPE_SYSTEM) -} - -func (workflowRequest *WorkflowRequest) getWorkflowJson() ([]byte, error) { - eventType := workflowRequest.GetEventTypeForWorkflowRequest() - ciCdTriggerEvent := CiCdTriggerEvent{ - Type: eventType, - CommonWorkflowRequest: workflowRequest, - } - workflowJson, err := json.Marshal(&ciCdTriggerEvent) - if err != nil { - return nil, err - } - return workflowJson, err -} - -func (workflowRequest *WorkflowRequest) AddNodeConstraintsFromConfig(workflowTemplate *bean.WorkflowTemplate, config *CiCdConfig) { - nodeConstraints := workflowRequest.GetNodeConstraints(config) - if workflowRequest.Type == bean.CD_WORKFLOW_PIPELINE_TYPE && nodeConstraints.TaintKey != "" { - workflowTemplate.NodeSelector = map[string]string{nodeConstraints.TaintKey: nodeConstraints.TaintValue} - } - workflowTemplate.ServiceAccountName = nodeConstraints.ServiceAccount - if nodeConstraints.TaintKey != "" || nodeConstraints.TaintValue != "" { - workflowTemplate.Tolerations = []v12.Toleration{{Key: nodeConstraints.TaintKey, Value: nodeConstraints.TaintValue, Operator: v12.TolerationOpEqual, Effect: v12.TaintEffectNoSchedule}} - } - // In the future, we will give support for NodeSelector for job currently we need to have a node without dedicated NodeLabel to run job - if len(nodeConstraints.NodeLabel) > 0 && !(nodeConstraints.SkipNodeSelector) { - workflowTemplate.NodeSelector = nodeConstraints.NodeLabel - } - workflowTemplate.ArchiveLogs = workflowRequest.BlobStorageConfigured && !workflowRequest.InAppLoggingEnabled - workflowTemplate.RestartPolicy = v12.RestartPolicyNever - -} - -func (workflowRequest *WorkflowRequest) GetGlobalCmCsNamePrefix() string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return strconv.Itoa(workflowRequest.WorkflowId) + "-" + bean.CI_WORKFLOW_NAME - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return strconv.Itoa(workflowRequest.WorkflowRunnerId) + "-" + bean.CD_WORKFLOW_NAME - default: - return "" - } -} - -func (workflowRequest *WorkflowRequest) GetConfiguredCmCs() (map[string]bool, map[string]bool, error) { - - cdPipelineLevelConfigMaps := make(map[string]bool) - cdPipelineLevelSecrets := make(map[string]bool) - - if workflowRequest.StageType == "PRE" { - preStageConfigMapSecretsJson := workflowRequest.Pipeline.PreStageConfigMapSecretNames - preStageConfigmapSecrets := bean3.PreStageConfigMapSecretNames{} - err := json.Unmarshal([]byte(preStageConfigMapSecretsJson), &preStageConfigmapSecrets) - if err != nil { - return cdPipelineLevelConfigMaps, cdPipelineLevelSecrets, err - } - for _, cm := range preStageConfigmapSecrets.ConfigMaps { - cdPipelineLevelConfigMaps[cm] = true - } - for _, secret := range preStageConfigmapSecrets.Secrets { - cdPipelineLevelSecrets[secret] = true - } - } - if workflowRequest.StageType == "POST" { - postStageConfigMapSecretsJson := workflowRequest.Pipeline.PostStageConfigMapSecretNames - postStageConfigmapSecrets := bean3.PostStageConfigMapSecretNames{} - err := json.Unmarshal([]byte(postStageConfigMapSecretsJson), &postStageConfigmapSecrets) - if err != nil { - return cdPipelineLevelConfigMaps, cdPipelineLevelSecrets, err - } - for _, cm := range postStageConfigmapSecrets.ConfigMaps { - cdPipelineLevelConfigMaps[cm] = true - } - for _, secret := range postStageConfigmapSecrets.Secrets { - cdPipelineLevelSecrets[secret] = true - } - } - return cdPipelineLevelConfigMaps, cdPipelineLevelSecrets, nil -} - -func (workflowRequest *WorkflowRequest) GetExistingCmCsNamePrefix() string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE: - return strconv.Itoa(workflowRequest.WorkflowId) + "-" + bean.CI_WORKFLOW_NAME - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return strconv.Itoa(workflowRequest.WorkflowRunnerId) + "-" + strconv.Itoa(workflowRequest.WorkflowRunnerId) - case bean.JOB_WORKFLOW_PIPELINE_TYPE: - return strconv.Itoa(workflowRequest.WorkflowId) + "-" + bean.CI_WORKFLOW_NAME - default: - return "" - } -} - -func (workflowRequest *WorkflowRequest) CheckForJob() bool { - return workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE -} - -func (workflowRequest *WorkflowRequest) GetNodeConstraints(config *CiCdConfig) *bean.NodeConstraints { - nodeLabel, err := getNodeLabel(config, workflowRequest.Type, workflowRequest.IsExtRun) - if err != nil { - return nil - } - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return &bean.NodeConstraints{ - ServiceAccount: config.CiWorkflowServiceAccount, - TaintKey: config.CiTaintKey, - TaintValue: config.CiTaintValue, - NodeLabel: nodeLabel, - SkipNodeSelector: false, - } - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return &bean.NodeConstraints{ - ServiceAccount: config.CdWorkflowServiceAccount, - TaintKey: config.CdTaintKey, - TaintValue: config.CdTaintValue, - NodeLabel: nodeLabel, - SkipNodeSelector: false, - } - default: - return nil - } -} - -func (workflowRequest *WorkflowRequest) GetLimitReqCpuMem(config *CiCdConfig) v12.ResourceRequirements { - limitReqCpuMem := &bean.LimitReqCpuMem{} - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - limitReqCpuMem = &bean.LimitReqCpuMem{ - LimitCpu: config.CiLimitCpu, - LimitMem: config.CiLimitMem, - ReqCpu: config.CiReqCpu, - ReqMem: config.CiReqMem, - } - case bean.CD_WORKFLOW_PIPELINE_TYPE: - limitReqCpuMem = &bean.LimitReqCpuMem{ - LimitCpu: config.CdLimitCpu, - LimitMem: config.CdLimitMem, - ReqCpu: config.CdReqCpu, - ReqMem: config.CdReqMem, - } - } - return v12.ResourceRequirements{ - Limits: v12.ResourceList{ - v12.ResourceCPU: resource.MustParse(limitReqCpuMem.LimitCpu), - v12.ResourceMemory: resource.MustParse(limitReqCpuMem.LimitMem), - }, - Requests: v12.ResourceList{ - v12.ResourceCPU: resource.MustParse(limitReqCpuMem.ReqCpu), - v12.ResourceMemory: resource.MustParse(limitReqCpuMem.ReqMem), - }, - } -} - -func (workflowRequest *WorkflowRequest) getWorkflowImage() string { - switch workflowRequest.Type { - case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: - return workflowRequest.CiImage - case bean.CD_WORKFLOW_PIPELINE_TYPE: - return workflowRequest.CdImage - default: - return "" - } -} -func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdConfig, workflowJson []byte, workflowTemplate *bean.WorkflowTemplate, workflowConfigMaps []bean2.ConfigSecretMap, workflowSecrets []bean2.ConfigSecretMap) (v12.Container, error) { - privileged := true - pvc := workflowRequest.getPVCForWorkflowRequest() - containerEnvVariables := workflowRequest.getContainerEnvVariables(config, workflowJson) - workflowMainContainer := v12.Container{ - Env: containerEnvVariables, - Name: common.MainContainerName, - Image: workflowRequest.getWorkflowImage(), - SecurityContext: &v12.SecurityContext{ - Privileged: &privileged, - }, - Resources: workflowRequest.GetLimitReqCpuMem(config), - } - if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE || workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { - workflowMainContainer.Name = "" - workflowMainContainer.Ports = []v12.ContainerPort{{ - //exposed for user specific data from ci container - Name: "app-data", - ContainerPort: 9102, - }} - err := updateVolumeMountsForCi(config, workflowTemplate, &workflowMainContainer) - if err != nil { - return workflowMainContainer, err - } - } - - if len(pvc) != 0 { - buildPvcCachePath := config.BuildPvcCachePath - buildxPvcCachePath := config.BuildxPvcCachePath - defaultPvcCachePath := config.DefaultPvcCachePath - - workflowTemplate.Volumes = append(workflowTemplate.Volumes, v12.Volume{ - Name: "root-vol", - VolumeSource: v12.VolumeSource{ - PersistentVolumeClaim: &v12.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc, - ReadOnly: false, - }, - }, - }) - workflowMainContainer.VolumeMounts = append(workflowMainContainer.VolumeMounts, - v12.VolumeMount{ - Name: "root-vol", - MountPath: buildPvcCachePath, - }, - v12.VolumeMount{ - Name: "root-vol", - MountPath: buildxPvcCachePath, - }, - v12.VolumeMount{ - Name: "root-vol", - MountPath: defaultPvcCachePath, - }) - } - UpdateContainerEnvsFromCmCs(&workflowMainContainer, workflowConfigMaps, workflowSecrets) - return workflowMainContainer, nil -} - func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool { return ((status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && message == POD_DELETED_MESSAGE) && workflowRunnerStatus != WorkflowCancel } - -func updateVolumeMountsForCi(config *CiCdConfig, workflowTemplate *bean.WorkflowTemplate, workflowMainContainer *v12.Container) error { - volume, volumeMounts, err := config.GetWorkflowVolumeAndVolumeMounts() - if err != nil { - return err - } - workflowTemplate.Volumes = volume - workflowMainContainer.VolumeMounts = volumeMounts - return nil -} diff --git a/pkg/pipeline/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go similarity index 98% rename from pkg/pipeline/CiCdConfig.go rename to pkg/pipeline/types/CiCdConfig.go index 6cf984bef9..0c9be97905 100644 --- a/pkg/pipeline/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -1,4 +1,4 @@ -package pipeline +package types import ( "encoding/json" @@ -191,7 +191,7 @@ func GetCiCdConfig() (*CiCdConfig, error) { return cfg, err } -func getNodeLabel(cfg *CiCdConfig, pipelineType bean.WorkflowPipelineType, isExt bool) (map[string]string, error) { +func GetNodeLabel(cfg *CiCdConfig, pipelineType bean.WorkflowPipelineType, isExt bool) (map[string]string, error) { node := []string{} if pipelineType == bean.CI_WORKFLOW_PIPELINE_TYPE || pipelineType == bean.JOB_WORKFLOW_PIPELINE_TYPE { node = cfg.CiNodeLabelSelector @@ -464,3 +464,11 @@ func getWorkflowVolumeMounts(volumeMountForCi CiVolumeMount) v12.VolumeMount { MountPath: volumeMountForCi.ContainerMountPath, } } + +const BLOB_STORAGE_AZURE = "AZURE" + +const BLOB_STORAGE_S3 = "S3" + +const BLOB_STORAGE_GCP = "GCP" + +const BLOB_STORAGE_MINIO = "MINIO" diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go new file mode 100644 index 0000000000..0bd4a9e514 --- /dev/null +++ b/pkg/pipeline/types/Workflow.go @@ -0,0 +1,551 @@ +package types + +import ( + "encoding/json" + "fmt" + "github.com/argoproj/argo-workflows/v3/workflow/common" + "github.com/devtron-labs/common-lib/blob-storage" + bean3 "github.com/devtron-labs/devtron/api/bean" + repository2 "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "strconv" + "strings" + "time" +) + +type WorkflowRequest struct { + WorkflowNamePrefix string `json:"workflowNamePrefix"` + PipelineName string `json:"pipelineName"` + PipelineId int `json:"pipelineId"` + DockerImageTag string `json:"dockerImageTag"` + DockerRegistryId string `json:"dockerRegistryId"` + DockerRegistryType string `json:"dockerRegistryType"` + DockerRegistryURL string `json:"dockerRegistryURL"` + DockerConnection string `json:"dockerConnection"` + DockerCert string `json:"dockerCert"` + DockerRepository string `json:"dockerRepository"` + CheckoutPath string `json:"checkoutPath"` + DockerUsername string `json:"dockerUsername"` + DockerPassword string `json:"dockerPassword"` + AwsRegion string `json:"awsRegion"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + CiCacheLocation string `json:"ciCacheLocation"` + CiCacheRegion string `json:"ciCacheRegion"` + CiCacheFileName string `json:"ciCacheFileName"` + CiProjectDetails []bean.CiProjectDetails `json:"ciProjectDetails"` + ContainerResources bean.ContainerResources `json:"containerResources"` + ActiveDeadlineSeconds int64 `json:"activeDeadlineSeconds"` + CiImage string `json:"ciImage"` + Namespace string `json:"namespace"` + WorkflowId int `json:"workflowId"` + TriggeredBy int32 `json:"triggeredBy"` + CacheLimit int64 `json:"cacheLimit"` + BeforeDockerBuildScripts []*bean2.CiScript `json:"beforeDockerBuildScripts"` + AfterDockerBuildScripts []*bean2.CiScript `json:"afterDockerBuildScripts"` + CiArtifactLocation string `json:"ciArtifactLocation"` + CiArtifactBucket string `json:"ciArtifactBucket"` + CiArtifactFileName string `json:"ciArtifactFileName"` + CiArtifactRegion string `json:"ciArtifactRegion"` + ScanEnabled bool `json:"scanEnabled"` + CloudProvider blob_storage.BlobStorageType `json:"cloudProvider"` + BlobStorageConfigured bool `json:"blobStorageConfigured"` + BlobStorageS3Config *blob_storage.BlobStorageS3Config `json:"blobStorageS3Config"` + AzureBlobConfig *blob_storage.AzureBlobConfig `json:"azureBlobConfig"` + GcpBlobConfig *blob_storage.GcpBlobConfig `json:"gcpBlobConfig"` + BlobStorageLogsKey string `json:"blobStorageLogsKey"` + InAppLoggingEnabled bool `json:"inAppLoggingEnabled"` + DefaultAddressPoolBaseCidr string `json:"defaultAddressPoolBaseCidr"` + DefaultAddressPoolSize int `json:"defaultAddressPoolSize"` + PreCiSteps []*bean.StepObject `json:"preCiSteps"` + PostCiSteps []*bean.StepObject `json:"postCiSteps"` + RefPlugins []*bean.RefPluginObject `json:"refPlugins"` + AppName string `json:"appName"` + TriggerByAuthor string `json:"triggerByAuthor"` + CiBuildConfig *bean.CiBuildConfigBean `json:"ciBuildConfig"` + CiBuildDockerMtuValue int `json:"ciBuildDockerMtuValue"` + IgnoreDockerCachePush bool `json:"ignoreDockerCachePush"` + IgnoreDockerCachePull bool `json:"ignoreDockerCachePull"` + CacheInvalidate bool `json:"cacheInvalidate"` + IsPvcMounted bool `json:"IsPvcMounted"` + ExtraEnvironmentVariables map[string]string `json:"extraEnvironmentVariables"` + EnableBuildContext bool `json:"enableBuildContext"` + AppId int `json:"appId"` + EnvironmentId int `json:"environmentId"` + OrchestratorHost string `json:"orchestratorHost"` + OrchestratorToken string `json:"orchestratorToken"` + IsExtRun bool `json:"isExtRun"` + ImageRetryCount int `json:"imageRetryCount"` + ImageRetryInterval int `json:"imageRetryInterval"` + // Data from CD Workflow service + WorkflowRunnerId int `json:"workflowRunnerId"` + CdPipelineId int `json:"cdPipelineId"` + StageYaml string `json:"stageYaml"` + ArtifactLocation string `json:"artifactLocation"` + CiArtifactDTO CiArtifactDTO `json:"ciArtifactDTO"` + CdImage string `json:"cdImage"` + StageType string `json:"stageType"` + CdCacheLocation string `json:"cdCacheLocation"` + CdCacheRegion string `json:"cdCacheRegion"` + WorkflowPrefixForLog string `json:"workflowPrefixForLog"` + DeploymentTriggeredBy string `json:"deploymentTriggeredBy,omitempty"` + DeploymentTriggerTime time.Time `json:"deploymentTriggerTime,omitempty"` + DeploymentReleaseCounter int `json:"deploymentReleaseCounter,omitempty"` + WorkflowExecutor pipelineConfig.WorkflowExecutorType `json:"workflowExecutor"` + PrePostDeploySteps []*bean.StepObject `json:"prePostDeploySteps"` + CiArtifactLastFetch time.Time `json:"ciArtifactLastFetch"` + Type bean.WorkflowPipelineType + Pipeline *pipelineConfig.Pipeline + Env *repository.Environment + AppLabels map[string]string +} + +func (workflowRequest *WorkflowRequest) updateExternalRunMetadata() { + pipeline := workflowRequest.Pipeline + env := workflowRequest.Env + // Check for external in case of PRE-/POST-CD + if (workflowRequest.StageType == PRE && pipeline.RunPreStageInEnv) || (workflowRequest.StageType == POST && pipeline.RunPostStageInEnv) { + workflowRequest.IsExtRun = true + } + // Check for external in case of JOB + if env != nil && env.Id != 0 && workflowRequest.CheckForJob() { + workflowRequest.EnvironmentId = env.Id + workflowRequest.IsExtRun = true + } +} + +func (workflowRequest *WorkflowRequest) CheckBlobStorageConfig(config *CiCdConfig) bool { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return config.UseBlobStorageConfigInCiWorkflow + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return config.UseBlobStorageConfigInCdWorkflow + default: + return false + } + +} + +func (workflowRequest *WorkflowRequest) GetWorkflowTemplate(workflowJson []byte, config *CiCdConfig) bean.WorkflowTemplate { + + ttl := int32(config.BuildLogTTLValue) + workflowTemplate := bean.WorkflowTemplate{} + workflowTemplate.TTLValue = &ttl + workflowTemplate.WorkflowId = workflowRequest.WorkflowId + workflowTemplate.WorkflowRequestJson = string(workflowJson) + workflowTemplate.RefPlugins = workflowRequest.RefPlugins + workflowTemplate.ActiveDeadlineSeconds = &workflowRequest.ActiveDeadlineSeconds + workflowTemplate.Namespace = workflowRequest.Namespace + workflowTemplate.WorkflowNamePrefix = workflowRequest.WorkflowNamePrefix + if workflowRequest.Type == bean.CD_WORKFLOW_PIPELINE_TYPE { + workflowTemplate.WorkflowRunnerId = workflowRequest.WorkflowRunnerId + workflowTemplate.PrePostDeploySteps = workflowRequest.PrePostDeploySteps + } + return workflowTemplate +} + +func (workflowRequest *WorkflowRequest) checkConfigType(config *CiCdConfig) { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + config.Type = CiConfigType + case bean.CD_WORKFLOW_PIPELINE_TYPE: + config.Type = CdConfigType + } +} + +func (workflowRequest *WorkflowRequest) GetBlobStorageLogsKey(config *CiCdConfig) string { + return fmt.Sprintf("%s/%s", config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowPrefixForLog) +} + +func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]byte, error) { + workflowRequest.updateBlobStorageLogsKey(config) + workflowRequest.updateExternalRunMetadata() + workflowJson, err := workflowRequest.getWorkflowJson() + if err != nil { + return nil, err + } + return workflowJson, err +} + +func (workflowRequest *WorkflowRequest) GetEventTypeForWorkflowRequest() string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return bean.CiStage + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return bean.CdStage + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) GetWorkflowTypeForWorkflowRequest() string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return bean.CI_WORKFLOW_NAME + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return bean.CD_WORKFLOW_NAME + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdConfig, workflowJson []byte) (containerEnvVariables []v1.EnvVar) { + if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE || + workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { + containerEnvVariables = []v1.EnvVar{{Name: "IMAGE_SCANNER_ENDPOINT", Value: config.ImageScannerEndpoint}} + } + if config.CloudProvider == BLOB_STORAGE_S3 && config.BlobStorageS3AccessKey != "" { + miniCred := []v1.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: config.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: config.BlobStorageS3SecretKey}} + containerEnvVariables = append(containerEnvVariables, miniCred...) + } + eventEnv := v1.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)} + inAppLoggingEnv := v1.EnvVar{Name: "IN_APP_LOGGING", Value: strconv.FormatBool(workflowRequest.InAppLoggingEnabled)} + containerEnvVariables = append(containerEnvVariables, eventEnv, inAppLoggingEnv) + return containerEnvVariables +} + +func (workflowRequest *WorkflowRequest) getPVCForWorkflowRequest() string { + var pvc string + workflowRequestType := workflowRequest.Type + if workflowRequestType == bean.CI_WORKFLOW_PIPELINE_TYPE || + workflowRequestType == bean.JOB_WORKFLOW_PIPELINE_TYPE { + pvc = workflowRequest.AppLabels[strings.ToLower(fmt.Sprintf("%s-%s", CI_NODE_PVC_PIPELINE_PREFIX, workflowRequest.PipelineName))] + if len(pvc) == 0 { + pvc = workflowRequest.AppLabels[CI_NODE_PVC_ALL_ENV] + } + if len(pvc) != 0 { + workflowRequest.IsPvcMounted = true + workflowRequest.IgnoreDockerCachePush = true + workflowRequest.IgnoreDockerCachePull = true + } + } else { + //pvc not supported for other then ci and job currently + } + return pvc +} + +func (workflowRequest *WorkflowRequest) getDefaultBuildLogsKeyPrefix(config *CiCdConfig) string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return config.CiDefaultBuildLogsKeyPrefix + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return config.CdDefaultBuildLogsKeyPrefix + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) getBlobStorageLogsPrefix() string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return workflowRequest.WorkflowNamePrefix + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return workflowRequest.WorkflowPrefixForLog + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) updateBlobStorageLogsKey(config *CiCdConfig) { + workflowRequest.BlobStorageLogsKey = fmt.Sprintf("%s/%s", workflowRequest.getDefaultBuildLogsKeyPrefix(config), workflowRequest.getBlobStorageLogsPrefix()) + workflowRequest.InAppLoggingEnabled = config.InAppLoggingEnabled || (workflowRequest.WorkflowExecutor == pipelineConfig.WORKFLOW_EXECUTOR_TYPE_SYSTEM) +} + +func (workflowRequest *WorkflowRequest) getWorkflowJson() ([]byte, error) { + eventType := workflowRequest.GetEventTypeForWorkflowRequest() + ciCdTriggerEvent := CiCdTriggerEvent{ + Type: eventType, + CommonWorkflowRequest: workflowRequest, + } + workflowJson, err := json.Marshal(&ciCdTriggerEvent) + if err != nil { + return nil, err + } + return workflowJson, err +} + +func (workflowRequest *WorkflowRequest) AddNodeConstraintsFromConfig(workflowTemplate *bean.WorkflowTemplate, config *CiCdConfig) { + nodeConstraints := workflowRequest.GetNodeConstraints(config) + if workflowRequest.Type == bean.CD_WORKFLOW_PIPELINE_TYPE && nodeConstraints.TaintKey != "" { + workflowTemplate.NodeSelector = map[string]string{nodeConstraints.TaintKey: nodeConstraints.TaintValue} + } + workflowTemplate.ServiceAccountName = nodeConstraints.ServiceAccount + if nodeConstraints.TaintKey != "" || nodeConstraints.TaintValue != "" { + workflowTemplate.Tolerations = []v1.Toleration{{Key: nodeConstraints.TaintKey, Value: nodeConstraints.TaintValue, Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}} + } + // In the future, we will give support for NodeSelector for job currently we need to have a node without dedicated NodeLabel to run job + if len(nodeConstraints.NodeLabel) > 0 && !(nodeConstraints.SkipNodeSelector) { + workflowTemplate.NodeSelector = nodeConstraints.NodeLabel + } + workflowTemplate.ArchiveLogs = workflowRequest.BlobStorageConfigured && !workflowRequest.InAppLoggingEnabled + workflowTemplate.RestartPolicy = v1.RestartPolicyNever + +} + +func (workflowRequest *WorkflowRequest) GetGlobalCmCsNamePrefix() string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return strconv.Itoa(workflowRequest.WorkflowId) + "-" + bean.CI_WORKFLOW_NAME + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return strconv.Itoa(workflowRequest.WorkflowRunnerId) + "-" + bean.CD_WORKFLOW_NAME + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) GetConfiguredCmCs() (map[string]bool, map[string]bool, error) { + + cdPipelineLevelConfigMaps := make(map[string]bool) + cdPipelineLevelSecrets := make(map[string]bool) + + if workflowRequest.StageType == "PRE" { + preStageConfigMapSecretsJson := workflowRequest.Pipeline.PreStageConfigMapSecretNames + preStageConfigmapSecrets := bean2.PreStageConfigMapSecretNames{} + err := json.Unmarshal([]byte(preStageConfigMapSecretsJson), &preStageConfigmapSecrets) + if err != nil { + return cdPipelineLevelConfigMaps, cdPipelineLevelSecrets, err + } + for _, cm := range preStageConfigmapSecrets.ConfigMaps { + cdPipelineLevelConfigMaps[cm] = true + } + for _, secret := range preStageConfigmapSecrets.Secrets { + cdPipelineLevelSecrets[secret] = true + } + } + if workflowRequest.StageType == "POST" { + postStageConfigMapSecretsJson := workflowRequest.Pipeline.PostStageConfigMapSecretNames + postStageConfigmapSecrets := bean2.PostStageConfigMapSecretNames{} + err := json.Unmarshal([]byte(postStageConfigMapSecretsJson), &postStageConfigmapSecrets) + if err != nil { + return cdPipelineLevelConfigMaps, cdPipelineLevelSecrets, err + } + for _, cm := range postStageConfigmapSecrets.ConfigMaps { + cdPipelineLevelConfigMaps[cm] = true + } + for _, secret := range postStageConfigmapSecrets.Secrets { + cdPipelineLevelSecrets[secret] = true + } + } + return cdPipelineLevelConfigMaps, cdPipelineLevelSecrets, nil +} + +func (workflowRequest *WorkflowRequest) GetExistingCmCsNamePrefix() string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE: + return strconv.Itoa(workflowRequest.WorkflowId) + "-" + bean.CI_WORKFLOW_NAME + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return strconv.Itoa(workflowRequest.WorkflowRunnerId) + "-" + strconv.Itoa(workflowRequest.WorkflowRunnerId) + case bean.JOB_WORKFLOW_PIPELINE_TYPE: + return strconv.Itoa(workflowRequest.WorkflowId) + "-" + bean.CI_WORKFLOW_NAME + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) CheckForJob() bool { + return workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE +} + +func (workflowRequest *WorkflowRequest) GetNodeConstraints(config *CiCdConfig) *bean.NodeConstraints { + nodeLabel, err := GetNodeLabel(config, workflowRequest.Type, workflowRequest.IsExtRun) + if err != nil { + return nil + } + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return &bean.NodeConstraints{ + ServiceAccount: config.CiWorkflowServiceAccount, + TaintKey: config.CiTaintKey, + TaintValue: config.CiTaintValue, + NodeLabel: nodeLabel, + SkipNodeSelector: false, + } + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return &bean.NodeConstraints{ + ServiceAccount: config.CdWorkflowServiceAccount, + TaintKey: config.CdTaintKey, + TaintValue: config.CdTaintValue, + NodeLabel: nodeLabel, + SkipNodeSelector: false, + } + default: + return nil + } +} + +func (workflowRequest *WorkflowRequest) GetLimitReqCpuMem(config *CiCdConfig) v1.ResourceRequirements { + limitReqCpuMem := &bean.LimitReqCpuMem{} + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + limitReqCpuMem = &bean.LimitReqCpuMem{ + LimitCpu: config.CiLimitCpu, + LimitMem: config.CiLimitMem, + ReqCpu: config.CiReqCpu, + ReqMem: config.CiReqMem, + } + case bean.CD_WORKFLOW_PIPELINE_TYPE: + limitReqCpuMem = &bean.LimitReqCpuMem{ + LimitCpu: config.CdLimitCpu, + LimitMem: config.CdLimitMem, + ReqCpu: config.CdReqCpu, + ReqMem: config.CdReqMem, + } + } + return v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(limitReqCpuMem.LimitCpu), + v1.ResourceMemory: resource.MustParse(limitReqCpuMem.LimitMem), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(limitReqCpuMem.ReqCpu), + v1.ResourceMemory: resource.MustParse(limitReqCpuMem.ReqMem), + }, + } +} + +func (workflowRequest *WorkflowRequest) getWorkflowImage() string { + switch workflowRequest.Type { + case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: + return workflowRequest.CiImage + case bean.CD_WORKFLOW_PIPELINE_TYPE: + return workflowRequest.CdImage + default: + return "" + } +} + +func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdConfig, workflowJson []byte, workflowTemplate *bean.WorkflowTemplate, workflowConfigMaps []bean3.ConfigSecretMap, workflowSecrets []bean3.ConfigSecretMap) (v1.Container, error) { + privileged := true + pvc := workflowRequest.getPVCForWorkflowRequest() + containerEnvVariables := workflowRequest.getContainerEnvVariables(config, workflowJson) + workflowMainContainer := v1.Container{ + Env: containerEnvVariables, + Name: common.MainContainerName, + Image: workflowRequest.getWorkflowImage(), + SecurityContext: &v1.SecurityContext{ + Privileged: &privileged, + }, + Resources: workflowRequest.GetLimitReqCpuMem(config), + } + if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE || workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { + workflowMainContainer.Name = "" + workflowMainContainer.Ports = []v1.ContainerPort{{ + //exposed for user specific data from ci container + Name: "app-data", + ContainerPort: 9102, + }} + err := workflowRequest.updateVolumeMountsForCi(config, workflowTemplate, &workflowMainContainer) + if err != nil { + return workflowMainContainer, err + } + } + + if len(pvc) != 0 { + buildPvcCachePath := config.BuildPvcCachePath + buildxPvcCachePath := config.BuildxPvcCachePath + defaultPvcCachePath := config.DefaultPvcCachePath + + workflowTemplate.Volumes = append(workflowTemplate.Volumes, v1.Volume{ + Name: "root-vol", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc, + ReadOnly: false, + }, + }, + }) + workflowMainContainer.VolumeMounts = append(workflowMainContainer.VolumeMounts, + v1.VolumeMount{ + Name: "root-vol", + MountPath: buildPvcCachePath, + }, + v1.VolumeMount{ + Name: "root-vol", + MountPath: buildxPvcCachePath, + }, + v1.VolumeMount{ + Name: "root-vol", + MountPath: defaultPvcCachePath, + }) + } + UpdateContainerEnvsFromCmCs(&workflowMainContainer, workflowConfigMaps, workflowSecrets) + return workflowMainContainer, nil +} + +func (workflowRequest *WorkflowRequest) updateVolumeMountsForCi(config *CiCdConfig, workflowTemplate *bean.WorkflowTemplate, workflowMainContainer *v1.Container) error { + volume, volumeMounts, err := config.GetWorkflowVolumeAndVolumeMounts() + if err != nil { + return err + } + workflowTemplate.Volumes = volume + workflowMainContainer.VolumeMounts = volumeMounts + return nil +} + +func UpdateContainerEnvsFromCmCs(workflowMainContainer *v1.Container, configMaps []bean3.ConfigSecretMap, secrets []bean3.ConfigSecretMap) { + for _, configMap := range configMaps { + updateContainerEnvs(true, workflowMainContainer, configMap) + } + + for _, secret := range secrets { + updateContainerEnvs(false, workflowMainContainer, secret) + } +} + +func updateContainerEnvs(isCM bool, workflowMainContainer *v1.Container, configSecretMap bean3.ConfigSecretMap) { + if configSecretMap.Type == repository2.VOLUME_CONFIG { + workflowMainContainer.VolumeMounts = append(workflowMainContainer.VolumeMounts, v1.VolumeMount{ + Name: configSecretMap.Name + "-vol", + MountPath: configSecretMap.MountPath, + }) + } else if configSecretMap.Type == repository2.ENVIRONMENT_CONFIG { + var envFrom v1.EnvFromSource + if isCM { + envFrom = v1.EnvFromSource{ + ConfigMapRef: &v1.ConfigMapEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configSecretMap.Name, + }, + }, + } + } else { + envFrom = v1.EnvFromSource{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configSecretMap.Name, + }, + }, + } + } + workflowMainContainer.EnvFrom = append(workflowMainContainer.EnvFrom, envFrom) + } +} + +const PRE = "PRE" + +const POST = "POST" + +const CI_NODE_PVC_ALL_ENV = "devtron.ai/ci-pvc-all" + +const CI_NODE_PVC_PIPELINE_PREFIX = "devtron.ai/ci-pvc" + +type CiArtifactDTO struct { + Id int `json:"id"` + PipelineId int `json:"pipelineId"` //id of the ci pipeline from which this webhook was triggered + Image string `json:"image"` + ImageDigest string `json:"imageDigest"` + MaterialInfo string `json:"materialInfo"` //git material metadata json array string + DataSource string `json:"dataSource"` + WorkflowId *int `json:"workflowId"` + ciArtifactRepository repository2.CiArtifactRepository +} + +type CiCdTriggerEvent struct { + Type string `json:"type"` + CommonWorkflowRequest *WorkflowRequest `json:"commonWorkflowRequest"` +} diff --git a/pkg/security/policyService.go b/pkg/security/policyService.go index 660ce3fda3..0aa06c90a8 100644 --- a/pkg/security/policyService.go +++ b/pkg/security/policyService.go @@ -23,6 +23,7 @@ import ( "fmt" repository1 "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/helper" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" "net/http" "strings" @@ -34,7 +35,6 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/pkg/cluster" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -63,7 +63,7 @@ type PolicyServiceImpl struct { imageScanObjectMetaRepository security.ImageScanObjectMetaRepository client *http.Client ciArtifactRepository repository.CiArtifactRepository - ciConfig *pipeline.CiCdConfig + ciConfig *types.CiCdConfig scanHistoryRepository security.ImageScanHistoryRepository cveStoreRepository security.CveStoreRepository ciTemplateRepository pipelineConfig.CiTemplateRepository @@ -79,7 +79,7 @@ func NewPolicyServiceImpl(environmentService cluster.EnvironmentService, scanResultRepository security.ImageScanResultRepository, imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, imageScanObjectMetaRepository security.ImageScanObjectMetaRepository, client *http.Client, - ciArtifactRepository repository.CiArtifactRepository, ciConfig *pipeline.CiCdConfig, + ciArtifactRepository repository.CiArtifactRepository, ciConfig *types.CiCdConfig, scanHistoryRepository security.ImageScanHistoryRepository, cveStoreRepository security.CveStoreRepository, ciTemplateRepository pipelineConfig.CiTemplateRepository) *PolicyServiceImpl { return &PolicyServiceImpl{ diff --git a/wire_gen.go b/wire_gen.go index 454a739d54..d1d40bf2f4 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -123,6 +123,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/history" repository6 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" repository11 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" repository12 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/projectManagementService/jira" @@ -415,7 +416,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciCdConfig, err := pipeline.GetCiCdConfig() + ciCdConfig, err := types.GetCiCdConfig() if err != nil { return nil, err } From 5726b4f1bed568b0b8493406b50c041cb428de4e Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Sun, 29 Oct 2023 19:37:43 -0400 Subject: [PATCH 2/7] added license text --- pkg/pipeline/types/Workflow.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 0bd4a9e514..4985dea9aa 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2020 Devtron Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package types import ( From eec0342aa2cd927fe0b38950cda1182ed26971cc Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Sun, 29 Oct 2023 23:01:00 -0400 Subject: [PATCH 3/7] moved structs to workflow --- pkg/pipeline/WorkflowDagExecutor.go | 28 ++++++---------------------- pkg/pipeline/types/Workflow.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 8aa7c48ed0..9f05f31ea7 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -225,22 +225,6 @@ type CdStageCompleteEvent struct { CiArtifactDTO pipelineConfig.CiArtifactDTO `json:"ciArtifactDTO"` } -type GitMetadata struct { - GitCommitHash string `json:"GIT_COMMIT_HASH"` - GitSourceType string `json:"GIT_SOURCE_TYPE"` - GitSourceValue string `json:"GIT_SOURCE_VALUE"` -} - -type AppLabelMetadata struct { - AppLabelKey string `json:"APP_LABEL_KEY"` - AppLabelValue string `json:"APP_LABEL_VALUE"` -} - -type ChildCdMetadata struct { - ChildCdEnvName string `json:"CHILD_CD_ENV_NAME"` - ChildCdClusterName string `json:"CHILD_CD_CLUSTER_NAME"` -} - func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pubsubClient *pubsub.PubSubClientServiceImpl, @@ -1146,14 +1130,14 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor var webhookAndCiData *gitSensorClient.WebhookAndCiData if ciWf != nil && ciWf.GitTriggers != nil { i := 1 - var gitCommitEnvVariables []GitMetadata + var gitCommitEnvVariables []types.GitMetadata for ciPipelineMaterialId, gitTrigger := range ciWf.GitTriggers { extraEnvVariables[fmt.Sprintf("%s_%d", GIT_COMMIT_HASH_PREFIX, i)] = gitTrigger.Commit extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_TYPE_PREFIX, i)] = string(gitTrigger.CiConfigureSourceType) extraEnvVariables[fmt.Sprintf("%s_%d", GIT_SOURCE_VALUE_PREFIX, i)] = gitTrigger.CiConfigureSourceValue - gitCommitEnvVariables = append(gitCommitEnvVariables, GitMetadata{ + gitCommitEnvVariables = append(gitCommitEnvVariables, types.GitMetadata{ GitCommitHash: gitTrigger.Commit, GitSourceType: string(gitTrigger.CiConfigureSourceType), GitSourceValue: gitTrigger.CiConfigureSourceValue, @@ -1204,12 +1188,12 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor impl.logger.Errorw("error in getting pipelines by ids", "err", err, "ids", childCdIds) return nil, err } - var childCdEnvVariables []ChildCdMetadata + var childCdEnvVariables []types.ChildCdMetadata for i, childPipeline := range childPipelines { extraEnvVariables[fmt.Sprintf("%s_%d", CHILD_CD_ENV_NAME_PREFIX, i+1)] = childPipeline.Environment.Name extraEnvVariables[fmt.Sprintf("%s_%d", CHILD_CD_CLUSTER_NAME_PREFIX, i+1)] = childPipeline.Environment.Cluster.ClusterName - childCdEnvVariables = append(childCdEnvVariables, ChildCdMetadata{ + childCdEnvVariables = append(childCdEnvVariables, types.ChildCdMetadata{ ChildCdEnvName: childPipeline.Environment.Name, ChildCdClusterName: childPipeline.Environment.Cluster.ClusterName, }) @@ -1254,11 +1238,11 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor impl.logger.Errorw("error in getting labels by appId", "err", err, "appId", cdPipeline.AppId) return nil, err } - var appLabelEnvVariables []AppLabelMetadata + var appLabelEnvVariables []types.AppLabelMetadata for i, appLabel := range appLabels { extraEnvVariables[fmt.Sprintf("%s_%d", APP_LABEL_KEY_PREFIX, i+1)] = appLabel.Key extraEnvVariables[fmt.Sprintf("%s_%d", APP_LABEL_VALUE_PREFIX, i+1)] = appLabel.Value - appLabelEnvVariables = append(appLabelEnvVariables, AppLabelMetadata{ + appLabelEnvVariables = append(appLabelEnvVariables, types.AppLabelMetadata{ AppLabelKey: appLabel.Key, AppLabelValue: appLabel.Value, }) diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 4985dea9aa..5bf0c8063d 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -566,3 +566,19 @@ type CiCdTriggerEvent struct { Type string `json:"type"` CommonWorkflowRequest *WorkflowRequest `json:"commonWorkflowRequest"` } + +type GitMetadata struct { + GitCommitHash string `json:"GIT_COMMIT_HASH"` + GitSourceType string `json:"GIT_SOURCE_TYPE"` + GitSourceValue string `json:"GIT_SOURCE_VALUE"` +} + +type AppLabelMetadata struct { + AppLabelKey string `json:"APP_LABEL_KEY"` + AppLabelValue string `json:"APP_LABEL_VALUE"` +} + +type ChildCdMetadata struct { + ChildCdEnvName string `json:"CHILD_CD_ENV_NAME"` + ChildCdClusterName string `json:"CHILD_CD_CLUSTER_NAME"` +} From 93793de5bcd3846de21c5925c008e6cb87c06aa7 Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Sun, 29 Oct 2023 23:17:19 -0400 Subject: [PATCH 4/7] moved to ci and imagetagging --- .../app/BuildPipelineRestHandler.go | 13 +- pkg/pipeline/CdHandler.go | 18 +-- pkg/pipeline/CiHandler.go | 141 ++++-------------- pkg/pipeline/CiService.go | 12 +- pkg/pipeline/ImageTaggingService.go | 33 ++-- pkg/pipeline/ImageTaggingService_test.go | 7 +- pkg/pipeline/types/CiCdConfig.go | 49 ++++++ pkg/pipeline/types/ImageTagging.go | 19 +++ pkg/pipeline/types/Workflow.go | 33 ++++ 9 files changed, 166 insertions(+), 159 deletions(-) create mode 100644 pkg/pipeline/types/ImageTagging.go diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index ea5a2342a7..225fef0dcc 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -16,6 +16,7 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline" bean1 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/user/casbin" util2 "github.com/devtron-labs/devtron/util" @@ -31,10 +32,10 @@ import ( const GIT_MATERIAL_DELETE_SUCCESS_RESP = "Git material deleted successfully." type BuildHistoryResponse struct { - HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` - TagsEditable bool `json:"tagsEditable"` - AppReleaseTagNames []string `json:"appReleaseTagNames"` //unique list of tags exists in the app - CiWorkflows []pipeline.WorkflowResponse `json:"ciWorkflows"` + HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` + TagsEditable bool `json:"tagsEditable"` + AppReleaseTagNames []string `json:"appReleaseTagNames"` //unique list of tags exists in the app + CiWorkflows []types.WorkflowResponse `json:"ciWorkflows"` } type DevtronAppBuildRestHandler interface { CreateCiConfig(w http.ResponseWriter, r *http.Request) @@ -1850,7 +1851,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateUpdateImageTagging(w http.Res return } decoder := json.NewDecoder(r.Body) - req := &pipeline.ImageTaggingRequestDTO{} + req := &types.ImageTaggingRequestDTO{} err = decoder.Decode(&req) if err != nil { handler.Logger.Errorw("request err, CreateUpdateImageTagging", "err", err, "payload", req) @@ -1909,7 +1910,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateUpdateImageTagging(w http.Res handler.Logger.Errorw("error occurred in getting unique tags in app", "err", err1, "appId", appId) err = err1 } - resp = &pipeline.ImageTaggingResponseDTO{} + resp = &types.ImageTaggingResponseDTO{} resp.AppReleaseTags = appReleaseTags } handler.Logger.Errorw("error occurred in creating/updating image tagging data", "err", err, "ciPipelineId", ciPipelineId) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 4077f8bddd..3bf21406bd 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -72,7 +72,7 @@ type CdHandler interface { UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineConfig.CdWorkflowWithArtifact, error) GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) - FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (WorkflowResponse, error) + FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) DownloadCdWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineConfig.CdWorkflowWithArtifact, error) CancelStage(workflowRunnerId int, userId int32) (int, error) @@ -975,20 +975,20 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip return logReader, cleanUp, err } -func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (WorkflowResponse, error) { +func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } else if err == pg.ErrNoRows { - return WorkflowResponse{}, nil + return types.WorkflowResponse{}, nil } workflow := impl.converterWFR(*workflowR) triggeredByUser, err := impl.userService.GetById(workflow.TriggeredBy) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } if triggeredByUser == nil { triggeredByUser = &bean.UserInfo{EmailId: "anonymous"} @@ -998,7 +998,7 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, ciArtifact, err := impl.ciArtifactRepository.Get(ciArtifactId) if err != nil { impl.Logger.Errorw("error fetching artifact data", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } // handling linked ci pipeline @@ -1009,12 +1009,12 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, ciWf, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(ciArtifactId) if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("error in fetching ci wf", "artifactId", workflow.CiArtifactId, "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineIdForRegexAndFixed(ciWf.CiPipelineId) if err != nil { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse @@ -1035,7 +1035,7 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, gitTriggers = ciWf.GitTriggers } - workflowResponse := WorkflowResponse{ + workflowResponse := types.WorkflowResponse{ Id: workflow.Id, Name: workflow.Name, Status: workflow.Status, diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index e2ee7ceaca..3bccd7922d 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -29,7 +29,6 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" - repository2 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/pkg/cluster" repository3 "github.com/devtron-labs/devtron/pkg/cluster/repository" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" @@ -65,8 +64,8 @@ type CiHandler interface { CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) - FetchWorkflowDetails(appId int, pipelineId int, buildId int) (WorkflowResponse, error) - FetchArtifactsForCiJob(buildId int) (*ArtifactsForCiJob, error) + FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) + FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) CancelBuild(workflowId int) (int, error) @@ -74,14 +73,14 @@ type CiHandler interface { GetHistoricBuildLogs(pipelineId int, workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) //SyncWorkflows() error - GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]WorkflowResponse, error) + GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) - FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*GitTriggerInfoResponse, error) + FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) WriteToCreateTestSuites(pipelineId int, buildId int, triggeredBy int) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest) ([]*pipelineConfig.CiWorkflowStatus, error) @@ -147,86 +146,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -type WorkflowResponse struct { - Id int `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"podStatus"` - Message string `json:"message"` - StartedOn time.Time `json:"startedOn"` - FinishedOn time.Time `json:"finishedOn"` - CiPipelineId int `json:"ciPipelineId"` - Namespace string `json:"namespace"` - LogLocation string `json:"logLocation"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredBy int32 `json:"triggeredBy"` - Artifact string `json:"artifact"` - TriggeredByEmail string `json:"triggeredByEmail"` - Stage string `json:"stage"` - ArtifactId int `json:"artifactId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - IsVirtualEnvironment bool `json:"isVirtualEnvironment"` - PodName string `json:"podName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - ImageReleaseTags []*repository2.ImageTag `json:"imageReleaseTags"` - ImageComment *repository2.ImageComment `json:"imageComment"` - AppWorkflowId int `json:"appWorkflowId"` - CustomTag *bean2.CustomTagErrorResponse `json:"customTag,omitempty"` - PipelineType string `json:"pipelineType"` - ReferenceWorkflowId int `json:"referenceWorkflowId"` -} - -type ArtifactsForCiJob struct { - Artifacts []string `json:"artifacts"` -} - -type GitTriggerInfoResponse struct { - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredByEmail string `json:"triggeredByEmail"` - LastDeployedTime string `json:"lastDeployedTime,omitempty"` - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - Default bool `json:"default,omitempty"` - ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` - Image string `json:"image"` -} - -type Trigger struct { - PipelineId int - CommitHashes map[int]pipelineConfig.GitCommit - CiMaterials []*pipelineConfig.CiPipelineMaterial - TriggeredBy int32 - InvalidateCache bool - ExtraEnvironmentVariables map[string]string // extra env variables which will be used for CI - EnvironmentId int - PipelineType string - CiArtifactLastFetch time.Time - ReferenceCiWorkflowId int -} - -func (obj Trigger) BuildTriggerObject(refCiWorkflow *pipelineConfig.CiWorkflow, - ciMaterials []*pipelineConfig.CiPipelineMaterial, triggeredBy int32, - invalidateCache bool, extraEnvironmentVariables map[string]string, - pipelineType string) { - - obj.PipelineId = refCiWorkflow.CiPipelineId - obj.CommitHashes = refCiWorkflow.GitTriggers - obj.CiMaterials = ciMaterials - obj.TriggeredBy = triggeredBy - obj.InvalidateCache = invalidateCache - obj.EnvironmentId = refCiWorkflow.EnvironmentId - obj.ReferenceCiWorkflowId = refCiWorkflow.Id - obj.InvalidateCache = invalidateCache - obj.ExtraEnvironmentVariables = extraEnvironmentVariables - obj.PipelineType = pipelineType - -} - const WorkflowCancel = "CANCELLED" const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" @@ -281,7 +200,7 @@ func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineCo return err } - trigger := Trigger{} + trigger := types.Trigger{} trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, 1, true, nil, "") _, err = impl.ciService.TriggerCiPipeline(trigger) @@ -310,7 +229,7 @@ func (impl *CiHandlerImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest createdOn = ciArtifact.CreatedOn } - trigger := Trigger{ + trigger := types.Trigger{ PipelineId: ciTriggerRequest.PipelineId, CommitHashes: commitHashes, CiMaterials: nil, @@ -356,7 +275,7 @@ func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTrigger return 0, err } - trigger := Trigger{ + trigger := types.Trigger{ PipelineId: ciPipeline.Id, CommitHashes: commitHashes, CiMaterials: ciMaterials, @@ -556,7 +475,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo return ciPipelineMaterialResponses, nil } -func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]WorkflowResponse, error) { +func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineIdForRegexAndFixed(pipelineId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) @@ -596,9 +515,9 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return nil, err } - var ciWorkLowResponses []WorkflowResponse + var ciWorkLowResponses []types.WorkflowResponse for _, w := range workFlows { - wfResponse := WorkflowResponse{ + wfResponse := types.WorkflowResponse{ Id: w.Id, Name: w.Name, Status: w.Status, @@ -698,33 +617,33 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) { return workflow.Id, nil } -func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (WorkflowResponse, error) { +func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } triggeredByUser, err := impl.userService.GetById(workflow.TriggeredBy) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } if workflow.CiPipeline.AppId != appId { impl.Logger.Error("pipeline does not exist for this app") - return WorkflowResponse{}, errors.New("invalid app and pipeline combination") + return types.WorkflowResponse{}, errors.New("invalid app and pipeline combination") } ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } ciArtifact, err := impl.ciArtifactRepository.GetByWfId(workflow.Id) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse @@ -745,11 +664,11 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build env, err := impl.envRepository.FindById(workflow.EnvironmentId) if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("error in fetching environment details ", "err", err) - return WorkflowResponse{}, err + return types.WorkflowResponse{}, err } environmentName = env.Name } - workflowResponse := WorkflowResponse{ + workflowResponse := types.WorkflowResponse{ Id: workflow.Id, Name: workflow.Name, Status: workflow.Status, @@ -775,13 +694,13 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build return workflowResponse, nil } -func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*ArtifactsForCiJob, error) { +func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) { artifacts, err := impl.ciArtifactRepository.GetArtifactsByParentCiWorkflowId(buildId) if err != nil { impl.Logger.Errorw("error in fetching artifacts by parent ci workflow id", "err", err, "buildId", buildId) return nil, err } - artifactsResponse := &ArtifactsForCiJob{ + artifactsResponse := &types.ArtifactsForCiJob{ Artifacts: artifacts, } return artifactsResponse, nil @@ -1426,30 +1345,30 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerView(appId int) ([]*pipelineCo return ciWorkflowStatuses, nil } -func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*GitTriggerInfoResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) { ciArtifact, err := impl.ciArtifactRepository.Get(ciArtifactId) if err != nil { impl.Logger.Errorw("err", "ciArtifactId", ciArtifactId, "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } ciPipeline, err := impl.ciPipelineRepository.FindById(ciArtifact.PipelineId) if err != nil { impl.Logger.Errorw("err", "ciArtifactId", ciArtifactId, "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) if err != nil { impl.Logger.Errorw("err", "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } deployDetail, err := impl.appListingRepository.DeploymentDetailByArtifactId(ciArtifactId, envId) if err != nil { impl.Logger.Errorw("err", "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) @@ -1461,20 +1380,20 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId workflow, err = impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(ciArtifact.ParentCiArtifact) if err != nil { impl.Logger.Errorw("err", "ciArtifactId", ciArtifact.ParentCiArtifact, "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } } else { workflow, err = impl.ciWorkflowRepository.FindLastTriggeredWorkflowByArtifactId(ciArtifactId) if err != nil { impl.Logger.Errorw("err", "ciArtifactId", ciArtifactId, "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } } triggeredByUser, err = impl.userService.GetById(workflow.TriggeredBy) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "err", err) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } for _, m := range ciMaterials { @@ -1522,9 +1441,9 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId imageTaggingData, err := impl.imageTaggingService.GetTagsData(ciPipeline.Id, ciPipeline.AppId, ciArtifactId, false) if err != nil { impl.Logger.Errorw("error in fetching imageTaggingData", "err", err, "ciPipelineId", ciPipeline.Id, "appId", ciPipeline.AppId, "ciArtifactId", ciArtifactId) - return &GitTriggerInfoResponse{}, err + return &types.GitTriggerInfoResponse{}, err } - gitTriggerInfoResponse := &GitTriggerInfoResponse{ + gitTriggerInfoResponse := &types.GitTriggerInfoResponse{ //GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredByEmail: triggeredByUser.EmailId, diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 2e90e9ac74..f34785f4f5 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -52,7 +52,7 @@ import ( ) type CiService interface { - TriggerCiPipeline(trigger Trigger) (int, error) + TriggerCiPipeline(trigger types.Trigger) (int, error) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) } @@ -130,7 +130,7 @@ func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelin } } -func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { +func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { impl.Logger.Debug("ci pipeline manual trigger") ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) if err != nil { @@ -280,7 +280,7 @@ func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *types.Workflo return nil } -func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger Trigger) (*repository1.Environment, bool, error) { +func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository1.Environment, bool, error) { app, err := impl.appRepository.FindById(pipeline.AppId) if err != nil { impl.Logger.Errorw("could not find app", "err", err) @@ -303,7 +303,7 @@ func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipel return nil, isJob, nil } -func (impl *CiServiceImpl) WriteCITriggerEvent(trigger Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { +func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) material := &client.MaterialTriggerInfo{} @@ -319,7 +319,7 @@ func (impl *CiServiceImpl) WriteCITriggerEvent(trigger Trigger, pipeline *pipeli } // TODO: Send all trigger data -func (impl *CiServiceImpl) BuildPayload(trigger Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { +func (impl *CiServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { payload := &client.Payload{} payload.AppName = pipeline.App.AppName payload.PipelineName = pipeline.Name @@ -383,7 +383,7 @@ func (impl *CiServiceImpl) buildDefaultArtifactLocation(ciWorkflowConfig *pipeli return ArtifactLocation } -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger Trigger, +func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfig *pipelineConfig.CiWorkflowConfig, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*bean2.StepObject, postCiSteps []*bean2.StepObject, refPluginsData []*bean2.RefPluginObject) (*types.WorkflowRequest, error) { diff --git a/pkg/pipeline/ImageTaggingService.go b/pkg/pipeline/ImageTaggingService.go index 57e074f422..1462474fca 100644 --- a/pkg/pipeline/ImageTaggingService.go +++ b/pkg/pipeline/ImageTaggingService.go @@ -24,6 +24,7 @@ import ( repository "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/go-pg/pg" "go.uber.org/zap" "strings" @@ -38,34 +39,18 @@ type ImageTaggingServiceConfig struct { HideImageTaggingHardDelete bool `env:"HIDE_IMAGE_TAGGING_HARD_DELETE" envDefault:"false"` } -type ImageTaggingResponseDTO struct { - ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` - AppReleaseTags []string `json:"appReleaseTags"` - ImageComment *repository.ImageComment `json:"imageComment"` - ProdEnvExists bool `json:"tagsEditable"` - HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` -} - -type ImageTaggingRequestDTO struct { - CreateTags []*repository.ImageTag `json:"createTags"` - SoftDeleteTags []*repository.ImageTag `json:"softDeleteTags"` - ImageComment repository.ImageComment `json:"imageComment"` - HardDeleteTags []*repository.ImageTag `json:"hardDeleteTags"` - ExternalCi bool `json:"-"` -} - type ImageTaggingService interface { // GetTagsData returns the following fields in reponse Object //ImageReleaseTags -> this will get the tags of the artifact, //AppReleaseTags -> all the tags of the given appId, //imageComment -> comment of the given artifactId, // ProdEnvExists -> implies the existence of prod environment in any workflow of given ciPipelineId or its child ciPipelineRequest's - GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool) (*ImageTaggingResponseDTO, error) - CreateOrUpdateImageTagging(ciPipelineId, appId, artifactId, userId int, imageTaggingRequest *ImageTaggingRequestDTO) (*ImageTaggingResponseDTO, error) + GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool) (*types.ImageTaggingResponseDTO, error) + CreateOrUpdateImageTagging(ciPipelineId, appId, artifactId, userId int, imageTaggingRequest *types.ImageTaggingRequestDTO) (*types.ImageTaggingResponseDTO, error) GetProdEnvFromParentAndLinkedWorkflow(ciPipelineId int) (bool, error) GetProdEnvByCdPipelineId(pipelineId int) (bool, error) // ValidateImageTaggingRequest validates the requested payload - ValidateImageTaggingRequest(imageTaggingRequest *ImageTaggingRequestDTO, appId, artifactId int) (bool, error) + ValidateImageTaggingRequest(imageTaggingRequest *types.ImageTaggingRequestDTO, appId, artifactId int) (bool, error) GetTagsByArtifactId(artifactId int) ([]*repository.ImageTag, error) // GetTaggingDataMapByAppId this will fetch a map of artifact vs []tags for given appId GetTagsDataMapByAppId(appId int) (map[int][]*repository.ImageTag, error) @@ -115,8 +100,8 @@ func (impl ImageTaggingServiceImpl) GetImageTaggingServiceConfig() ImageTaggingS // AppReleaseTags -> all the tags of the given appId, // imageComment -> comment of the given artifactId, // ProdEnvExists -> implies the existence of prod environment in any workflow of given ciPipelineId or its child ciPipelineRequest's -func (impl ImageTaggingServiceImpl) GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool) (*ImageTaggingResponseDTO, error) { - resp := &ImageTaggingResponseDTO{} +func (impl ImageTaggingServiceImpl) GetTagsData(ciPipelineId, appId, artifactId int, externalCi bool) (*types.ImageTaggingResponseDTO, error) { + resp := &types.ImageTaggingResponseDTO{} imageComment, err := impl.imageTaggingRepo.GetImageComment(artifactId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching image comment using artifactId", "err", err, "artifactId", artifactId) @@ -198,7 +183,7 @@ func (impl ImageTaggingServiceImpl) GetImageCommentsDataMapByArtifactIds(artifac return result, nil } -func (impl ImageTaggingServiceImpl) ValidateImageTaggingRequest(imageTaggingRequest *ImageTaggingRequestDTO, appId, artifactId int) (bool, error) { +func (impl ImageTaggingServiceImpl) ValidateImageTaggingRequest(imageTaggingRequest *types.ImageTaggingRequestDTO, appId, artifactId int) (bool, error) { if imageTaggingRequest == nil { return false, errors.New("inValid payload") } @@ -257,7 +242,7 @@ func tagNameValidation(tag string) error { } return nil } -func (impl ImageTaggingServiceImpl) CreateOrUpdateImageTagging(ciPipelineId, appId, artifactId, userId int, imageTaggingRequest *ImageTaggingRequestDTO) (*ImageTaggingResponseDTO, error) { +func (impl ImageTaggingServiceImpl) CreateOrUpdateImageTagging(ciPipelineId, appId, artifactId, userId int, imageTaggingRequest *types.ImageTaggingRequestDTO) (*types.ImageTaggingResponseDTO, error) { tx, err := impl.imageTaggingRepo.StartTx() defer func() { @@ -329,7 +314,7 @@ func (impl ImageTaggingServiceImpl) CreateOrUpdateImageTagging(ciPipelineId, app return impl.GetTagsData(ciPipelineId, appId, artifactId, imageTaggingRequest.ExternalCi) } -func (impl ImageTaggingServiceImpl) performTagOperationsAndGetAuditList(tx *pg.Tx, appId, artifactId, userId int, imageTaggingRequest *ImageTaggingRequestDTO) ([]*repository.ImageTaggingAudit, error) { +func (impl ImageTaggingServiceImpl) performTagOperationsAndGetAuditList(tx *pg.Tx, appId, artifactId, userId int, imageTaggingRequest *types.ImageTaggingRequestDTO) ([]*repository.ImageTaggingAudit, error) { //first perform delete and then perform create operation. //case : user can delete existing tag and then create a new tag with same name, this is a valid request diff --git a/pkg/pipeline/ImageTaggingService_test.go b/pkg/pipeline/ImageTaggingService_test.go index 3a9921bb16..19f969fcdd 100644 --- a/pkg/pipeline/ImageTaggingService_test.go +++ b/pkg/pipeline/ImageTaggingService_test.go @@ -10,6 +10,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" repository1 "github.com/devtron-labs/devtron/pkg/cluster/repository" mocks3 "github.com/devtron-labs/devtron/pkg/cluster/repository/mocks" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/go-pg/pg" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -23,7 +24,7 @@ func TestImageTaggingService(t *testing.T) { assert.True(t, err == nil, err) appId, artifactId, ciPipelineId, userId := 1, 3, 1, 2 - testPayload := &ImageTaggingRequestDTO{ + testPayload := &types.ImageTaggingRequestDTO{ CreateTags: []*repository.ImageTag{{ TagName: "devtron-v1.1.6", AppId: appId, @@ -451,7 +452,7 @@ func TestImageTaggingService(t *testing.T) { }) t.Run("ValidateImageTaggingRequest", func(tt *testing.T) { - testImageTaggingRequest := ImageTaggingRequestDTO{ + testImageTaggingRequest := types.ImageTaggingRequestDTO{ ImageComment: repository.ImageComment{ Id: 1, Comment: "test Comment", @@ -584,7 +585,7 @@ func TestImageTaggingService(t *testing.T) { t.Run("performTagOperationsAndGetAuditList", func(tt *testing.T) { - testImageTaggingRequest := ImageTaggingRequestDTO{ + testImageTaggingRequest := types.ImageTaggingRequestDTO{ ImageComment: repository.ImageComment{ Id: 1, Comment: "test Comment", diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 0c9be97905..7bddb1300f 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -14,6 +14,7 @@ import ( "os/user" "path/filepath" "strings" + "time" ) type CiCdConfig struct { @@ -472,3 +473,51 @@ const BLOB_STORAGE_S3 = "S3" const BLOB_STORAGE_GCP = "GCP" const BLOB_STORAGE_MINIO = "MINIO" + +type ArtifactsForCiJob struct { + Artifacts []string `json:"artifacts"` +} + +type GitTriggerInfoResponse struct { + CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredByEmail string `json:"triggeredByEmail"` + LastDeployedTime string `json:"lastDeployedTime,omitempty"` + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + Default bool `json:"default,omitempty"` + ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` + Image string `json:"image"` +} + +type Trigger struct { + PipelineId int + CommitHashes map[int]pipelineConfig.GitCommit + CiMaterials []*pipelineConfig.CiPipelineMaterial + TriggeredBy int32 + InvalidateCache bool + ExtraEnvironmentVariables map[string]string // extra env variables which will be used for CI + EnvironmentId int + PipelineType string + CiArtifactLastFetch time.Time + ReferenceCiWorkflowId int +} + +func (obj Trigger) BuildTriggerObject(refCiWorkflow *pipelineConfig.CiWorkflow, + ciMaterials []*pipelineConfig.CiPipelineMaterial, triggeredBy int32, + invalidateCache bool, extraEnvironmentVariables map[string]string, + pipelineType string) { + + obj.PipelineId = refCiWorkflow.CiPipelineId + obj.CommitHashes = refCiWorkflow.GitTriggers + obj.CiMaterials = ciMaterials + obj.TriggeredBy = triggeredBy + obj.InvalidateCache = invalidateCache + obj.EnvironmentId = refCiWorkflow.EnvironmentId + obj.ReferenceCiWorkflowId = refCiWorkflow.Id + obj.InvalidateCache = invalidateCache + obj.ExtraEnvironmentVariables = extraEnvironmentVariables + obj.PipelineType = pipelineType + +} diff --git a/pkg/pipeline/types/ImageTagging.go b/pkg/pipeline/types/ImageTagging.go new file mode 100644 index 0000000000..b868c1e0f1 --- /dev/null +++ b/pkg/pipeline/types/ImageTagging.go @@ -0,0 +1,19 @@ +package types + +import "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + +type ImageTaggingResponseDTO struct { + ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` + AppReleaseTags []string `json:"appReleaseTags"` + ImageComment *repository.ImageComment `json:"imageComment"` + ProdEnvExists bool `json:"tagsEditable"` + HideImageTaggingHardDelete bool `json:"hideImageTaggingHardDelete"` +} + +type ImageTaggingRequestDTO struct { + CreateTags []*repository.ImageTag `json:"createTags"` + SoftDeleteTags []*repository.ImageTag `json:"softDeleteTags"` + ImageComment repository.ImageComment `json:"imageComment"` + HardDeleteTags []*repository.ImageTag `json:"hardDeleteTags"` + ExternalCi bool `json:"-"` +} diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 5bf0c8063d..99b1bc76b0 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -24,6 +24,7 @@ import ( "github.com/devtron-labs/common-lib/blob-storage" bean3 "github.com/devtron-labs/devtron/api/bean" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" @@ -582,3 +583,35 @@ type ChildCdMetadata struct { ChildCdEnvName string `json:"CHILD_CD_ENV_NAME"` ChildCdClusterName string `json:"CHILD_CD_CLUSTER_NAME"` } + +type WorkflowResponse struct { + Id int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"podStatus"` + Message string `json:"message"` + StartedOn time.Time `json:"startedOn"` + FinishedOn time.Time `json:"finishedOn"` + CiPipelineId int `json:"ciPipelineId"` + Namespace string `json:"namespace"` + LogLocation string `json:"logLocation"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredBy int32 `json:"triggeredBy"` + Artifact string `json:"artifact"` + TriggeredByEmail string `json:"triggeredByEmail"` + Stage string `json:"stage"` + ArtifactId int `json:"artifactId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + PodName string `json:"podName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` + ImageComment *repository3.ImageComment `json:"imageComment"` + AppWorkflowId int `json:"appWorkflowId"` + CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` + PipelineType string `json:"pipelineType"` + ReferenceWorkflowId int `json:"referenceWorkflowId"` +} From 42a92c0e401525325786541d0ecf404ef60910a9 Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Sun, 29 Oct 2023 23:59:37 -0400 Subject: [PATCH 5/7] refactored buildlog type --- pkg/pipeline/CdHandler.go | 4 ++-- pkg/pipeline/CiHandler.go | 6 +++--- pkg/pipeline/CiLogService.go | 22 +++++----------------- pkg/pipeline/types/CiCdConfig.go | 13 +++++++++++++ 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 3bf21406bd..d858a4a18b 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -903,7 +903,7 @@ func (impl *CdHandlerImpl) GetRunningWorkflowLogs(environmentId int, pipelineId } func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { - cdLogRequest := BuildLogRequest{ + cdLogRequest := types.BuildLogRequest{ PodName: cdWorkflow.PodName, Namespace: cdWorkflow.Namespace, } @@ -939,7 +939,7 @@ func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pip cdConfig.CdCacheRegion = impl.config.GetDefaultCdLogsBucketRegion() } - cdLogRequest := BuildLogRequest{ + cdLogRequest := types.BuildLogRequest{ PipelineId: cdWorkflow.CdWorkflow.PipelineId, WorkflowId: cdWorkflow.Id, PodName: cdWorkflow.PodName, diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 3bccd7922d..e8779b7db3 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -718,7 +718,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { return bufio.NewReader(strings.NewReader("")), nil, nil } - ciLogRequest := BuildLogRequest{ + ciLogRequest := types.BuildLogRequest{ PodName: ciWorkflow.PodName, Namespace: ciWorkflow.Namespace, } @@ -775,7 +775,7 @@ func (impl *CiHandlerImpl) getLogsFromRepository(pipelineId int, ciWorkflow *pip if strings.Contains(ciWorkflow.LogLocation, "main.log") { logsFilePath = ciWorkflow.LogLocation } - ciLogRequest := BuildLogRequest{ + ciLogRequest := types.BuildLogRequest{ PipelineId: ciWorkflow.CiPipelineId, WorkflowId: ciWorkflow.Id, PodName: ciWorkflow.PodName, @@ -906,7 +906,7 @@ func (impl *CiHandlerImpl) GetHistoricBuildLogs(pipelineId int, workflowId int, if ciConfig.LogsBucket == "" { ciConfig.LogsBucket = impl.config.GetDefaultBuildLogsBucket() } - ciLogRequest := BuildLogRequest{ + ciLogRequest := types.BuildLogRequest{ PipelineId: ciWorkflow.CiPipelineId, WorkflowId: ciWorkflow.Id, PodName: ciWorkflow.PodName, diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index 52430cfa1b..220dcbcd9f 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -22,6 +22,7 @@ import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "go.uber.org/zap" "io" "k8s.io/client-go/kubernetes" @@ -30,8 +31,8 @@ import ( ) type CiLogService interface { - FetchRunningWorkflowLogs(ciLogRequest BuildLogRequest, clusterConfig *k8s.ClusterConfig, isExt bool) (io.ReadCloser, func() error, error) - FetchLogs(baseLogLocationPathConfig string, ciLogRequest BuildLogRequest) (*os.File, func() error, error) + FetchRunningWorkflowLogs(ciLogRequest types.BuildLogRequest, clusterConfig *k8s.ClusterConfig, isExt bool) (io.ReadCloser, func() error, error) + FetchLogs(baseLogLocationPathConfig string, ciLogRequest types.BuildLogRequest) (*os.File, func() error, error) } type CiLogServiceImpl struct { @@ -41,19 +42,6 @@ type CiLogServiceImpl struct { k8sUtil *k8s.K8sUtil } -type BuildLogRequest struct { - PipelineId int - WorkflowId int - PodName string - LogsFilePath string - Namespace string - CloudProvider blob_storage.BlobStorageType - AwsS3BaseConfig *blob_storage.AwsS3BaseConfig - AzureBlobConfig *blob_storage.AzureBlobBaseConfig - GcpBlobBaseConfig *blob_storage.GcpBlobBaseConfig - MinioEndpoint string -} - func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil *k8s.K8sUtil) (*CiLogServiceImpl, error) { _, _, clientSet, err := k8sUtil.GetK8sInClusterConfigAndClients() if err != nil { @@ -68,7 +56,7 @@ func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil }, nil } -func (impl *CiLogServiceImpl) FetchRunningWorkflowLogs(ciLogRequest BuildLogRequest, clusterConfig *k8s.ClusterConfig, isExt bool) (io.ReadCloser, func() error, error) { +func (impl *CiLogServiceImpl) FetchRunningWorkflowLogs(ciLogRequest types.BuildLogRequest, clusterConfig *k8s.ClusterConfig, isExt bool) (io.ReadCloser, func() error, error) { var kubeClient *kubernetes.Clientset kubeClient = impl.kubeClient var err error @@ -96,7 +84,7 @@ func (impl *CiLogServiceImpl) FetchRunningWorkflowLogs(ciLogRequest BuildLogRequ return podLogs, cleanUpFunc, nil } -func (impl *CiLogServiceImpl) FetchLogs(baseLogLocationPathConfig string, logRequest BuildLogRequest) (*os.File, func() error, error) { +func (impl *CiLogServiceImpl) FetchLogs(baseLogLocationPathConfig string, logRequest types.BuildLogRequest) (*os.File, func() error, error) { tempFile := baseLogLocationPathConfig tempFile = filepath.Clean(filepath.Join(tempFile, logRequest.PodName+".log")) diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 7bddb1300f..41a4f8ea3f 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -521,3 +521,16 @@ func (obj Trigger) BuildTriggerObject(refCiWorkflow *pipelineConfig.CiWorkflow, obj.PipelineType = pipelineType } + +type BuildLogRequest struct { + PipelineId int + WorkflowId int + PodName string + LogsFilePath string + Namespace string + CloudProvider blob_storage.BlobStorageType + AwsS3BaseConfig *blob_storage.AwsS3BaseConfig + AzureBlobConfig *blob_storage.AzureBlobBaseConfig + GcpBlobBaseConfig *blob_storage.GcpBlobBaseConfig + MinioEndpoint string +} From 507a719349524bfa680a9f90dfc6ad35fe3b6317 Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Mon, 30 Oct 2023 00:43:49 -0400 Subject: [PATCH 6/7] moved task to type --- pkg/pipeline/StageServiceUtil.go | 36 +++++++++----------------------- pkg/pipeline/types/Task.go | 20 ++++++++++++++++++ 2 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 pkg/pipeline/types/Task.go diff --git a/pkg/pipeline/StageServiceUtil.go b/pkg/pipeline/StageServiceUtil.go index 0a45a6f10f..b3c5f9560c 100644 --- a/pkg/pipeline/StageServiceUtil.go +++ b/pkg/pipeline/StageServiceUtil.go @@ -6,28 +6,12 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin/repository" "gopkg.in/yaml.v2" "strings" ) -type TaskYaml struct { - Version string `yaml:"version"` - CdPipelineConfig []CdPipelineConfig `yaml:"cdPipelineConf"` -} -type CdPipelineConfig struct { - BeforeTasks []*Task `yaml:"beforeStages"` - AfterTasks []*Task `yaml:"afterStages"` -} -type Task struct { - Id int `json:"id,omitempty"` - Index int `json:"index,omitempty"` - Name string `json:"name" yaml:"name"` - Script string `json:"script" yaml:"script"` - OutputLocation string `json:"outputLocation" yaml:"outputLocation"` // file/dir - RunStatus bool `json:"-,omitempty"` // task run was attempted or not -} - var globalInputVariableList = []string{DOCKER_IMAGE, DEPLOYMENT_RELEASE_ID, DEPLOYMENT_UNIQUE_ID, DEVTRON_CD_TRIGGER_TIME, DEVTRON_CD_TRIGGERED_BY, CD_PIPELINE_ENV_NAME_KEY, CD_PIPELINE_CLUSTER_NAME_KEY, APP_NAME} func ConvertStageYamlScriptsToPipelineStageSteps(cdPipeline *bean2.CDPipelineConfigObject) (*bean2.CDPipelineConfigObject, error) { @@ -79,13 +63,13 @@ func StageStepsToCdStageAdapter(deployStage *bean.PipelineStageDto) (*bean2.CdSt Name: deployStage.Name, TriggerType: deployStage.TriggerType, } - cdPipelineConfig := make([]CdPipelineConfig, 0) - beforeTasks := make([]*Task, 0) - afterTasks := make([]*Task, 0) + cdPipelineConfig := make([]types.CdPipelineConfig, 0) + beforeTasks := make([]*types.Task, 0) + afterTasks := make([]*types.Task, 0) for _, step := range deployStage.Steps { if step.InlineStepDetail != nil && checkForOtherParamsInInlineStepDetail(step) { if deployStage.Type == repository2.PIPELINE_STAGE_TYPE_PRE_CD { - beforeTask := &Task{ + beforeTask := &types.Task{ Name: step.Name, Script: step.InlineStepDetail.Script, OutputLocation: strings.Join(step.OutputDirectoryPath, ","), @@ -94,7 +78,7 @@ func StageStepsToCdStageAdapter(deployStage *bean.PipelineStageDto) (*bean2.CdSt } if deployStage.Type == repository2.PIPELINE_STAGE_TYPE_POST_CD { - afterTask := &Task{ + afterTask := &types.Task{ Name: step.Name, Script: step.InlineStepDetail.Script, OutputLocation: strings.Join(step.OutputDirectoryPath, ","), @@ -107,11 +91,11 @@ func StageStepsToCdStageAdapter(deployStage *bean.PipelineStageDto) (*bean2.CdSt } } - cdPipelineConfig = append(cdPipelineConfig, CdPipelineConfig{ + cdPipelineConfig = append(cdPipelineConfig, types.CdPipelineConfig{ BeforeTasks: beforeTasks, AfterTasks: afterTasks, }) - taskYaml := TaskYaml{ + taskYaml := types.TaskYaml{ Version: "", CdPipelineConfig: cdPipelineConfig, } @@ -296,8 +280,8 @@ func StageYamlToPipelineStageAdapter(stageConfig string, stageType repository2.P return pipelineStageDto, nil } -func ToTaskYaml(yamlFile []byte) (*TaskYaml, error) { - taskYaml := &TaskYaml{} +func ToTaskYaml(yamlFile []byte) (*types.TaskYaml, error) { + taskYaml := &types.TaskYaml{} err := yaml.Unmarshal(yamlFile, taskYaml) return taskYaml, err } diff --git a/pkg/pipeline/types/Task.go b/pkg/pipeline/types/Task.go new file mode 100644 index 0000000000..7f9c21e879 --- /dev/null +++ b/pkg/pipeline/types/Task.go @@ -0,0 +1,20 @@ +package types + +type TaskYaml struct { + Version string `yaml:"version"` + CdPipelineConfig []CdPipelineConfig `yaml:"cdPipelineConf"` +} + +type CdPipelineConfig struct { + BeforeTasks []*Task `yaml:"beforeStages"` + AfterTasks []*Task `yaml:"afterStages"` +} + +type Task struct { + Id int `json:"id,omitempty"` + Index int `json:"index,omitempty"` + Name string `json:"name" yaml:"name"` + Script string `json:"script" yaml:"script"` + OutputLocation string `json:"outputLocation" yaml:"outputLocation"` // file/dir + RunStatus bool `json:"-,omitempty"` // task run was attempted or not +} From 1616315bfacc25cf695354f05d22439e1ce08d47 Mon Sep 17 00:00:00 2001 From: Prashant Ghildiyal Date: Mon, 30 Oct 2023 00:57:33 -0400 Subject: [PATCH 7/7] moved git to types --- api/restHandler/GitHostRestHandler.go | 7 ++-- api/restHandler/GitProviderRestHandler.go | 9 +++-- pkg/delete/DeleteServiceFullMode.go | 5 ++- pkg/git/WebhookSecretValidator.go | 6 +-- pkg/pipeline/GitHostConfig.go | 35 ++++++---------- pkg/pipeline/GitRegistryConfig.go | 49 +++++++++-------------- pkg/pipeline/types/Git.go | 29 ++++++++++++++ 7 files changed, 74 insertions(+), 66 deletions(-) create mode 100644 pkg/pipeline/types/Git.go diff --git a/api/restHandler/GitHostRestHandler.go b/api/restHandler/GitHostRestHandler.go index 2cd0627bb2..947f0507f9 100644 --- a/api/restHandler/GitHostRestHandler.go +++ b/api/restHandler/GitHostRestHandler.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/user" "github.com/devtron-labs/devtron/pkg/user/casbin" "github.com/gorilla/mux" @@ -84,7 +85,7 @@ func (impl GitHostRestHandlerImpl) GetGitHosts(w http.ResponseWriter, r *http.Re // RBAC enforcer applying token := r.Header.Get("token") - result := make([]pipeline.GitHostRequest, 0) + result := make([]types.GitHostRequest, 0) for _, item := range res { if ok := impl.enforcer.Enforce(token, casbin.ResourceGit, casbin.ActionGet, strings.ToLower(item.Name)); ok { result = append(result, item) @@ -136,7 +137,7 @@ func (impl GitHostRestHandlerImpl) CreateGitHost(w http.ResponseWriter, r *http. decoder := json.NewDecoder(r.Body) - var bean pipeline.GitHostRequest + var bean types.GitHostRequest err = decoder.Decode(&bean) if err != nil { impl.logger.Errorw("request err, CreateGitHost", "err", err, "payload", bean) @@ -291,6 +292,6 @@ func (impl GitHostRestHandlerImpl) GetWebhookDataMetaConfig(w http.ResponseWrite type WebhookDataMetaConfigResponse struct { GitHostId int `json:"gitHostId"` - GitHost *pipeline.GitHostRequest `json:"gitHost"` + GitHost *types.GitHostRequest `json:"gitHost"` WebhookEvents []*gitSensor.WebhookEventConfig `json:"webhookEvents"` } diff --git a/api/restHandler/GitProviderRestHandler.go b/api/restHandler/GitProviderRestHandler.go index 16bbdd9d32..16a5f4359f 100644 --- a/api/restHandler/GitProviderRestHandler.go +++ b/api/restHandler/GitProviderRestHandler.go @@ -22,6 +22,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" delete2 "github.com/devtron-labs/devtron/pkg/delete" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/team" "github.com/devtron-labs/devtron/pkg/user" "github.com/devtron-labs/devtron/pkg/user/casbin" @@ -81,7 +82,7 @@ func (impl GitProviderRestHandlerImpl) SaveGitRepoConfig(w http.ResponseWriter, common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - var bean pipeline.GitRegistry + var bean types.GitRegistry err = decoder.Decode(&bean) if err != nil { impl.logger.Errorw("request err, SaveGitRepoConfig", "err", err, "payload", bean) @@ -135,7 +136,7 @@ func (impl GitProviderRestHandlerImpl) FetchAllGitProviders(w http.ResponseWrite // RBAC enforcer applying token := r.Header.Get("token") - result := make([]pipeline.GitRegistry, 0) + result := make([]types.GitRegistry, 0) for _, item := range res { if ok := impl.enforcer.Enforce(token, casbin.ResourceGit, casbin.ActionGet, strings.ToLower(item.Name)); ok { result = append(result, item) @@ -174,7 +175,7 @@ func (impl GitProviderRestHandlerImpl) UpdateGitRepoConfig(w http.ResponseWriter common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - var bean pipeline.GitRegistry + var bean types.GitRegistry err = decoder.Decode(&bean) if err != nil { impl.logger.Errorw("request err, UpdateGitRepoConfig", "err", err, "payload", bean) @@ -213,7 +214,7 @@ func (impl GitProviderRestHandlerImpl) DeleteGitRepoConfig(w http.ResponseWriter common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - var bean pipeline.GitRegistry + var bean types.GitRegistry err = decoder.Decode(&bean) if err != nil { impl.logger.Errorw("request err, DeleteGitRepoConfig", "err", err, "payload", bean) diff --git a/pkg/delete/DeleteServiceFullMode.go b/pkg/delete/DeleteServiceFullMode.go index 474334022b..f8d9ddd3ae 100644 --- a/pkg/delete/DeleteServiceFullMode.go +++ b/pkg/delete/DeleteServiceFullMode.go @@ -5,12 +5,13 @@ import ( dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/go-pg/pg" "go.uber.org/zap" ) type DeleteServiceFullMode interface { - DeleteGitProvider(deleteRequest *pipeline.GitRegistry) error + DeleteGitProvider(deleteRequest *types.GitRegistry) error DeleteDockerRegistryConfig(deleteRequest *pipeline.DockerArtifactStoreBean) error CanDeleteContainerRegistryConfig(storeId string) bool } @@ -40,7 +41,7 @@ func NewDeleteServiceFullModeImpl(logger *zap.SugaredLogger, dockerRegistryRepository: dockerRegistryRepository, } } -func (impl DeleteServiceFullModeImpl) DeleteGitProvider(deleteRequest *pipeline.GitRegistry) error { +func (impl DeleteServiceFullModeImpl) DeleteGitProvider(deleteRequest *types.GitRegistry) error { //finding if this git account is used in any git material, if yes then will not delete materials, err := impl.gitMaterialRepository.FindByGitProviderId(deleteRequest.Id) if err != nil && err != pg.ErrNoRows { diff --git a/pkg/git/WebhookSecretValidator.go b/pkg/git/WebhookSecretValidator.go index 6e29198a21..05ee5c5ad7 100644 --- a/pkg/git/WebhookSecretValidator.go +++ b/pkg/git/WebhookSecretValidator.go @@ -21,14 +21,14 @@ import ( "crypto/hmac" "crypto/sha1" "encoding/hex" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "go.uber.org/zap" "net/http" "strings" ) type WebhookSecretValidator interface { - ValidateSecret(r *http.Request, secretInUrl string, requestBodyBytes []byte, gitHost *pipeline.GitHostRequest) bool + ValidateSecret(r *http.Request, secretInUrl string, requestBodyBytes []byte, gitHost *types.GitHostRequest) bool } type WebhookSecretValidatorImpl struct { @@ -51,7 +51,7 @@ const ( // URL_APPEND : Secret will come in URL (last path param of URL) // PLAIN_TEXT : Plain text value in request header // SHA1 : SHA1 encrypted text in request header -func (impl *WebhookSecretValidatorImpl) ValidateSecret(r *http.Request, secretInUrl string, requestBodyBytes []byte, gitHost *pipeline.GitHostRequest) bool { +func (impl *WebhookSecretValidatorImpl) ValidateSecret(r *http.Request, secretInUrl string, requestBodyBytes []byte, gitHost *types.GitHostRequest) bool { secretValidator := gitHost.SecretValidator impl.logger.Debug("Validating signature for secret validator : ", secretValidator) diff --git a/pkg/pipeline/GitHostConfig.go b/pkg/pipeline/GitHostConfig.go index 6369c9911c..b535598862 100644 --- a/pkg/pipeline/GitHostConfig.go +++ b/pkg/pipeline/GitHostConfig.go @@ -22,6 +22,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/attributes" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" "github.com/juju/errors" "go.uber.org/zap" @@ -29,9 +30,9 @@ import ( ) type GitHostConfig interface { - GetAll() ([]GitHostRequest, error) - GetById(id int) (*GitHostRequest, error) - Create(request *GitHostRequest) (int, error) + GetAll() ([]types.GitHostRequest, error) + GetById(id int) (*types.GitHostRequest, error) + Create(request *types.GitHostRequest) (int, error) } type GitHostConfigImpl struct { @@ -48,29 +49,17 @@ func NewGitHostConfigImpl(gitHostRepo repository.GitHostRepository, logger *zap. } } -type GitHostRequest struct { - Id int `json:"id,omitempty" validate:"number"` - Name string `json:"name,omitempty" validate:"required"` - Active bool `json:"active"` - WebhookUrl string `json:"webhookUrl"` - WebhookSecret string `json:"webhookSecret"` - EventTypeHeader string `json:"eventTypeHeader"` - SecretHeader string `json:"secretHeader"` - SecretValidator string `json:"secretValidator"` - UserId int32 `json:"-"` -} - -//get all git hosts -func (impl GitHostConfigImpl) GetAll() ([]GitHostRequest, error) { +// get all git hosts +func (impl GitHostConfigImpl) GetAll() ([]types.GitHostRequest, error) { impl.logger.Debug("get all hosts request") hosts, err := impl.gitHostRepo.FindAll() if err != nil { impl.logger.Errorw("error in fetching all git hosts", "err", err) return nil, err } - var gitHosts []GitHostRequest + var gitHosts []types.GitHostRequest for _, host := range hosts { - hostRes := GitHostRequest{ + hostRes := types.GitHostRequest{ Id: host.Id, Name: host.Name, Active: host.Active, @@ -80,8 +69,8 @@ func (impl GitHostConfigImpl) GetAll() ([]GitHostRequest, error) { return gitHosts, err } -//get git host by Id -func (impl GitHostConfigImpl) GetById(id int) (*GitHostRequest, error) { +// get git host by Id +func (impl GitHostConfigImpl) GetById(id int) (*types.GitHostRequest, error) { impl.logger.Debug("get hosts request for Id", id) host, err := impl.gitHostRepo.FindOneById(id) if err != nil { @@ -104,7 +93,7 @@ func (impl GitHostConfigImpl) GetById(id int) (*GitHostRequest, error) { } webhookUrl := webhookUrlPrepend + host.WebhookUrl - gitHost := &GitHostRequest{ + gitHost := &types.GitHostRequest{ Id: host.Id, Name: host.Name, Active: host.Active, @@ -119,7 +108,7 @@ func (impl GitHostConfigImpl) GetById(id int) (*GitHostRequest, error) { } // Create in DB -func (impl GitHostConfigImpl) Create(request *GitHostRequest) (int, error) { +func (impl GitHostConfigImpl) Create(request *types.GitHostRequest) (int, error) { impl.logger.Debugw("get git host create request", "req", request) exist, err := impl.gitHostRepo.Exists(request.Name) if err != nil { diff --git a/pkg/pipeline/GitRegistryConfig.go b/pkg/pipeline/GitRegistryConfig.go index ccea48d7f4..93b2140be7 100644 --- a/pkg/pipeline/GitRegistryConfig.go +++ b/pkg/pipeline/GitRegistryConfig.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/internal/constants" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/sql" "github.com/juju/errors" "go.uber.org/zap" @@ -32,12 +33,12 @@ import ( ) type GitRegistryConfig interface { - Create(request *GitRegistry) (*GitRegistry, error) - GetAll() ([]GitRegistry, error) - FetchAllGitProviders() ([]GitRegistry, error) - FetchOneGitProvider(id string) (*GitRegistry, error) - Update(request *GitRegistry) (*GitRegistry, error) - Delete(request *GitRegistry) error + Create(request *types.GitRegistry) (*types.GitRegistry, error) + GetAll() ([]types.GitRegistry, error) + FetchAllGitProviders() ([]types.GitRegistry, error) + FetchOneGitProvider(id string) (*types.GitRegistry, error) + Update(request *types.GitRegistry) (*types.GitRegistry, error) + Delete(request *types.GitRegistry) error } type GitRegistryConfigImpl struct { logger *zap.SugaredLogger @@ -45,20 +46,6 @@ type GitRegistryConfigImpl struct { GitSensorGrpcClient gitSensor.Client } -type GitRegistry struct { - Id int `json:"id,omitempty" validate:"number"` - Name string `json:"name,omitempty" validate:"required"` - Url string `json:"url,omitempty"` - UserName string `json:"userName,omitempty"` - Password string `json:"password,omitempty"` - SshPrivateKey string `json:"sshPrivateKey,omitempty"` - AccessToken string `json:"accessToken,omitempty"` - AuthMode repository.AuthMode `json:"authMode,omitempty" validate:"required"` - Active bool `json:"active"` - UserId int32 `json:"-"` - GitHostId int `json:"gitHostId"` -} - func NewGitRegistryConfigImpl(logger *zap.SugaredLogger, gitProviderRepo repository.GitProviderRepository, GitSensorClient gitSensor.Client) *GitRegistryConfigImpl { return &GitRegistryConfigImpl{ @@ -68,7 +55,7 @@ func NewGitRegistryConfigImpl(logger *zap.SugaredLogger, gitProviderRepo reposit } } -func (impl GitRegistryConfigImpl) Create(request *GitRegistry) (*GitRegistry, error) { +func (impl GitRegistryConfigImpl) Create(request *types.GitRegistry) (*types.GitRegistry, error) { impl.logger.Debugw("get repo create request", "req", request) exist, err := impl.gitProviderRepo.ProviderExists(request.Url) if err != nil { @@ -128,16 +115,16 @@ func (impl GitRegistryConfigImpl) Create(request *GitRegistry) (*GitRegistry, er } // get all active git providers -func (impl GitRegistryConfigImpl) GetAll() ([]GitRegistry, error) { +func (impl GitRegistryConfigImpl) GetAll() ([]types.GitRegistry, error) { impl.logger.Debug("get all provider request") providers, err := impl.gitProviderRepo.FindAllActiveForAutocomplete() if err != nil { impl.logger.Errorw("error in fetch all git providers", "err", err) return nil, err } - var gitProviders []GitRegistry + var gitProviders []types.GitRegistry for _, provider := range providers { - providerRes := GitRegistry{ + providerRes := types.GitRegistry{ Id: provider.Id, Name: provider.Name, Url: provider.Url, @@ -149,16 +136,16 @@ func (impl GitRegistryConfigImpl) GetAll() ([]GitRegistry, error) { return gitProviders, err } -func (impl GitRegistryConfigImpl) FetchAllGitProviders() ([]GitRegistry, error) { +func (impl GitRegistryConfigImpl) FetchAllGitProviders() ([]types.GitRegistry, error) { impl.logger.Debug("fetch all git providers from db") providers, err := impl.gitProviderRepo.FindAll() if err != nil { impl.logger.Errorw("error in fetch all git providers", "err", err) return nil, err } - var gitProviders []GitRegistry + var gitProviders []types.GitRegistry for _, provider := range providers { - providerRes := GitRegistry{ + providerRes := types.GitRegistry{ Id: provider.Id, Name: provider.Name, Url: provider.Url, @@ -176,7 +163,7 @@ func (impl GitRegistryConfigImpl) FetchAllGitProviders() ([]GitRegistry, error) return gitProviders, err } -func (impl GitRegistryConfigImpl) FetchOneGitProvider(providerId string) (*GitRegistry, error) { +func (impl GitRegistryConfigImpl) FetchOneGitProvider(providerId string) (*types.GitRegistry, error) { impl.logger.Debug("fetch git provider by ID from db") provider, err := impl.gitProviderRepo.FindOne(providerId) if err != nil { @@ -184,7 +171,7 @@ func (impl GitRegistryConfigImpl) FetchOneGitProvider(providerId string) (*GitRe return nil, err } - providerRes := &GitRegistry{ + providerRes := &types.GitRegistry{ Id: provider.Id, Name: provider.Name, Url: provider.Url, @@ -201,7 +188,7 @@ func (impl GitRegistryConfigImpl) FetchOneGitProvider(providerId string) (*GitRe return providerRes, err } -func (impl GitRegistryConfigImpl) Update(request *GitRegistry) (*GitRegistry, error) { +func (impl GitRegistryConfigImpl) Update(request *types.GitRegistry) (*types.GitRegistry, error) { impl.logger.Debugw("get repo create request", "req", request) /* @@ -274,7 +261,7 @@ func (impl GitRegistryConfigImpl) Update(request *GitRegistry) (*GitRegistry, er return request, nil } -func (impl GitRegistryConfigImpl) Delete(request *GitRegistry) error { +func (impl GitRegistryConfigImpl) Delete(request *types.GitRegistry) error { providerId := strconv.Itoa(request.Id) gitProviderConfig, err := impl.gitProviderRepo.FindOne(providerId) if err != nil { diff --git a/pkg/pipeline/types/Git.go b/pkg/pipeline/types/Git.go new file mode 100644 index 0000000000..4c4ee084f0 --- /dev/null +++ b/pkg/pipeline/types/Git.go @@ -0,0 +1,29 @@ +package types + +import "github.com/devtron-labs/devtron/internal/sql/repository" + +type GitRegistry struct { + Id int `json:"id,omitempty" validate:"number"` + Name string `json:"name,omitempty" validate:"required"` + Url string `json:"url,omitempty"` + UserName string `json:"userName,omitempty"` + Password string `json:"password,omitempty"` + SshPrivateKey string `json:"sshPrivateKey,omitempty"` + AccessToken string `json:"accessToken,omitempty"` + AuthMode repository.AuthMode `json:"authMode,omitempty" validate:"required"` + Active bool `json:"active"` + UserId int32 `json:"-"` + GitHostId int `json:"gitHostId"` +} + +type GitHostRequest struct { + Id int `json:"id,omitempty" validate:"number"` + Name string `json:"name,omitempty" validate:"required"` + Active bool `json:"active"` + WebhookUrl string `json:"webhookUrl"` + WebhookSecret string `json:"webhookSecret"` + EventTypeHeader string `json:"eventTypeHeader"` + SecretHeader string `json:"secretHeader"` + SecretValidator string `json:"secretValidator"` + UserId int32 `json:"-"` +}