feat: Variables support in pre-post CI, CD and Jobs

api/bean/ValuesOverrideRequest.go

@@ -20,6 +20,7 @@ package bean
 import (
 	"encoding/json"
 	"github.com/devtron-labs/devtron/internal/sql/models"
+	"github.com/devtron-labs/devtron/pkg/pipeline/repository"
 	"time"
 )
 
@@ -39,6 +40,17 @@ const (
 	DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER DeploymentConfigurationType = "SPECIFIC_TRIGGER_CONFIG"
 )
 
+func (workflowType WorkflowType) WorkflowTypeToStageType() repository.PipelineStageType {
+	switch workflowType {
+	case CD_WORKFLOW_TYPE_PRE:
+		return repository.PIPELINE_STAGE_TYPE_PRE_CD
+	case CD_WORKFLOW_TYPE_POST:
+		return repository.PIPELINE_STAGE_TYPE_POST_CD
+	default:
+		return ""
+	}
+}
+
 type ValuesOverrideRequest struct {
 	PipelineId                            int                         `json:"pipelineId" validate:"required"`
 	AppId                                 int                         `json:"appId" validate:"required"`

pkg/bulkAction/BulkUpdateService.go

@@ -562,7 +562,7 @@ func (impl BulkUpdateServiceImpl) extractAndMapVariables(template string, entity
 	err = impl.variableEntityMappingService.UpdateVariablesForEntity(usedVariables, repository5.Entity{
 		EntityType: entityType,
 		EntityId:   entityId,
-	}, userId)
+	}, userId, nil)
 	if err != nil {
 		return err
 	}

pkg/chart/ChartService.go

@@ -173,9 +173,6 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository,
 		envLevelAppMetricsRepository:     envLevelAppMetricsRepository,
 		client:                           client,
 		deploymentTemplateHistoryService: deploymentTemplateHistoryService,
-		variableEntityMappingService:     variableEntityMappingService,
-		variableTemplateParser:           variableTemplateParser,
-		scopedVariableService:            scopedVariableService,
 	}
 }
 
@@ -483,7 +480,7 @@ func (impl ChartServiceImpl) extractAndMapVariables(template string, entityId in
 	err = impl.variableEntityMappingService.UpdateVariablesForEntity(usedVariables, repository5.Entity{
 		EntityType: entityType,
 		EntityId:   entityId,
-	}, userId)
+	}, userId, nil)
 	if err != nil {
 		return err
 	}

pkg/cluster/EnvironmentService.go

@@ -419,6 +419,7 @@ func (impl EnvironmentServiceImpl) GetEnvironmentListForAutocomplete(isDeploymen
 				Description:            model.Description,
 				IsVirtualEnvironment:   model.IsVirtualEnvironment,
 				AllowedDeploymentTypes: allowedDeploymentConfigString,
+				ClusterId:              model.ClusterId,
 			})
 		}
 	} else {
@@ -432,6 +433,7 @@ func (impl EnvironmentServiceImpl) GetEnvironmentListForAutocomplete(isDeploymen
 				ClusterName:           model.Cluster.ClusterName,
 				Description:           model.Description,
 				IsVirtualEnvironment:  model.IsVirtualEnvironment,
+				ClusterId:             model.ClusterId,
 			})
 		}
 	}

pkg/pipeline/CiCdPipelineOrchestrator_test.go

@@ -145,7 +145,7 @@ func InitClusterNoteService() {
 	userAuthService := user.NewUserAuthServiceImpl(nil, nil, nil, nil, nil, nil, nil)
 	prePostCdScriptHistoryService := history.NewPrePostCdScriptHistoryServiceImpl(logger, nil, nil, nil)
 	prePostCiScriptHistoryService := history.NewPrePostCiScriptHistoryServiceImpl(logger, nil)
-	pipelineStageService := NewPipelineStageService(logger, nil, nil, nil)
+	pipelineStageService := NewPipelineStageService(logger, nil, nil, nil, nil, nil, nil)
 	ciTemplateOverrideRepository := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(conn, logger)
 	ciTemplateService := *NewCiTemplateServiceImpl(logger, nil, nil, nil)
 	gitMaterialHistoryService := history.NewGitMaterialHistoryServiceImpl(nil, logger)

pkg/pipeline/CiService.go

@@ -31,6 +31,9 @@ import (
 	"github.com/devtron-labs/devtron/pkg/pipeline/repository"
 	repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository"
 	"github.com/devtron-labs/devtron/pkg/user"
+	"github.com/devtron-labs/devtron/pkg/variables"
+	"github.com/devtron-labs/devtron/pkg/variables/models"
+	repository4 "github.com/devtron-labs/devtron/pkg/variables/repository"
 	"github.com/go-pg/pg"
 	"path/filepath"
 	"strconv"
@@ -70,6 +73,7 @@ type CiServiceImpl struct {
 	appCrudOperationService       app.AppCrudOperationService
 	envRepository                 repository1.EnvironmentRepository
 	appRepository                 appRepository.AppRepository
+	variableSnapshotHistoryService variables.VariableSnapshotHistoryService
 }
 
 func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService,
@@ -79,7 +83,9 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
 	prePostCiScriptHistoryService history.PrePostCiScriptHistoryService,
 	pipelineStageService PipelineStageService,
 	userService user.UserService,
-	ciTemplateService CiTemplateService, appCrudOperationService app.AppCrudOperationService, envRepository repository1.EnvironmentRepository, appRepository appRepository.AppRepository) *CiServiceImpl {
+	ciTemplateService CiTemplateService, appCrudOperationService app.AppCrudOperationService, envRepository repository1.EnvironmentRepository, appRepository appRepository.AppRepository,
+	variableSnapshotHistoryService variables.VariableSnapshotHistoryService,
+) *CiServiceImpl {
 	return &CiServiceImpl{
 		Logger:                        Logger,
 		workflowService:               workflowService,
@@ -97,6 +103,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService
 		appCrudOperationService:       appCrudOperationService,
 		envRepository:                 envRepository,
 		appRepository:                 appRepository,
+		variableSnapshotHistoryService: variableSnapshotHistoryService,
 	}
 }
 
@@ -137,23 +144,35 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) {
 		impl.Logger.Errorw("could not fetch ci config", "pipeline", trigger.PipelineId)
 		return 0, err
 	}
+
+	scope := models.Scope{
+		AppId: pipeline.App.Id,
+	}
 	env, isJob, err := impl.getEnvironmentForJob(pipeline, trigger)
 	if err != nil {
 		return 0, err
 	}
 	if isJob && env != nil {
 		ciWorkflowConfig.Namespace = env.Namespace
+
+		//This will be populated for jobs running in selected environment
+		scope.EnvId = env.Id
+		scope.ClusterId = env.ClusterId
 	}
 	if ciWorkflowConfig.Namespace == "" {
 		ciWorkflowConfig.Namespace = impl.ciConfig.DefaultNamespace
 	}
 
-	preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent)
-
+	//preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent)
+	prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent, scope)
 	if err != nil {
 		impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id)
 		return 0, err
 	}
+	preCiSteps := prePostAndRefPluginResponse.PreStageSteps
+	postCiSteps := prePostAndRefPluginResponse.PostStageSteps
+	refPluginsData := prePostAndRefPluginResponse.RefPluginData
+	variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot
 
 	if len(preCiSteps) == 0 && isJob {
 		return 0, &util.ApiError{
@@ -196,6 +215,21 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) {
 	}
 	impl.Logger.Debugw("ci triggered", "wf name ", createdWf.Name, " pipeline ", trigger.PipelineId)
 
+	//Save Scoped VariableSnapshot
+	if len(variableSnapshot) > 0 {
+		variableMapBytes, _ := json.Marshal(variableSnapshot)
+		err := impl.variableSnapshotHistoryService.SaveVariableHistoriesForTrigger([]*repository4.VariableSnapshotHistoryBean{{
+			VariableSnapshot: variableMapBytes,
+			HistoryReference: repository4.HistoryReference{
+				HistoryReferenceId:   savedCiWf.Id,
+				HistoryReferenceType: repository4.HistoryReferenceTypeCIWORKFLOW,
+			},
+		}}, trigger.TriggeredBy)
+		if err != nil {
+			impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err)
+		}
+	}
+
 	middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc()
 	go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest)
 	return savedCiWf.Id, err
@@ -434,22 +468,14 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.
 		}
 	}
 
-	var err error
 	if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) {
 		//found beforeDockerBuildScripts/afterDockerBuildScripts
 		//building preCiSteps & postCiSteps from them, refPluginsData not needed
 		preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts)
 		postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts)
 		refPluginsData = []*bean2.RefPluginObject{}
-	} else {
-		//beforeDockerBuildScripts & afterDockerBuildScripts not found
-		//getting preCiStepsData, postCiStepsData & refPluginsData
-		preCiSteps, postCiSteps, refPluginsData, err = impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent)
-		if err != nil {
-			impl.Logger.Errorw("error in getting pre, post & refPlugin steps data for wf request", "err", err, "ciPipelineId", pipeline.Id)
-			return nil, err
-		}
 	}
+
 	dockerImageTag := impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id)
 	if ciWorkflowConfig.CiCacheBucket == "" {
 		ciWorkflowConfig.CiCacheBucket = impl.ciConfig.DefaultCacheBucket

pkg/pipeline/ConfigMapService_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/cluster/repository"
 	mocks6 "github.com/devtron-labs/devtron/pkg/cluster/repository/mocks"
 	mocks3 "github.com/devtron-labs/devtron/pkg/commonService/mocks"
+	"github.com/devtron-labs/devtron/pkg/pipeline/bean"
 	mocks5 "github.com/devtron-labs/devtron/pkg/pipeline/history/mocks"
 	"github.com/go-pg/pg"
 	"github.com/stretchr/testify/assert"
@@ -45,7 +46,7 @@ func TestConfigMapServiceImpl_ConfigSecretEnvironmentCreate(t *testing.T) {
 		Deleted:       true,
 	}
 	type args struct {
-		createJobEnvOverrideRequest *CreateJobEnvOverridePayload
+		createJobEnvOverrideRequest *bean.CreateJobEnvOverridePayload
 		getByAppError               error
 		getByAppResponse            *chartConfig.ConfigMapEnvModel
 	}
@@ -58,7 +59,7 @@ func TestConfigMapServiceImpl_ConfigSecretEnvironmentCreate(t *testing.T) {
 		{
 			name: "create environment override",
 			args: args{
-				createJobEnvOverrideRequest: &CreateJobEnvOverridePayload{
+				createJobEnvOverrideRequest: &bean.CreateJobEnvOverridePayload{
 					AppId: 22,
 					EnvId: 5,
 				},
@@ -71,7 +72,7 @@ func TestConfigMapServiceImpl_ConfigSecretEnvironmentCreate(t *testing.T) {
 		{
 			name: "create deleted override",
 			args: args{
-				createJobEnvOverrideRequest: &CreateJobEnvOverridePayload{
+				createJobEnvOverrideRequest: &bean.CreateJobEnvOverridePayload{
 					AppId: 22,
 					EnvId: 5,
 				},
@@ -129,7 +130,7 @@ func TestConfigMapServiceImpl_ConfigSecretEnvironmentDelete(t *testing.T) {
 	envRepository := mocks6.NewEnvironmentRepository(t)
 
 	type args struct {
-		createJobEnvOverrideRequest *CreateJobEnvOverridePayload
+		createJobEnvOverrideRequest *bean.CreateJobEnvOverridePayload
 	}
 	configMap := &chartConfig.ConfigMapEnvModel{
 		AppId:         1,
@@ -144,7 +145,7 @@ func TestConfigMapServiceImpl_ConfigSecretEnvironmentDelete(t *testing.T) {
 		{
 			name: "Delete configMap",
 			args: args{
-				createJobEnvOverrideRequest: &CreateJobEnvOverridePayload{
+				createJobEnvOverrideRequest: &bean.CreateJobEnvOverridePayload{
 					AppId: 1,
 					EnvId: 1,
 				},

pkg/pipeline/GitopsOrHelmOption_test.go

@@ -24,7 +24,7 @@ func TestGitopsOrHelmOption(t *testing.T) {
 			nil, nil, nil,
 			nil, nil, nil, nil,
 			nil, nil, nil, nil,
-			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 
 		pipelineCreateRequest := &bean.CdPipelines{
 			Pipelines: []*bean.CDPipelineConfigObject{
@@ -79,7 +79,7 @@ func TestGitopsOrHelmOption(t *testing.T) {
 			nil, nil, nil,
 			nil, nil, nil, nil,
 			nil, nil, nil, nil,
-			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 
 		pipelineCreateRequest := &bean.CdPipelines{
 			Pipelines: []*bean.CDPipelineConfigObject{
@@ -135,7 +135,7 @@ func TestGitopsOrHelmOption(t *testing.T) {
 			nil, nil, nil,
 			nil, nil, nil, nil,
 			nil, nil, nil, nil,
-			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 
 		pipelineCreateRequest := &bean.CdPipelines{
 			Pipelines: []*bean.CDPipelineConfigObject{
@@ -190,7 +190,7 @@ func TestGitopsOrHelmOption(t *testing.T) {
 			nil, nil, nil,
 			nil, nil, nil, nil,
 			nil, nil, nil, nil,
-			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 
 		pipelineCreateRequestHelm := &bean.CdPipelines{
 			Pipelines: []*bean.CDPipelineConfigObject{
@@ -284,7 +284,7 @@ func TestGitopsOrHelmOption(t *testing.T) {
 			nil, nil, nil,
 			nil, nil, nil, nil,
 			nil, nil, nil, nil,
-			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: false}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 
 		pipelineCreateRequest := &bean.CdPipelines{
 			Pipelines: []*bean.CDPipelineConfigObject{
@@ -341,7 +341,7 @@ func TestGitopsOrHelmOption(t *testing.T) {
 			nil, nil, nil,
 			nil, nil, nil, nil,
 			nil, nil, nil, nil,
-			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+			nil, nil, nil, nil, nil, nil, nil, nil, &DeploymentServiceTypeConfig{IsInternalUse: true}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
 
 		pipelineCreateRequest := &bean.CdPipelines{
 			Pipelines: []*bean.CDPipelineConfigObject{

pkg/pipeline/PipelineBuilder.go

@@ -3132,7 +3132,7 @@ func (impl *PipelineBuilderImpl) createCdPipeline(ctx context.Context, app *app2
 		return 0, err
 	}
 	//VARIABLE_MAPPING_UPDATE
-	err = impl.extractAndMapVariables(envOverride.EnvOverrideValues, envOverride.Id, repository6.EntityTypeDeploymentTemplateEnvLevel, envOverride.UpdatedBy)
+	err = impl.extractAndMapVariables(envOverride.EnvOverrideValues, envOverride.Id, repository6.EntityTypeDeploymentTemplateEnvLevel, envOverride.UpdatedBy, tx)
 	if err != nil {
 		return 0, err
 	}
@@ -3177,15 +3177,15 @@ func (impl *PipelineBuilderImpl) createCdPipeline(ctx context.Context, app *app2
 	return pipelineId, nil
 }
 
-func (impl PipelineBuilderImpl) extractAndMapVariables(template string, entityId int, entityType repository6.EntityType, userId int32) error {
+func (impl PipelineBuilderImpl) extractAndMapVariables(template string, entityId int, entityType repository6.EntityType, userId int32, tx *pg.Tx) error {
 	usedVariables, err := impl.variableTemplateParser.ExtractVariables(template)
 	if err != nil {
 		return err
 	}
 	err = impl.variableEntityMappingService.UpdateVariablesForEntity(usedVariables, repository6.Entity{
 		EntityType: entityType,
 		EntityId:   entityId,
-	}, userId)
+	}, userId, tx)
 	if err != nil {
 		return err
 	}