feat: Scoped variables primitive handling (#4033)

* adding system variables in get scoped data

* system variables

* bug fixes

* minor fix

* image tag variable exposed

* refactoring

* refactoring

* fix escaping $

* wf resolver to type string

* fixing logic

* addressed comments and cleaned up code

* cleaning up

* parser to return if scoped variables disabled

---------

Co-authored-by: Kripansh <[email protected]>

Author: subhashish-devtron

api/bean/ValuesOverrideRequest.go

@@ -72,6 +72,7 @@ type ValuesOverrideRequest struct {
 	AppName                               string                      `json:"-"`
 	PipelineName                          string                      `json:"-"`
 	DeploymentAppType                     string                      `json:"-"`
+	ImageTag                              string                      `json:"-"`
 }
 
 type BulkCdDeployEvent struct {

pkg/app/AppService.go

@@ -1256,13 +1256,8 @@ func (impl *AppServiceImpl) GetDeploymentStrategyByTriggerType(overrideRequest *
 
 func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) {
 
-	//VARIABLE different cases for variable resolution
-	scope := resourceQualifiers.Scope{
-		AppId:     overrideRequest.AppId,
-		EnvId:     overrideRequest.EnvId,
-		ClusterId: overrideRequest.ClusterId,
-	}
 	envOverride := &chartConfig.EnvConfigOverride{}
+
 	var err error
 	if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER {
 		_, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId")
@@ -1375,7 +1370,30 @@ func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.Va
 			envOverride.Chart = chart
 		}
 
+		_, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById")
+		env, err := impl.envRepository.FindById(envOverride.TargetEnvironment)
+		span.End()
+		if err != nil {
+			impl.logger.Errorw("unable to find env", "err", err)
+			return nil, err
+		}
+		envOverride.Environment = env
+
+		//VARIABLE different cases for variable resolution
+		scope := resourceQualifiers.Scope{
+			AppId:     overrideRequest.AppId,
+			EnvId:     overrideRequest.EnvId,
+			ClusterId: overrideRequest.ClusterId,
+			SystemMetadata: &resourceQualifiers.SystemMetadata{
+				EnvironmentName: env.Name,
+				ClusterName:     env.Cluster.ClusterName,
+				Namespace:       env.Namespace,
+				ImageTag:        overrideRequest.ImageTag,
+			},
+		}
+
 		if envOverride.IsOverride {
+
 			resolvedTemplate, variableMap, err := impl.extractVariablesAndResolveTemplate(scope, envOverride.EnvOverrideValues, repository6.Entity{
 				EntityType: repository6.EntityTypeDeploymentTemplateEnvLevel,
 				EntityId:   envOverride.Id,
@@ -1397,15 +1415,7 @@ func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.Va
 			envOverride.VariableSnapshot = variableMap
 		}
 	}
-	_, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById")
-	env, err := impl.envRepository.FindById(envOverride.TargetEnvironment)
-	span.End()
-	if err != nil {
-		impl.logger.Errorw("unable to find env", "err", err)
-		return nil, err
-	}
-	envOverride.Environment = env
-	//VARIABLE_RESOLVE
+
 	return envOverride, nil
 }
 
@@ -1490,6 +1500,14 @@ func (impl *AppServiceImpl) GetValuesOverrideForTrigger(overrideRequest *bean.Va
 		return valuesOverrideResponse, err
 	}
 
+	_, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get")
+	artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId)
+	span.End()
+	if err != nil {
+		return valuesOverrideResponse, err
+	}
+	overrideRequest.ImageTag = artifact.Image
+
 	envOverride, err := impl.GetEnvOverrideByTriggerType(overrideRequest, triggeredAt, ctx)
 	if err != nil {
 		impl.logger.Errorw("error in getting env override by trigger type", "err", err)
@@ -1505,12 +1523,6 @@ func (impl *AppServiceImpl) GetValuesOverrideForTrigger(overrideRequest *bean.Va
 		impl.logger.Errorw("error in getting strategy by trigger type", "err", err)
 		return valuesOverrideResponse, err
 	}
-	_, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get")
-	artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId)
-	span.End()
-	if err != nil {
-		return valuesOverrideResponse, err
-	}
 	_, span = otel.Tracer("orchestrator").Start(ctx, "getDbMigrationOverride")
 	//FIXME: how to determine rollback
 	//we can't depend on ciArtifact ID because CI pipeline can be manually triggered in any order regardless of sourcecode status

pkg/pipeline/CiService.go

@@ -166,6 +166,12 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) {
 		//This will be populated for jobs running in selected environment
 		scope.EnvId = env.Id
 		scope.ClusterId = env.ClusterId
+
+		scope.SystemMetadata = &resourceQualifiers.SystemMetadata{
+			EnvironmentName: env.Name,
+			ClusterName:     env.Cluster.ClusterName,
+			Namespace:       env.Namespace,
+		}
 	}
 	if ciWorkflowConfig.Namespace == "" {
 		ciWorkflowConfig.Namespace = impl.config.GetDefaultNamespace()

pkg/pipeline/WorkflowDagExecutor.go

@@ -902,6 +902,12 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor
 			AppId:     cdPipeline.App.Id,
 			EnvId:     env.Id,
 			ClusterId: env.ClusterId,
+			SystemMetadata: &resourceQualifiers.SystemMetadata{
+				EnvironmentName: env.Name,
+				ClusterName:     env.Cluster.ClusterName,
+				Namespace:       env.Namespace,
+				ImageTag:        artifact.Image,
+			},
 		}
 		var variableSnapshot map[string]string
 		if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE {

pkg/resourceQualifiers/bean.go

@@ -4,6 +4,29 @@ type Scope struct {
 	AppId     int `json:"appId"`
 	EnvId     int `json:"envId"`
 	ClusterId int `json:"clusterId"`
+
+	SystemMetadata *SystemMetadata `json:"-"`
+}
+
+type SystemMetadata struct {
+	EnvironmentName string
+	ClusterName     string
+	Namespace       string
+	ImageTag        string
+}
+
+func (metadata *SystemMetadata) GetDataFromSystemVariable(variable SystemVariableName) string {
+	switch variable {
+	case DevtronNamespace:
+		return metadata.Namespace
+	case DevtronClusterName:
+		return metadata.ClusterName
+	case DevtronEnvName:
+		return metadata.EnvironmentName
+	case DevtronImageTag:
+		return metadata.ImageTag
+	}
+	return ""
 }
 
 type Qualifier int

pkg/resourceQualifiers/constants.go

@@ -0,0 +1,12 @@
+package resourceQualifiers
+
+type SystemVariableName string
+
+const (
+	DevtronNamespace   SystemVariableName = "DEVTRON_NAMESPACE"
+	DevtronClusterName SystemVariableName = "DEVTRON_CLUSTER_NAME"
+	DevtronEnvName     SystemVariableName = "DEVTRON_ENV_NAME"
+	DevtronImageTag    SystemVariableName = "DEVTRON_IMAGE_TAG"
+)
+
+var SystemVariables = []SystemVariableName{DevtronNamespace, DevtronClusterName, DevtronEnvName, DevtronImageTag}

pkg/variables/ScopedVariableService.go

@@ -53,6 +53,7 @@ func NewScopedVariableServiceImpl(logger *zap.SugaredLogger, scopedVariableRepos
 type VariableConfig struct {
 	VariableNameRegex    string `env:"SCOPED_VARIABLE_NAME_REGEX" envDefault:"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$"`
 	VariableCacheEnabled bool   `env:"VARIABLE_CACHE_ENABLED" envDefault:"true"`
+	SystemVariablePrefix string `env:"SYSTEM_VAR_PREFIX" envDefault:"DEVTRON_"`
 }
 
 func loadVariableCache(cfg *VariableConfig, service *ScopedVariableServiceImpl) {
@@ -318,12 +319,18 @@ func (impl *ScopedVariableServiceImpl) selectScopeForCompoundQualifier(scopes []
 
 func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifiers.Scope, varNames []string, maskSensitiveData bool) (scopedVariableDataObj []*models.ScopedVariableData, err error) {
 
+	//populating system variables from system metadata
+	if scope.SystemMetadata != nil {
+		systemVariableData := impl.getSystemVariablesData(scope.SystemMetadata, varNames)
+		scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...)
+	}
+
 	// getting all variables from cache
 	allVariableDefinitions := impl.VariableCache.GetData()
 
 	// cache is loaded and no active variables exist. Returns empty
 	if allVariableDefinitions != nil && len(allVariableDefinitions) == 0 {
-		return nil, nil
+		return scopedVariableDataObj, nil
 	}
 
 	// Need to get from repo for isSensitive even if cache is loaded since cache only contains metadata
@@ -332,7 +339,7 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie
 
 		//Cache was not loaded and no active variables found
 		if len(allVariableDefinitions) == 0 {
-			return nil, nil
+			return scopedVariableDataObj, nil
 		}
 	}
 
@@ -354,7 +361,7 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie
 
 	// This to prevent corner case where no variables were found for the provided names
 	if len(varNames) > 0 && len(variableIds) == 0 {
-		return make([]*models.ScopedVariableData, 0), nil
+		return scopedVariableDataObj, nil
 	}
 
 	varScope, err := impl.qualifierMappingService.GetQualifierMappings(resourceQualifiers.Variable, &scope, variableIds)
@@ -424,6 +431,20 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie
 	}
 
 	return scopedVariableDataObj, err
+
+}
+
+func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata, varNames []string) []*models.ScopedVariableData {
+	systemVariables := make([]*models.ScopedVariableData, 0)
+	for _, variable := range resourceQualifiers.SystemVariables {
+		if len(metadata.GetDataFromSystemVariable(variable)) > 0 && slices.Contains(varNames, string(variable)) {
+			systemVariables = append(systemVariables, &models.ScopedVariableData{
+				VariableName:  string(variable),
+				VariableValue: &models.VariableValue{Value: metadata.GetDataFromSystemVariable(variable)},
+			})
+		}
+	}
+	return systemVariables
 }
 
 func (impl *ScopedVariableServiceImpl) GetJsonForVariables() (*models.Payload, error) {

pkg/variables/ScopedVariableValidator.go

@@ -7,6 +7,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/variables/utils"
 	"golang.org/x/exp/slices"
 	"regexp"
+	"strings"
 )
 
 func (impl *ScopedVariableServiceImpl) isValidPayload(payload models.Payload) (error, bool) {
@@ -15,6 +16,11 @@ func (impl *ScopedVariableServiceImpl) isValidPayload(payload models.Payload) (e
 		if slices.Contains(variableNamesList, variable.Definition.VarName) {
 			return models.ValidationError{Err: fmt.Errorf("duplicate variable name %s", variable.Definition.VarName)}, false
 		}
+
+		if strings.HasPrefix(variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix) {
+			return models.ValidationError{Err: fmt.Errorf("%s is not allowed (Prefix %s is reserved for system variables)", variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix)}, false
+		}
+
 		regex := impl.VariableNameConfig.VariableNameRegex
 
 		regexExpression := regexp.MustCompile(regex)