From 8526440b6ff7ae9ca4c9d64f475a549f396f02dc Mon Sep 17 00:00:00 2001 From: Subhashish Date: Thu, 5 Oct 2023 19:22:07 +0530 Subject: [PATCH 01/13] adding system variables in get scoped data --- pkg/variables/ScopedVariableService.go | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index a69b948fb0..2b2a23a753 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -423,7 +423,57 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie } } + systemVariableData, err := impl.getSystemVariablesData(scope) + if err != nil { + return nil, err + } + scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...) + return scopedVariableDataObj, err + +} + +func (impl *ScopedVariableServiceImpl) getSystemVariablesData(scope resourceQualifiers.Scope) ([]*models.ScopedVariableData, error) { + systemVariables := make([]*models.ScopedVariableData, 0) + if scope.AppId > 0 { + apps, err := impl.appRepository.FindAppAndProjectByIdsOrderByTeam([]int{scope.AppId}) + if err != nil { + return nil, err + } + application := apps[0] + + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: "DEVTRON_APP_NAME", + VariableValue: &models.VariableValue{Value: application.AppName}, + }) + + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: "DEVTRON_PROJECT_NAME", + VariableValue: &models.VariableValue{Value: application.Team.Name}, + }) + } + + if scope.EnvId > 0 { + environment, err := impl.environmentRepository.FindById(scope.EnvId) + if err != nil { + return nil, err + } + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: "DEVTRON_ENV_NAME", + VariableValue: &models.VariableValue{Value: environment.Name}, + }) + + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: "DEVTRON_NAMESPACE", + VariableValue: &models.VariableValue{Value: environment.Namespace}, + }) + + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: "DEVTRON_CLUSTER_NAME", + VariableValue: &models.VariableValue{Value: environment.Cluster.ClusterName}, + }) + } + return systemVariables, nil } func (impl *ScopedVariableServiceImpl) GetJsonForVariables() (*models.Payload, error) { From 3e08189317de75d38a80fe94e4599eecf727403a Mon Sep 17 00:00:00 2001 From: Subhashish Date: Fri, 6 Oct 2023 11:10:36 +0530 Subject: [PATCH 02/13] system variables --- pkg/app/AppService.go | 29 ++++++++----- pkg/pipeline/CiService.go | 6 +++ pkg/pipeline/WorkflowDagExecutor.go | 5 +++ pkg/resourceQualifiers/bean.go | 9 ++++ pkg/variables/ScopedVariableService.go | 52 +++++++++--------------- pkg/variables/ScopedVariableValidator.go | 6 +++ pkg/variables/models/constants.go | 7 ++++ 7 files changed, 70 insertions(+), 44 deletions(-) create mode 100644 pkg/variables/models/constants.go diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 7397b9336d..d4b49e7db7 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1256,14 +1256,28 @@ func (impl *AppServiceImpl) GetDeploymentStrategyByTriggerType(overrideRequest * func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) { + envOverride := &chartConfig.EnvConfigOverride{} + _, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") + env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) + span.End() + if err != nil { + impl.logger.Errorw("unable to find env", "err", err) + return nil, err + } + envOverride.Environment = env + //VARIABLE different cases for variable resolution scope := resourceQualifiers.Scope{ AppId: overrideRequest.AppId, EnvId: overrideRequest.EnvId, ClusterId: overrideRequest.ClusterId, + SystemMetadata: &resourceQualifiers.SystemMetadata{ + EnvironmentName: env.Name, + ClusterName: env.Cluster.ClusterName, + Namespace: env.Namespace, + }, } - envOverride := &chartConfig.EnvConfigOverride{} - var err error + //var err error if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) @@ -1376,6 +1390,7 @@ func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.Va } if envOverride.IsOverride { + resolvedTemplate, variableMap, err := impl.extractVariablesAndResolveTemplate(scope, envOverride.EnvOverrideValues, repository6.Entity{ EntityType: repository6.EntityTypeDeploymentTemplateEnvLevel, EntityId: envOverride.Id, @@ -1397,15 +1412,7 @@ func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.Va envOverride.VariableSnapshot = variableMap } } - _, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") - env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) - span.End() - if err != nil { - impl.logger.Errorw("unable to find env", "err", err) - return nil, err - } - envOverride.Environment = env - //VARIABLE_RESOLVE + return envOverride, nil } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 383a65d6b8..e387d3f74c 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -166,6 +166,12 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { //This will be populated for jobs running in selected environment scope.EnvId = env.Id scope.ClusterId = env.ClusterId + + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + EnvironmentName: env.Name, + ClusterName: env.Cluster.ClusterName, + Namespace: env.Namespace, + } } if ciWorkflowConfig.Namespace == "" { ciWorkflowConfig.Namespace = impl.config.GetDefaultNamespace() diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 210199a4c4..d4d9847316 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -902,6 +902,11 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor AppId: cdPipeline.App.Id, EnvId: env.Id, ClusterId: env.ClusterId, + SystemMetadata: &resourceQualifiers.SystemMetadata{ + EnvironmentName: env.Name, + ClusterName: env.Cluster.ClusterName, + Namespace: env.Namespace, + }, } var variableSnapshot map[string]string if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { diff --git a/pkg/resourceQualifiers/bean.go b/pkg/resourceQualifiers/bean.go index 39dc408cb9..1f25f758fa 100644 --- a/pkg/resourceQualifiers/bean.go +++ b/pkg/resourceQualifiers/bean.go @@ -4,6 +4,15 @@ type Scope struct { AppId int `json:"appId"` EnvId int `json:"envId"` ClusterId int `json:"clusterId"` + + SystemMetadata *SystemMetadata `json:"-"` + +} + +type SystemMetadata struct { + EnvironmentName string + ClusterName string + Namespace string } type Qualifier int diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index 2b2a23a753..0bd9a8757e 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -53,6 +53,7 @@ func NewScopedVariableServiceImpl(logger *zap.SugaredLogger, scopedVariableRepos type VariableConfig struct { VariableNameRegex string `env:"SCOPED_VARIABLE_NAME_REGEX" envDefault:"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$"` VariableCacheEnabled bool `env:"VARIABLE_CACHE_ENABLED" envDefault:"true"` + SystemVariablePrefix string `env:"SYSTEM_VAR_PREFIX" envDefault:"DEVTRON_"` } func loadVariableCache(cfg *VariableConfig, service *ScopedVariableServiceImpl) { @@ -423,54 +424,39 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie } } - systemVariableData, err := impl.getSystemVariablesData(scope) - if err != nil { - return nil, err + //populating system variables from system metadata + if scope.SystemMetadata != nil { + systemVariableData, err := impl.getSystemVariablesData(scope.SystemMetadata) + if err != nil { + return nil, err + } + scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...) } - scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...) return scopedVariableDataObj, err } -func (impl *ScopedVariableServiceImpl) getSystemVariablesData(scope resourceQualifiers.Scope) ([]*models.ScopedVariableData, error) { +func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata) ([]*models.ScopedVariableData, error) { systemVariables := make([]*models.ScopedVariableData, 0) - if scope.AppId > 0 { - apps, err := impl.appRepository.FindAppAndProjectByIdsOrderByTeam([]int{scope.AppId}) - if err != nil { - return nil, err - } - application := apps[0] - + if len(metadata.Namespace) > 0 { systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: "DEVTRON_APP_NAME", - VariableValue: &models.VariableValue{Value: application.AppName}, - }) - - systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: "DEVTRON_PROJECT_NAME", - VariableValue: &models.VariableValue{Value: application.Team.Name}, + VariableName: models.DevtronNamespace, + VariableValue: &models.VariableValue{Value: metadata.Namespace}, }) } - if scope.EnvId > 0 { - environment, err := impl.environmentRepository.FindById(scope.EnvId) - if err != nil { - return nil, err - } + if len(metadata.ClusterName) > 0 { systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: "DEVTRON_ENV_NAME", - VariableValue: &models.VariableValue{Value: environment.Name}, - }) - - systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: "DEVTRON_NAMESPACE", - VariableValue: &models.VariableValue{Value: environment.Namespace}, + VariableName: models.DevtronClusterName, + VariableValue: &models.VariableValue{Value: metadata.ClusterName}, }) + } + if len(metadata.EnvironmentName) > 0 { systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: "DEVTRON_CLUSTER_NAME", - VariableValue: &models.VariableValue{Value: environment.Cluster.ClusterName}, + VariableName: models.DevtronEnvName, + VariableValue: &models.VariableValue{Value: metadata.EnvironmentName}, }) } return systemVariables, nil diff --git a/pkg/variables/ScopedVariableValidator.go b/pkg/variables/ScopedVariableValidator.go index 9150085a98..84570197ce 100644 --- a/pkg/variables/ScopedVariableValidator.go +++ b/pkg/variables/ScopedVariableValidator.go @@ -7,6 +7,7 @@ import ( "github.com/devtron-labs/devtron/pkg/variables/utils" "golang.org/x/exp/slices" "regexp" + "strings" ) func (impl *ScopedVariableServiceImpl) isValidPayload(payload models.Payload) (error, bool) { @@ -15,6 +16,11 @@ func (impl *ScopedVariableServiceImpl) isValidPayload(payload models.Payload) (e if slices.Contains(variableNamesList, variable.Definition.VarName) { return models.ValidationError{Err: fmt.Errorf("duplicate variable name %s", variable.Definition.VarName)}, false } + + if strings.HasPrefix(variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix) { + return models.ValidationError{Err: fmt.Errorf("%s is not allowed. Prefix %s is reserved for system variables)", variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix)}, false + } + regex := impl.VariableNameConfig.VariableNameRegex regexExpression := regexp.MustCompile(regex) diff --git a/pkg/variables/models/constants.go b/pkg/variables/models/constants.go new file mode 100644 index 0000000000..b0836e60e3 --- /dev/null +++ b/pkg/variables/models/constants.go @@ -0,0 +1,7 @@ +package models + +const ( + DevtronNamespace = "DEVTRON_NAMESPACE" + DevtronClusterName = "DEVTRON_CLUSTER_NAME" + DevtronEnvName = "DEVTRON_ENV_NAME" +) From c160267e139798fc352c99e46c90d7828a523a33 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Fri, 6 Oct 2023 12:48:03 +0530 Subject: [PATCH 03/13] bug fixes --- pkg/app/AppService.go | 44 +++++++++++++----------- pkg/variables/ScopedVariableService.go | 32 ++++++++--------- pkg/variables/ScopedVariableValidator.go | 2 +- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index d4b49e7db7..d05364a368 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1257,27 +1257,8 @@ func (impl *AppServiceImpl) GetDeploymentStrategyByTriggerType(overrideRequest * func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) { envOverride := &chartConfig.EnvConfigOverride{} - _, span := otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") - env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) - span.End() - if err != nil { - impl.logger.Errorw("unable to find env", "err", err) - return nil, err - } - envOverride.Environment = env - - //VARIABLE different cases for variable resolution - scope := resourceQualifiers.Scope{ - AppId: overrideRequest.AppId, - EnvId: overrideRequest.EnvId, - ClusterId: overrideRequest.ClusterId, - SystemMetadata: &resourceQualifiers.SystemMetadata{ - EnvironmentName: env.Name, - ClusterName: env.Cluster.ClusterName, - Namespace: env.Namespace, - }, - } - //var err error + + var err error if overrideRequest.DeploymentWithConfig == bean.DEPLOYMENT_CONFIG_TYPE_SPECIFIC_TRIGGER { _, span := otel.Tracer("orchestrator").Start(ctx, "deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId") deploymentTemplateHistory, err := impl.deploymentTemplateHistoryRepository.GetHistoryByPipelineIdAndWfrId(overrideRequest.PipelineId, overrideRequest.WfrIdForDeploymentWithSpecificTrigger) @@ -1389,6 +1370,27 @@ func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.Va envOverride.Chart = chart } + _, span = otel.Tracer("orchestrator").Start(ctx, "envRepository.FindById") + env, err := impl.envRepository.FindById(envOverride.TargetEnvironment) + span.End() + if err != nil { + impl.logger.Errorw("unable to find env", "err", err) + return nil, err + } + envOverride.Environment = env + + //VARIABLE different cases for variable resolution + scope := resourceQualifiers.Scope{ + AppId: overrideRequest.AppId, + EnvId: overrideRequest.EnvId, + ClusterId: overrideRequest.ClusterId, + SystemMetadata: &resourceQualifiers.SystemMetadata{ + EnvironmentName: env.Name, + ClusterName: env.Cluster.ClusterName, + Namespace: env.Namespace, + }, + } + if envOverride.IsOverride { resolvedTemplate, variableMap, err := impl.extractVariablesAndResolveTemplate(scope, envOverride.EnvOverrideValues, repository6.Entity{ diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index 0bd9a8757e..efc7404b9f 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -319,12 +319,21 @@ func (impl *ScopedVariableServiceImpl) selectScopeForCompoundQualifier(scopes [] func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifiers.Scope, varNames []string, maskSensitiveData bool) (scopedVariableDataObj []*models.ScopedVariableData, err error) { + //populating system variables from system metadata + if scope.SystemMetadata != nil { + systemVariableData, err := impl.getSystemVariablesData(scope.SystemMetadata, varNames) + if err != nil { + return nil, err + } + scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...) + } + // getting all variables from cache allVariableDefinitions := impl.VariableCache.GetData() // cache is loaded and no active variables exist. Returns empty if allVariableDefinitions != nil && len(allVariableDefinitions) == 0 { - return nil, nil + return scopedVariableDataObj, nil } // Need to get from repo for isSensitive even if cache is loaded since cache only contains metadata @@ -333,7 +342,7 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie //Cache was not loaded and no active variables found if len(allVariableDefinitions) == 0 { - return nil, nil + return scopedVariableDataObj, nil } } @@ -355,7 +364,7 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie // This to prevent corner case where no variables were found for the provided names if len(varNames) > 0 && len(variableIds) == 0 { - return make([]*models.ScopedVariableData, 0), nil + return scopedVariableDataObj, nil } varScope, err := impl.qualifierMappingService.GetQualifierMappings(resourceQualifiers.Variable, &scope, variableIds) @@ -424,36 +433,27 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie } } - //populating system variables from system metadata - if scope.SystemMetadata != nil { - systemVariableData, err := impl.getSystemVariablesData(scope.SystemMetadata) - if err != nil { - return nil, err - } - scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...) - } - return scopedVariableDataObj, err } -func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata) ([]*models.ScopedVariableData, error) { +func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata, varNames []string) ([]*models.ScopedVariableData, error) { systemVariables := make([]*models.ScopedVariableData, 0) - if len(metadata.Namespace) > 0 { + if len(metadata.Namespace) > 0 && slices.Contains(varNames, models.DevtronNamespace) { systemVariables = append(systemVariables, &models.ScopedVariableData{ VariableName: models.DevtronNamespace, VariableValue: &models.VariableValue{Value: metadata.Namespace}, }) } - if len(metadata.ClusterName) > 0 { + if len(metadata.ClusterName) > 0 && slices.Contains(varNames, models.DevtronClusterName) { systemVariables = append(systemVariables, &models.ScopedVariableData{ VariableName: models.DevtronClusterName, VariableValue: &models.VariableValue{Value: metadata.ClusterName}, }) } - if len(metadata.EnvironmentName) > 0 { + if len(metadata.EnvironmentName) > 0 && slices.Contains(varNames, models.DevtronEnvName) { systemVariables = append(systemVariables, &models.ScopedVariableData{ VariableName: models.DevtronEnvName, VariableValue: &models.VariableValue{Value: metadata.EnvironmentName}, diff --git a/pkg/variables/ScopedVariableValidator.go b/pkg/variables/ScopedVariableValidator.go index 84570197ce..cffe5983ab 100644 --- a/pkg/variables/ScopedVariableValidator.go +++ b/pkg/variables/ScopedVariableValidator.go @@ -18,7 +18,7 @@ func (impl *ScopedVariableServiceImpl) isValidPayload(payload models.Payload) (e } if strings.HasPrefix(variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix) { - return models.ValidationError{Err: fmt.Errorf("%s is not allowed. Prefix %s is reserved for system variables)", variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix)}, false + return models.ValidationError{Err: fmt.Errorf("%s is not allowed (Prefix %s is reserved for system variables)", variable.Definition.VarName, impl.VariableNameConfig.SystemVariablePrefix)}, false } regex := impl.VariableNameConfig.VariableNameRegex From a949b2c057e19b8493ebf1a891a32527fa74f32e Mon Sep 17 00:00:00 2001 From: Subhashish Date: Fri, 6 Oct 2023 12:57:09 +0530 Subject: [PATCH 04/13] minor fix --- pkg/variables/ScopedVariableService.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index efc7404b9f..64b4546865 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -321,10 +321,7 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie //populating system variables from system metadata if scope.SystemMetadata != nil { - systemVariableData, err := impl.getSystemVariablesData(scope.SystemMetadata, varNames) - if err != nil { - return nil, err - } + systemVariableData := impl.getSystemVariablesData(scope.SystemMetadata, varNames) scopedVariableDataObj = append(scopedVariableDataObj, systemVariableData...) } @@ -437,7 +434,7 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie } -func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata, varNames []string) ([]*models.ScopedVariableData, error) { +func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata, varNames []string) []*models.ScopedVariableData { systemVariables := make([]*models.ScopedVariableData, 0) if len(metadata.Namespace) > 0 && slices.Contains(varNames, models.DevtronNamespace) { systemVariables = append(systemVariables, &models.ScopedVariableData{ @@ -459,7 +456,7 @@ func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resource VariableValue: &models.VariableValue{Value: metadata.EnvironmentName}, }) } - return systemVariables, nil + return systemVariables } func (impl *ScopedVariableServiceImpl) GetJsonForVariables() (*models.Payload, error) { From 774cc7ccf437693e55b4eeac5d7721e4b5aa8676 Mon Sep 17 00:00:00 2001 From: Kripansh Date: Fri, 6 Oct 2023 17:19:07 +0530 Subject: [PATCH 05/13] image tag variable exposed --- api/bean/ValuesOverrideRequest.go | 1 + pkg/app/AppService.go | 15 +++++++++------ pkg/pipeline/WorkflowDagExecutor.go | 1 + pkg/resourceQualifiers/bean.go | 2 +- pkg/variables/ScopedVariableService.go | 6 ++++++ pkg/variables/models/constants.go | 1 + 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/api/bean/ValuesOverrideRequest.go b/api/bean/ValuesOverrideRequest.go index efa712109d..5dc7b24682 100644 --- a/api/bean/ValuesOverrideRequest.go +++ b/api/bean/ValuesOverrideRequest.go @@ -72,6 +72,7 @@ type ValuesOverrideRequest struct { AppName string `json:"-"` PipelineName string `json:"-"` DeploymentAppType string `json:"-"` + ImageTag string `json:"-"` } type BulkCdDeployEvent struct { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index d05364a368..1b0e84fbda 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1388,6 +1388,7 @@ func (impl *AppServiceImpl) GetEnvOverrideByTriggerType(overrideRequest *bean.Va EnvironmentName: env.Name, ClusterName: env.Cluster.ClusterName, Namespace: env.Namespace, + ImageTag: overrideRequest.ImageTag, }, } @@ -1499,6 +1500,14 @@ func (impl *AppServiceImpl) GetValuesOverrideForTrigger(overrideRequest *bean.Va return valuesOverrideResponse, err } + _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") + artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) + span.End() + if err != nil { + return valuesOverrideResponse, err + } + overrideRequest.ImageTag = artifact.Image + envOverride, err := impl.GetEnvOverrideByTriggerType(overrideRequest, triggeredAt, ctx) if err != nil { impl.logger.Errorw("error in getting env override by trigger type", "err", err) @@ -1514,12 +1523,6 @@ func (impl *AppServiceImpl) GetValuesOverrideForTrigger(overrideRequest *bean.Va impl.logger.Errorw("error in getting strategy by trigger type", "err", err) return valuesOverrideResponse, err } - _, span := otel.Tracer("orchestrator").Start(ctx, "ciArtifactRepository.Get") - artifact, err := impl.ciArtifactRepository.Get(overrideRequest.CiArtifactId) - span.End() - if err != nil { - return valuesOverrideResponse, err - } _, span = otel.Tracer("orchestrator").Start(ctx, "getDbMigrationOverride") //FIXME: how to determine rollback //we can't depend on ciArtifact ID because CI pipeline can be manually triggered in any order regardless of sourcecode status diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index d4d9847316..41b8f59104 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -906,6 +906,7 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor EnvironmentName: env.Name, ClusterName: env.Cluster.ClusterName, Namespace: env.Namespace, + ImageTag: artifact.Image, }, } var variableSnapshot map[string]string diff --git a/pkg/resourceQualifiers/bean.go b/pkg/resourceQualifiers/bean.go index 1f25f758fa..6dff8ab39b 100644 --- a/pkg/resourceQualifiers/bean.go +++ b/pkg/resourceQualifiers/bean.go @@ -6,13 +6,13 @@ type Scope struct { ClusterId int `json:"clusterId"` SystemMetadata *SystemMetadata `json:"-"` - } type SystemMetadata struct { EnvironmentName string ClusterName string Namespace string + ImageTag string } type Qualifier int diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index 64b4546865..b46a109aa2 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -456,6 +456,12 @@ func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resource VariableValue: &models.VariableValue{Value: metadata.EnvironmentName}, }) } + if len(metadata.ImageTag) > 0 && slices.Contains(varNames, models.DevtronImageTag) { + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: models.DevtronImageTag, + VariableValue: &models.VariableValue{Value: metadata.ImageTag}, + }) + } return systemVariables } diff --git a/pkg/variables/models/constants.go b/pkg/variables/models/constants.go index b0836e60e3..df3af27ba0 100644 --- a/pkg/variables/models/constants.go +++ b/pkg/variables/models/constants.go @@ -4,4 +4,5 @@ const ( DevtronNamespace = "DEVTRON_NAMESPACE" DevtronClusterName = "DEVTRON_CLUSTER_NAME" DevtronEnvName = "DEVTRON_ENV_NAME" + DevtronImageTag = "DEVTRON_IMAGE_TAG" ) From d24678cf796906b1f7dfb50b44ad6c765ba276d5 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Fri, 6 Oct 2023 17:35:29 +0530 Subject: [PATCH 06/13] refactoring --- pkg/app/AppService.go | 2 +- pkg/pipeline/DeploymentConfigService.go | 2 +- pkg/pipeline/PipelineStageService.go | 2 +- pkg/variables/models/variable-payload.go | 17 ++- .../parsers/VariableTemplateParser.go | 105 ++++++++++++++++-- pkg/variables/parsers/bean.go | 5 +- wire_gen.go | 5 +- 7 files changed, 121 insertions(+), 17 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index d05364a368..8748e97b32 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1477,7 +1477,7 @@ func (impl *AppServiceImpl) extractVariablesAndResolveTemplate(scope resourceQua } for _, variable := range scopedVariables { - variableMap[variable.VariableName] = variable.VariableValue.StringValue() + variableMap[variable.VariableName] = variable.VariableValue.StringValue(false) } resolvedTemplate := parserResponse.ResolvedTemplate diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index 7f5ec89832..dc9f8b8b61 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -112,7 +112,7 @@ func (impl *DeploymentConfigServiceImpl) extractVariablesAndGetScopedVariables(s } for _, variable := range scopedVariables { - variableMap[variable.VariableName] = variable.VariableValue.StringValue() + variableMap[variable.VariableName] = variable.VariableValue.StringValue(false) } return variableMap, nil } diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index aaa8db1502..8edd13af0a 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -2130,7 +2130,7 @@ func (impl *PipelineStageServiceImpl) fetchScopedVariablesAndResolveTemplate(unr variableSnapshot := make(map[string]string) for _, variable := range scopedVariables { - variableSnapshot[variable.VariableName] = variable.VariableValue.StringValue() + variableSnapshot[variable.VariableName] = variable.VariableValue.StringValue(false) } responseJson, err := json.Marshal(unresolvedResponse) diff --git a/pkg/variables/models/variable-payload.go b/pkg/variables/models/variable-payload.go index 3ee5e522b6..71c2c4b627 100644 --- a/pkg/variables/models/variable-payload.go +++ b/pkg/variables/models/variable-payload.go @@ -63,7 +63,7 @@ type VariableValue struct { Value interface{} `json:"value" validate:"required"` } -func (value VariableValue) StringValue() string { +func (value VariableValue) StringValue(escapeString bool) string { switch reflect.TypeOf(value.Value).Kind() { case reflect.Int: return strconv.Itoa(value.Value.(int)) @@ -72,5 +72,20 @@ func (value VariableValue) StringValue() string { case reflect.Bool: return strconv.FormatBool(value.Value.(bool)) } + if escapeString { + return "\"" + value.Value.(string) + "\"" + } return value.Value.(string) } + +//func (value VariableValue) StringValueWithStringsEscaped() string { +// switch reflect.TypeOf(value.Value).Kind() { +// case reflect.Int: +// return strconv.Itoa(value.Value.(int)) +// case reflect.Float64: +// return strconv.FormatFloat(value.Value.(float64), 'f', -1, 64) +// case reflect.Bool: +// return strconv.FormatBool(value.Value.(bool)) +// } +// return "\"" + value.Value.(string) + "\"" +//} diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index 36316ca92b..9bd825f6db 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/caarlos0/env" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl/hclsyntax" _ "github.com/hashicorp/hcl2/hcl/hclsyntax" @@ -21,18 +22,102 @@ type VariableTemplateParser interface { ExtractVariables(template string) ([]string, error) //ParseTemplate(template string, values map[string]string) string ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse + //ParseTemplateWithPrimitiveHandling(parserRequest VariableParserRequest) VariableParserResponse } type VariableTemplateParserImpl struct { - logger *zap.SugaredLogger + logger *zap.SugaredLogger + VariableTemplateParserConfig *VariableTemplateParserConfig } -func NewVariableTemplateParserImpl(logger *zap.SugaredLogger) *VariableTemplateParserImpl { - return &VariableTemplateParserImpl{logger: logger} +func NewVariableTemplateParserImpl(logger *zap.SugaredLogger) (*VariableTemplateParserImpl, error) { + impl := &VariableTemplateParserImpl{logger: logger} + cfg, err := GetVariableTemplateParserConfig() + if err != nil { + return nil, err + } + impl.VariableTemplateParserConfig = cfg + return impl, nil +} + +type VariableTemplateParserConfig struct { + ScopedVariableEnabled bool `env:"SCOPED_VARIABLE_ENABLED" envDefault:"false"` + ScopedVariableHandlePrimitives bool `env:"SCOPED_VARIABLE_HANDLE_PRIMITIVES" envDefault:"false"` +} + +func GetVariableTemplateParserConfig() (*VariableTemplateParserConfig, error) { + cfg := &VariableTemplateParserConfig{} + err := env.Parse(cfg) + return cfg, err +} + +func preProcessPlaceholder(template string) string { + pattern := `\"@{{([^}]+)}}\"` + //@\{\{[a-zA-Z0-9-+/*%_\s]+\}\} + + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(template, -1) + + // Replace the surrounding quotes + for _, match := range matches { + if len(match) == 2 { + originalMatch := match[0] + innerContent := match[1] + replacement := fmt.Sprintf("@{{%s}}", innerContent) + template = strings.Replace(template, originalMatch, replacement, 1) + } + } + return template +} + +func postProcessPlaceholder(template string) string { + pattern := `@{{([^}]+)}}` + + re := regexp.MustCompile(pattern) + matches := re.FindAllStringSubmatch(template, -1) + + // Replace the surrounding quotes + for _, match := range matches { + if len(match) == 2 { + originalMatch := match[0] + innerContent := match[1] + replacement := "\"" + innerContent + "\"" + template = strings.Replace(template, originalMatch, replacement, 1) + } + } + return template +} + +func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse { + + if impl.VariableTemplateParserConfig.ScopedVariableHandlePrimitives && parserRequest.TemplateType == JsonVariableTemplate { + template := preProcessPlaceholder(parserRequest.Template) + request := VariableParserRequest{ + TemplateType: StringVariableTemplate, + Template: template, + Variables: parserRequest.Variables, + IgnoreUnknownVariables: parserRequest.IgnoreUnknownVariables, + handlePrimitives: true, + } + response := impl.parseTemplate(request) + // for variables which were not resolved and were ignored + if response.Error == nil { + response.ResolvedTemplate = postProcessPlaceholder(response.ResolvedTemplate) + } + return response + } else { + return impl.parseTemplate(parserRequest) + } } func (impl *VariableTemplateParserImpl) ExtractVariables(template string) ([]string, error) { + var variables []string + + if !impl.VariableTemplateParserConfig.ScopedVariableEnabled { + return variables, nil + } + // preprocess existing template to comment template, err := impl.convertToHclCompatible(JsonVariableTemplate, template) if err != nil { @@ -58,10 +143,10 @@ func (impl *VariableTemplateParserImpl) extractVarNames(hclVariables []hcl.Trave return variables } -func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse { +func (impl *VariableTemplateParserImpl) parseTemplate(parserRequest VariableParserRequest) VariableParserResponse { template := parserRequest.Template response := VariableParserResponse{Request: parserRequest, ResolvedTemplate: template} - values := parserRequest.GetValuesMap() + values := parserRequest.GetValuesMap(parserRequest.handlePrimitives) templateType := parserRequest.TemplateType template, err := impl.convertToHclCompatible(templateType, template) if err != nil { @@ -102,7 +187,7 @@ func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariablePars func (impl *VariableTemplateParserImpl) checkForDefaultedVariables(parserRequest VariableParserRequest, variables []hcl.Traversal, template string, response *VariableParserResponse) (hclsyntax.Expression, string, bool) { var hclExpression hclsyntax.Expression var diagnostics hcl.Diagnostics - valuesMap := parserRequest.GetValuesMap() + valuesMap := parserRequest.GetValuesMap(parserRequest.handlePrimitives) defaultedVars := impl.getDefaultedVariables(variables, valuesMap) ignoreDefaultedVariables := parserRequest.IgnoreUnknownVariables if len(defaultedVars) > 0 { @@ -213,12 +298,12 @@ func (impl *VariableTemplateParserImpl) convertToHclCompatible(templateType Vari } template = fmt.Sprintf(`{"root":%s}`, template) } - template = impl.diluteExistingHclVars(template, "$") - template = impl.diluteExistingHclVars(template, "%") + template = impl.diluteExistingHclVars(template, "\\$", "$") + template = impl.diluteExistingHclVars(template, "%", "%") return impl.convertToHclExpression(template), nil } -func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, templateControlKeyword string) string { +func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, templateControlKeyword string, replaceKeyword string) string { hclVarRegex := regexp.MustCompile(templateControlKeyword + `\{`) indexesData := hclVarRegex.FindAllIndex([]byte(template), -1) var strBuilder strings.Builder @@ -227,7 +312,7 @@ func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, t for _, datum := range indexesData { startIndex := datum[0] endIndex := datum[1] - strBuilder.WriteString(template[currentIndex:startIndex] + templateControlKeyword + template[startIndex:endIndex]) + strBuilder.WriteString(template[currentIndex:startIndex] + replaceKeyword + template[startIndex:endIndex]) currentIndex = endIndex } if currentIndex <= len(template) { diff --git a/pkg/variables/parsers/bean.go b/pkg/variables/parsers/bean.go index c883449286..16fefd7b09 100644 --- a/pkg/variables/parsers/bean.go +++ b/pkg/variables/parsers/bean.go @@ -22,6 +22,7 @@ type VariableParserRequest struct { Template string Variables []*models.ScopedVariableData IgnoreUnknownVariables bool + handlePrimitives bool } type VariableParserResponse struct { @@ -31,11 +32,11 @@ type VariableParserResponse struct { DetailedError string } -func (request VariableParserRequest) GetValuesMap() map[string]string { +func (request VariableParserRequest) GetValuesMap(handlePrimitive bool) map[string]string { variablesMap := make(map[string]string) variables := request.Variables for _, variable := range variables { - variablesMap[variable.VariableName] = variable.VariableValue.StringValue() + variablesMap[variable.VariableName] = variable.VariableValue.StringValue(handlePrimitive) } return variablesMap } diff --git a/wire_gen.go b/wire_gen.go index 5cd01209d1..1a46b745e0 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -339,7 +339,10 @@ func InitializeApp() (*App, error) { repositoryServiceClientImpl := repository8.NewServiceClientImpl(sugaredLogger, argoCDConnectionManagerImpl) variableEntityMappingRepositoryImpl := repository7.NewVariableEntityMappingRepository(sugaredLogger, db) variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) - variableTemplateParserImpl := parsers.NewVariableTemplateParserImpl(sugaredLogger) + variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) + if err != nil { + return nil, err + } scopedVariableRepositoryImpl := repository7.NewScopedVariableRepository(db, sugaredLogger) qualifiersMappingRepositoryImpl, err := resourceQualifiers.NewQualifiersMappingRepositoryImpl(db, sugaredLogger) if err != nil { From 683dd8017265a22ea0257f1dbcab3d4edd186780 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Fri, 6 Oct 2023 18:39:08 +0530 Subject: [PATCH 07/13] refactoring --- pkg/resourceQualifiers/bean.go | 14 +++++++++++ pkg/resourceQualifiers/constants.go | 12 ++++++++++ pkg/variables/ScopedVariableService.go | 32 ++++++-------------------- pkg/variables/models/constants.go | 8 ------- 4 files changed, 33 insertions(+), 33 deletions(-) create mode 100644 pkg/resourceQualifiers/constants.go delete mode 100644 pkg/variables/models/constants.go diff --git a/pkg/resourceQualifiers/bean.go b/pkg/resourceQualifiers/bean.go index 6dff8ab39b..e65826787f 100644 --- a/pkg/resourceQualifiers/bean.go +++ b/pkg/resourceQualifiers/bean.go @@ -15,6 +15,20 @@ type SystemMetadata struct { ImageTag string } +func (metadata *SystemMetadata) GetDataFromSystemVariable(variable SystemVariableName) string { + switch variable { + case DevtronNamespace: + return metadata.Namespace + case DevtronClusterName: + return metadata.ClusterName + case DevtronEnvName: + return metadata.EnvironmentName + case DevtronImageTag: + return metadata.ImageTag + } + return "" +} + type Qualifier int const ( diff --git a/pkg/resourceQualifiers/constants.go b/pkg/resourceQualifiers/constants.go new file mode 100644 index 0000000000..900c9e2101 --- /dev/null +++ b/pkg/resourceQualifiers/constants.go @@ -0,0 +1,12 @@ +package resourceQualifiers + +type SystemVariableName string + +const ( + DevtronNamespace SystemVariableName = "DEVTRON_NAMESPACE" + DevtronClusterName SystemVariableName = "DEVTRON_CLUSTER_NAME" + DevtronEnvName SystemVariableName = "DEVTRON_ENV_NAME" + DevtronImageTag SystemVariableName = "DEVTRON_IMAGE_TAG" +) + +var SystemVariables = []SystemVariableName{DevtronNamespace, DevtronClusterName, DevtronEnvName, DevtronImageTag} diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index b46a109aa2..7022e06136 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -436,31 +436,13 @@ func (impl *ScopedVariableServiceImpl) GetScopedVariables(scope resourceQualifie func (impl *ScopedVariableServiceImpl) getSystemVariablesData(metadata *resourceQualifiers.SystemMetadata, varNames []string) []*models.ScopedVariableData { systemVariables := make([]*models.ScopedVariableData, 0) - if len(metadata.Namespace) > 0 && slices.Contains(varNames, models.DevtronNamespace) { - systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: models.DevtronNamespace, - VariableValue: &models.VariableValue{Value: metadata.Namespace}, - }) - } - - if len(metadata.ClusterName) > 0 && slices.Contains(varNames, models.DevtronClusterName) { - systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: models.DevtronClusterName, - VariableValue: &models.VariableValue{Value: metadata.ClusterName}, - }) - } - - if len(metadata.EnvironmentName) > 0 && slices.Contains(varNames, models.DevtronEnvName) { - systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: models.DevtronEnvName, - VariableValue: &models.VariableValue{Value: metadata.EnvironmentName}, - }) - } - if len(metadata.ImageTag) > 0 && slices.Contains(varNames, models.DevtronImageTag) { - systemVariables = append(systemVariables, &models.ScopedVariableData{ - VariableName: models.DevtronImageTag, - VariableValue: &models.VariableValue{Value: metadata.ImageTag}, - }) + for _, variable := range resourceQualifiers.SystemVariables { + if len(metadata.GetDataFromSystemVariable(variable)) > 0 && slices.Contains(varNames, string(variable)) { + systemVariables = append(systemVariables, &models.ScopedVariableData{ + VariableName: string(variable), + VariableValue: &models.VariableValue{Value: metadata.GetDataFromSystemVariable(variable)}, + }) + } } return systemVariables } diff --git a/pkg/variables/models/constants.go b/pkg/variables/models/constants.go deleted file mode 100644 index df3af27ba0..0000000000 --- a/pkg/variables/models/constants.go +++ /dev/null @@ -1,8 +0,0 @@ -package models - -const ( - DevtronNamespace = "DEVTRON_NAMESPACE" - DevtronClusterName = "DEVTRON_CLUSTER_NAME" - DevtronEnvName = "DEVTRON_ENV_NAME" - DevtronImageTag = "DEVTRON_IMAGE_TAG" -) From f5c717704c32714c26881f331ad06098a4270e22 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Fri, 6 Oct 2023 16:38:24 +0530 Subject: [PATCH 08/13] fix escaping $ --- pkg/variables/parsers/VariableTemplateParser.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index 36316ca92b..7729aecaa8 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -213,12 +213,12 @@ func (impl *VariableTemplateParserImpl) convertToHclCompatible(templateType Vari } template = fmt.Sprintf(`{"root":%s}`, template) } - template = impl.diluteExistingHclVars(template, "$") - template = impl.diluteExistingHclVars(template, "%") + template = impl.diluteExistingHclVars(template, "\\$", "$") + template = impl.diluteExistingHclVars(template, "%", "%") return impl.convertToHclExpression(template), nil } -func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, templateControlKeyword string) string { +func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, templateControlKeyword string, replaceKeyword string) string { hclVarRegex := regexp.MustCompile(templateControlKeyword + `\{`) indexesData := hclVarRegex.FindAllIndex([]byte(template), -1) var strBuilder strings.Builder @@ -227,7 +227,7 @@ func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, t for _, datum := range indexesData { startIndex := datum[0] endIndex := datum[1] - strBuilder.WriteString(template[currentIndex:startIndex] + templateControlKeyword + template[startIndex:endIndex]) + strBuilder.WriteString(template[currentIndex:startIndex] + replaceKeyword + template[startIndex:endIndex]) currentIndex = endIndex } if currentIndex <= len(template) { From fab12d77bf045d3bfa97f1f9c456351c557091bb Mon Sep 17 00:00:00 2001 From: Subhashish Date: Sun, 8 Oct 2023 13:22:20 +0530 Subject: [PATCH 09/13] wf resolver to type string --- pkg/pipeline/PipelineStageService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 8edd13af0a..ca30a34255 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -2139,7 +2139,7 @@ func (impl *PipelineStageServiceImpl) fetchScopedVariablesAndResolveTemplate(unr return nil, err } parserResponse := impl.variableTemplateParser.ParseTemplate(parsers.VariableParserRequest{ - TemplateType: parsers.JsonVariableTemplate, + TemplateType: parsers.StringVariableTemplate, Template: string(responseJson), Variables: scopedVariables, IgnoreUnknownVariables: true, From 35fe12d8902f2418280a4942cc0448f742de4546 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Sun, 8 Oct 2023 15:18:02 +0530 Subject: [PATCH 10/13] fixing logic --- pkg/pipeline/PipelineStageService.go | 2 +- pkg/variables/models/variable-payload.go | 6 ++-- .../parsers/VariableTemplateParser.go | 36 ++++++++++++++----- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index ca30a34255..8edd13af0a 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -2139,7 +2139,7 @@ func (impl *PipelineStageServiceImpl) fetchScopedVariablesAndResolveTemplate(unr return nil, err } parserResponse := impl.variableTemplateParser.ParseTemplate(parsers.VariableParserRequest{ - TemplateType: parsers.StringVariableTemplate, + TemplateType: parsers.JsonVariableTemplate, Template: string(responseJson), Variables: scopedVariables, IgnoreUnknownVariables: true, diff --git a/pkg/variables/models/variable-payload.go b/pkg/variables/models/variable-payload.go index 71c2c4b627..27eaaddde7 100644 --- a/pkg/variables/models/variable-payload.go +++ b/pkg/variables/models/variable-payload.go @@ -72,9 +72,9 @@ func (value VariableValue) StringValue(escapeString bool) string { case reflect.Bool: return strconv.FormatBool(value.Value.(bool)) } - if escapeString { - return "\"" + value.Value.(string) + "\"" - } + //if escapeString { + // return "\"" + value.Value.(string) + "\"" + //} return value.Value.(string) } diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index 9bd825f6db..b1d5a299a4 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -13,6 +13,7 @@ import ( "github.com/zclconf/go-cty/cty/function/stdlib" ctyJson "github.com/zclconf/go-cty/cty/json" "go.uber.org/zap" + "reflect" "regexp" "strconv" "strings" @@ -51,7 +52,18 @@ func GetVariableTemplateParserConfig() (*VariableTemplateParserConfig, error) { return cfg, err } -func preProcessPlaceholder(template string) string { +func isPrimitiveType(value interface{}) bool { + val := reflect.ValueOf(value) + kind := val.Kind() + + return kind == reflect.Int || kind == reflect.Int8 || kind == reflect.Int16 || + kind == reflect.Int32 || kind == reflect.Int64 || kind == reflect.Uint || + kind == reflect.Uint8 || kind == reflect.Uint16 || kind == reflect.Uint32 || + kind == reflect.Uint64 || kind == reflect.Float32 || kind == reflect.Float64 || + kind == reflect.Bool +} + +func preProcessPlaceholder(template string, variableValueMap map[string]interface{}) string { pattern := `\"@{{([^}]+)}}\"` //@\{\{[a-zA-Z0-9-+/*%_\s]+\}\} @@ -63,8 +75,10 @@ func preProcessPlaceholder(template string) string { if len(match) == 2 { originalMatch := match[0] innerContent := match[1] - replacement := fmt.Sprintf("@{{%s}}", innerContent) - template = strings.Replace(template, originalMatch, replacement, 1) + if val, ok := variableValueMap[innerContent]; ok && isPrimitiveType(val) { + replacement := fmt.Sprintf("@{{%s}}", innerContent) + template = strings.Replace(template, originalMatch, replacement, 1) + } } } return template @@ -91,7 +105,12 @@ func postProcessPlaceholder(template string) string { func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse { if impl.VariableTemplateParserConfig.ScopedVariableHandlePrimitives && parserRequest.TemplateType == JsonVariableTemplate { - template := preProcessPlaceholder(parserRequest.Template) + + var variableToValue = make(map[string]interface{}, 0) + for _, variable := range parserRequest.Variables { + variableToValue[variable.VariableName] = variable.VariableValue.Value + } + template := preProcessPlaceholder(parserRequest.Template, variableToValue) request := VariableParserRequest{ TemplateType: StringVariableTemplate, Template: template, @@ -100,10 +119,11 @@ func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariablePars handlePrimitives: true, } response := impl.parseTemplate(request) - // for variables which were not resolved and were ignored - if response.Error == nil { - response.ResolvedTemplate = postProcessPlaceholder(response.ResolvedTemplate) - } + //// for variables which were not resolved and were ignored + //if response.Error == nil { + // finalTemplate := postProcessPlaceholder(response.ResolvedTemplate) + // response.ResolvedTemplate = finalTemplate + //} return response } else { return impl.parseTemplate(parserRequest) From 961c9aca53777b093ad183fca3f497a05142d74d Mon Sep 17 00:00:00 2001 From: Subhashish Date: Sun, 8 Oct 2023 16:01:40 +0530 Subject: [PATCH 11/13] addressed comments and cleaned up code --- pkg/app/AppService.go | 2 +- pkg/pipeline/DeploymentConfigService.go | 2 +- pkg/pipeline/PipelineStageService.go | 2 +- pkg/variables/models/variable-payload.go | 5 +- .../parsers/VariableTemplateParser.go | 71 ++++++------------- pkg/variables/parsers/bean.go | 5 +- pkg/variables/utils/type-utils.go | 12 ++++ 7 files changed, 38 insertions(+), 61 deletions(-) diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index c9911b7459..1b0e84fbda 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -1478,7 +1478,7 @@ func (impl *AppServiceImpl) extractVariablesAndResolveTemplate(scope resourceQua } for _, variable := range scopedVariables { - variableMap[variable.VariableName] = variable.VariableValue.StringValue(false) + variableMap[variable.VariableName] = variable.VariableValue.StringValue() } resolvedTemplate := parserResponse.ResolvedTemplate diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index dc9f8b8b61..7f5ec89832 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -112,7 +112,7 @@ func (impl *DeploymentConfigServiceImpl) extractVariablesAndGetScopedVariables(s } for _, variable := range scopedVariables { - variableMap[variable.VariableName] = variable.VariableValue.StringValue(false) + variableMap[variable.VariableName] = variable.VariableValue.StringValue() } return variableMap, nil } diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 8edd13af0a..aaa8db1502 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -2130,7 +2130,7 @@ func (impl *PipelineStageServiceImpl) fetchScopedVariablesAndResolveTemplate(unr variableSnapshot := make(map[string]string) for _, variable := range scopedVariables { - variableSnapshot[variable.VariableName] = variable.VariableValue.StringValue(false) + variableSnapshot[variable.VariableName] = variable.VariableValue.StringValue() } responseJson, err := json.Marshal(unresolvedResponse) diff --git a/pkg/variables/models/variable-payload.go b/pkg/variables/models/variable-payload.go index 27eaaddde7..998312c134 100644 --- a/pkg/variables/models/variable-payload.go +++ b/pkg/variables/models/variable-payload.go @@ -63,7 +63,7 @@ type VariableValue struct { Value interface{} `json:"value" validate:"required"` } -func (value VariableValue) StringValue(escapeString bool) string { +func (value VariableValue) StringValue() string { switch reflect.TypeOf(value.Value).Kind() { case reflect.Int: return strconv.Itoa(value.Value.(int)) @@ -72,9 +72,6 @@ func (value VariableValue) StringValue(escapeString bool) string { case reflect.Bool: return strconv.FormatBool(value.Value.(bool)) } - //if escapeString { - // return "\"" + value.Value.(string) + "\"" - //} return value.Value.(string) } diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index b1d5a299a4..e8db224296 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/caarlos0/env" + "github.com/devtron-labs/devtron/pkg/variables/utils" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/hcl2/hcl/hclsyntax" _ "github.com/hashicorp/hcl2/hcl/hclsyntax" @@ -13,7 +14,6 @@ import ( "github.com/zclconf/go-cty/cty/function/stdlib" ctyJson "github.com/zclconf/go-cty/cty/json" "go.uber.org/zap" - "reflect" "regexp" "strconv" "strings" @@ -28,54 +28,47 @@ type VariableTemplateParser interface { type VariableTemplateParserImpl struct { logger *zap.SugaredLogger - VariableTemplateParserConfig *VariableTemplateParserConfig + variableTemplateParserConfig *VariableTemplateParserConfig } func NewVariableTemplateParserImpl(logger *zap.SugaredLogger) (*VariableTemplateParserImpl, error) { impl := &VariableTemplateParserImpl{logger: logger} - cfg, err := GetVariableTemplateParserConfig() + cfg, err := getVariableTemplateParserConfig() if err != nil { return nil, err } - impl.VariableTemplateParserConfig = cfg + impl.variableTemplateParserConfig = cfg return impl, nil } +const VariableRegex = `@\{\{[a-zA-Z0-9-+/*%_\s]+\}\}` +const VariableSubRegexWithQuotes = `\"@{{([a-zA-Z0-9-+/*%_\s]+)}}\"` + type VariableTemplateParserConfig struct { ScopedVariableEnabled bool `env:"SCOPED_VARIABLE_ENABLED" envDefault:"false"` ScopedVariableHandlePrimitives bool `env:"SCOPED_VARIABLE_HANDLE_PRIMITIVES" envDefault:"false"` } -func GetVariableTemplateParserConfig() (*VariableTemplateParserConfig, error) { +func getVariableTemplateParserConfig() (*VariableTemplateParserConfig, error) { cfg := &VariableTemplateParserConfig{} err := env.Parse(cfg) return cfg, err } -func isPrimitiveType(value interface{}) bool { - val := reflect.ValueOf(value) - kind := val.Kind() - - return kind == reflect.Int || kind == reflect.Int8 || kind == reflect.Int16 || - kind == reflect.Int32 || kind == reflect.Int64 || kind == reflect.Uint || - kind == reflect.Uint8 || kind == reflect.Uint16 || kind == reflect.Uint32 || - kind == reflect.Uint64 || kind == reflect.Float32 || kind == reflect.Float64 || - kind == reflect.Bool -} - func preProcessPlaceholder(template string, variableValueMap map[string]interface{}) string { - pattern := `\"@{{([^}]+)}}\"` + //pattern := `\"@{{([^}]+)}}\"` //@\{\{[a-zA-Z0-9-+/*%_\s]+\}\} - re := regexp.MustCompile(pattern) + re := regexp.MustCompile(VariableSubRegexWithQuotes) matches := re.FindAllStringSubmatch(template, -1) - // Replace the surrounding quotes + // Replace the surrounding quotes for variables whose value is known + // and type is primitive for _, match := range matches { if len(match) == 2 { originalMatch := match[0] innerContent := match[1] - if val, ok := variableValueMap[innerContent]; ok && isPrimitiveType(val) { + if val, ok := variableValueMap[innerContent]; ok && utils.IsPrimitiveType(val) { replacement := fmt.Sprintf("@{{%s}}", innerContent) template = strings.Replace(template, originalMatch, replacement, 1) } @@ -84,27 +77,9 @@ func preProcessPlaceholder(template string, variableValueMap map[string]interfac return template } -func postProcessPlaceholder(template string) string { - pattern := `@{{([^}]+)}}` - - re := regexp.MustCompile(pattern) - matches := re.FindAllStringSubmatch(template, -1) - - // Replace the surrounding quotes - for _, match := range matches { - if len(match) == 2 { - originalMatch := match[0] - innerContent := match[1] - replacement := "\"" + innerContent + "\"" - template = strings.Replace(template, originalMatch, replacement, 1) - } - } - return template -} - func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse { - if impl.VariableTemplateParserConfig.ScopedVariableHandlePrimitives && parserRequest.TemplateType == JsonVariableTemplate { + if impl.variableTemplateParserConfig.ScopedVariableHandlePrimitives && parserRequest.TemplateType == JsonVariableTemplate { var variableToValue = make(map[string]interface{}, 0) for _, variable := range parserRequest.Variables { @@ -116,15 +91,8 @@ func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariablePars Template: template, Variables: parserRequest.Variables, IgnoreUnknownVariables: parserRequest.IgnoreUnknownVariables, - handlePrimitives: true, } - response := impl.parseTemplate(request) - //// for variables which were not resolved and were ignored - //if response.Error == nil { - // finalTemplate := postProcessPlaceholder(response.ResolvedTemplate) - // response.ResolvedTemplate = finalTemplate - //} - return response + return impl.parseTemplate(request) } else { return impl.parseTemplate(parserRequest) } @@ -134,7 +102,7 @@ func (impl *VariableTemplateParserImpl) ExtractVariables(template string) ([]str var variables []string - if !impl.VariableTemplateParserConfig.ScopedVariableEnabled { + if !impl.variableTemplateParserConfig.ScopedVariableEnabled { return variables, nil } @@ -166,7 +134,7 @@ func (impl *VariableTemplateParserImpl) extractVarNames(hclVariables []hcl.Trave func (impl *VariableTemplateParserImpl) parseTemplate(parserRequest VariableParserRequest) VariableParserResponse { template := parserRequest.Template response := VariableParserResponse{Request: parserRequest, ResolvedTemplate: template} - values := parserRequest.GetValuesMap(parserRequest.handlePrimitives) + values := parserRequest.GetValuesMap() templateType := parserRequest.TemplateType template, err := impl.convertToHclCompatible(templateType, template) if err != nil { @@ -207,7 +175,7 @@ func (impl *VariableTemplateParserImpl) parseTemplate(parserRequest VariablePars func (impl *VariableTemplateParserImpl) checkForDefaultedVariables(parserRequest VariableParserRequest, variables []hcl.Traversal, template string, response *VariableParserResponse) (hclsyntax.Expression, string, bool) { var hclExpression hclsyntax.Expression var diagnostics hcl.Diagnostics - valuesMap := parserRequest.GetValuesMap(parserRequest.handlePrimitives) + valuesMap := parserRequest.GetValuesMap() defaultedVars := impl.getDefaultedVariables(variables, valuesMap) ignoreDefaultedVariables := parserRequest.IgnoreUnknownVariables if len(defaultedVars) > 0 { @@ -343,7 +311,8 @@ func (impl *VariableTemplateParserImpl) diluteExistingHclVars(template string, t } func (impl *VariableTemplateParserImpl) convertToHclExpression(template string) string { - var devtronRegexCompiledPattern = regexp.MustCompile(`@\{\{[a-zA-Z0-9-+/*%_\s]+\}\}`) //TODO KB: add support of Braces () also + + var devtronRegexCompiledPattern = regexp.MustCompile(VariableRegex) //TODO KB: add support of Braces () also indexesData := devtronRegexCompiledPattern.FindAllIndex([]byte(template), -1) var strBuilder strings.Builder strBuilder.Grow(len(template)) diff --git a/pkg/variables/parsers/bean.go b/pkg/variables/parsers/bean.go index 16fefd7b09..c883449286 100644 --- a/pkg/variables/parsers/bean.go +++ b/pkg/variables/parsers/bean.go @@ -22,7 +22,6 @@ type VariableParserRequest struct { Template string Variables []*models.ScopedVariableData IgnoreUnknownVariables bool - handlePrimitives bool } type VariableParserResponse struct { @@ -32,11 +31,11 @@ type VariableParserResponse struct { DetailedError string } -func (request VariableParserRequest) GetValuesMap(handlePrimitive bool) map[string]string { +func (request VariableParserRequest) GetValuesMap() map[string]string { variablesMap := make(map[string]string) variables := request.Variables for _, variable := range variables { - variablesMap[variable.VariableName] = variable.VariableValue.StringValue(handlePrimitive) + variablesMap[variable.VariableName] = variable.VariableValue.StringValue() } return variablesMap } diff --git a/pkg/variables/utils/type-utils.go b/pkg/variables/utils/type-utils.go index 7a51b03d68..ecdf5b718b 100644 --- a/pkg/variables/utils/type-utils.go +++ b/pkg/variables/utils/type-utils.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/ghodss/yaml" + "reflect" "strconv" "strings" ) @@ -53,3 +54,14 @@ func IsValidJSON(input string) bool { } return true } + +func IsPrimitiveType(value interface{}) bool { + val := reflect.ValueOf(value) + kind := val.Kind() + + return kind == reflect.Int || kind == reflect.Int8 || kind == reflect.Int16 || + kind == reflect.Int32 || kind == reflect.Int64 || kind == reflect.Uint || + kind == reflect.Uint8 || kind == reflect.Uint16 || kind == reflect.Uint32 || + kind == reflect.Uint64 || kind == reflect.Float32 || kind == reflect.Float64 || + kind == reflect.Bool +} From 45b490f62fc7ceae2ed25f921703f6d325c538c0 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Sun, 8 Oct 2023 16:07:56 +0530 Subject: [PATCH 12/13] cleaning up --- pkg/variables/models/variable-payload.go | 12 ------------ pkg/variables/parsers/VariableTemplateParser.go | 3 --- 2 files changed, 15 deletions(-) diff --git a/pkg/variables/models/variable-payload.go b/pkg/variables/models/variable-payload.go index 998312c134..3ee5e522b6 100644 --- a/pkg/variables/models/variable-payload.go +++ b/pkg/variables/models/variable-payload.go @@ -74,15 +74,3 @@ func (value VariableValue) StringValue() string { } return value.Value.(string) } - -//func (value VariableValue) StringValueWithStringsEscaped() string { -// switch reflect.TypeOf(value.Value).Kind() { -// case reflect.Int: -// return strconv.Itoa(value.Value.(int)) -// case reflect.Float64: -// return strconv.FormatFloat(value.Value.(float64), 'f', -1, 64) -// case reflect.Bool: -// return strconv.FormatBool(value.Value.(bool)) -// } -// return "\"" + value.Value.(string) + "\"" -//} diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index e8db224296..e0ca677dd5 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -23,7 +23,6 @@ type VariableTemplateParser interface { ExtractVariables(template string) ([]string, error) //ParseTemplate(template string, values map[string]string) string ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse - //ParseTemplateWithPrimitiveHandling(parserRequest VariableParserRequest) VariableParserResponse } type VariableTemplateParserImpl struct { @@ -56,8 +55,6 @@ func getVariableTemplateParserConfig() (*VariableTemplateParserConfig, error) { } func preProcessPlaceholder(template string, variableValueMap map[string]interface{}) string { - //pattern := `\"@{{([^}]+)}}\"` - //@\{\{[a-zA-Z0-9-+/*%_\s]+\}\} re := regexp.MustCompile(VariableSubRegexWithQuotes) matches := re.FindAllStringSubmatch(template, -1) From 6981c4d2de0814a12e17d1df7c16778abcdd7901 Mon Sep 17 00:00:00 2001 From: Subhashish Date: Sun, 8 Oct 2023 17:52:03 +0530 Subject: [PATCH 13/13] parser to return if scoped variables disabled --- pkg/variables/parsers/VariableTemplateParser.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index e0ca677dd5..20da46a45d 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -76,6 +76,13 @@ func preProcessPlaceholder(template string, variableValueMap map[string]interfac func (impl *VariableTemplateParserImpl) ParseTemplate(parserRequest VariableParserRequest) VariableParserResponse { + if !impl.variableTemplateParserConfig.ScopedVariableEnabled { + return VariableParserResponse{ + Request: parserRequest, + ResolvedTemplate: parserRequest.Template, + } + } + if impl.variableTemplateParserConfig.ScopedVariableHandlePrimitives && parserRequest.TemplateType == JsonVariableTemplate { var variableToValue = make(map[string]interface{}, 0)