feat: scoped variable CMCS support and manager layer refactorings (#4174)

* app name

* image tag

* refactoring - move to oss

* WIP

* added resolved template in API response

* reverting regex changes

* wip

* cleaned comments

* test commit

* fix

* fix env for specific deployment

* snapshot fixes for deployment

* fix

* sensitive variable masking in all flows

* removed dead comments

* validation for non-string sensitive

* TODO COMMIT

* scoped variable support added for add/update for cm/cs

* support added for trigger

* extra function call removed

* addressed comments

* env variable code commented and issue fixing

* commit minor fix

* used request context

* changes done in get apis

* prod issue fixed

* wire gen

* scopedVariable added in GetLatestCMCSConfig

* fix ctx for manifest flows

* resolved values in GetDeploymentTemplate

* secret issue fixed

* bug fixes

* encoder and decoder added

* fixes for helm generate

* secret data resolution issue fixed

* decoder resolution issue fixed

* issue fixed around GetLatestCMCSConfig

* encoder decoder added in GetLatestCMCSConfig

* commented code removed

* encoder decoder added in GetDeployedHistoryDetailForCMCSByPipelineIdAndWfrId

* encoder decoder added in GetHistoryForDeployedCMCSById

* minor fixes

* code review comments resolved

* some refactoring done around getConfigMapAndSecretJsonV2

* refactoring and testing done around last saved trigger

* comments removed

* manager layer refactoring

* code move

* refactoring cmcs flows

* removing redundant methods

* cleaning up

* todo added

* bug fixed around no variable case and get flows

* bug fixed around specific trigger type

* bug fixed around specific trigger type for encoding decoding

* encode/decode refactor

* ctx passed for isSuperAdmin check

* cleaned dead comments

* VariableSnapshotForCS,ResolvedTemplateDataForCS,VariableSnapshotForCM,ResolvedTemplateDataForCM removed

* code review comment resolved

* deployment-Component/detail issue fixed

* deployment-Component/detail issue fixed for CM

* resolved template issue fixed

* code refactored around component

* cmcs manager

* cleaning comments

* reverting cmcs merge function

* hook for stage trigger

* minor

* DT fix

* cmcs fixes

* granular snapshot for cmcs

* sensitive fix

* cmcs to use string type resolution

* secret parsing fix

* soft parsing for cmcs trigger

---------

Co-authored-by: adi6859 <[email protected]>

Author: subhashish-devtron

Wire.go

@@ -281,6 +281,12 @@ func InitializeApp() (*App, error) {
 		variables.NewVariableSnapshotHistoryServiceImpl,
 		wire.Bind(new(variables.VariableSnapshotHistoryService), new(*variables.VariableSnapshotHistoryServiceImpl)),
 
+		variables.NewScopedVariableManagerImpl,
+		wire.Bind(new(variables.ScopedVariableManager), new(*variables.ScopedVariableManagerImpl)),
+
+		variables.NewScopedVariableCMCSManagerImpl,
+		wire.Bind(new(variables.ScopedVariableCMCSManager), new(*variables.ScopedVariableCMCSManagerImpl)),
+
 		//end
 
 		chart.NewChartServiceImpl,

api/bean/ConfigMapAndSecret.go

@@ -69,3 +69,24 @@ func (configSecretJson ConfigSecretJson) GetDereferencedSecrets() []ConfigSecret
 func (configSecretJson *ConfigSecretJson) SetReferencedSecrets(secrets []ConfigSecretMap) {
 	configSecretJson.Secrets = util.GetReferencedArray(secrets)
 }
+
+func (ConfigSecretRootJson) GetTransformedDataForSecretData(data string, mode util.SecretTransformMode) (string, error) {
+	secretsJson := ConfigSecretRootJson{}
+	err := json.Unmarshal([]byte(data), &secretsJson)
+	if err != nil {
+		return "", err
+	}
+
+	for _, configData := range secretsJson.ConfigSecretJson.Secrets {
+		configData.Data, err = util.GetDecodedAndEncodedData(configData.Data, mode)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	marshal, err := json.Marshal(secretsJson)
+	if err != nil {
+		return "", err
+	}
+	return string(marshal), nil
+}

internal/sql/repository/chartConfig/EnvConfigOverrideRepository.go

@@ -46,6 +46,10 @@ type EnvConfigOverride struct {
 	sql.AuditLog
 	ResolvedEnvOverrideValues string            `sql:"-"`
 	VariableSnapshot          map[string]string `sql:"-"`
+	//ResolvedEnvOverrideValuesForCM string            `sql:"-"`
+	VariableSnapshotForCM map[string]string `sql:"-"`
+	//ResolvedEnvOverrideValuesForCS string            `sql:"-"`
+	VariableSnapshotForCS map[string]string `sql:"-"`
 }
 
 type EnvConfigOverrideRepository interface {

pkg/app/AppService.go

@@ -34,8 +34,8 @@ import (
 	"github.com/devtron-labs/devtron/pkg/k8s"
 	repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
 	repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository"
+	"github.com/devtron-labs/devtron/pkg/resourceQualifiers"
 	"github.com/devtron-labs/devtron/pkg/variables"
-	"github.com/devtron-labs/devtron/pkg/variables/parsers"
 	_ "github.com/devtron-labs/devtron/pkg/variables/repository"
 	"github.com/devtron-labs/devtron/util/argo"
 	"go.opentelemetry.io/otel"
@@ -160,10 +160,7 @@ type AppServiceImpl struct {
 	globalEnvVariables                     *util2.GlobalEnvVariables
 	manifestPushConfigRepository           repository5.ManifestPushConfigRepository
 	GitOpsManifestPushService              GitOpsPushService
-	variableSnapshotHistoryService         variables.VariableSnapshotHistoryService
-	scopedVariableService                  variables.ScopedVariableService
-	variableEntityMappingService           variables.VariableEntityMappingService
-	variableTemplateParser                 parsers.VariableTemplateParser
+	scopedVariableManager                  variables.ScopedVariableCMCSManager
 	argoClientWrapperService               argocdServer.ArgoClientWrapperService
 }
 
@@ -174,7 +171,7 @@ type AppService interface {
 	//TriggerCD(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error
 	GetConfigMapAndSecretJson(appId int, envId int, pipelineId int) ([]byte, error)
 	UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Application, cdWfrId int, updateTimedOutStatus bool) error
-	GetCmSecretNew(appId int, envId int, isJob bool) (*bean.ConfigMapJson, *bean.ConfigSecretJson, error)
+	GetCmSecretNew(appId int, envId int, isJob bool, scope resourceQualifiers.Scope) (*bean.ConfigMapJson, *bean.ConfigSecretJson, error)
 	//MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error
 	UpdateDeploymentStatusForGitOpsPipelines(app *v1alpha1.Application, statusTime time.Time, isAppStore bool) (bool, bool, *chartConfig.PipelineOverride, error)
 	WriteCDSuccessEvent(appId int, envId int, override *chartConfig.PipelineOverride)
@@ -245,11 +242,8 @@ func NewAppService(
 	globalEnvVariables *util2.GlobalEnvVariables, helmAppService client2.HelmAppService,
 	manifestPushConfigRepository repository5.ManifestPushConfigRepository,
 	GitOpsManifestPushService GitOpsPushService,
-	variableSnapshotHistoryService variables.VariableSnapshotHistoryService,
-	scopedVariableService variables.ScopedVariableService,
-	variableEntityMappingService variables.VariableEntityMappingService,
-	variableTemplateParser parsers.VariableTemplateParser,
 	argoClientWrapperService argocdServer.ArgoClientWrapperService,
+	scopedVariableManager variables.ScopedVariableCMCSManager,
 ) *AppServiceImpl {
 	appServiceImpl := &AppServiceImpl{
 		environmentConfigRepository:            environmentConfigRepository,
@@ -311,11 +305,8 @@ func NewAppService(
 		helmAppService:                         helmAppService,
 		manifestPushConfigRepository:           manifestPushConfigRepository,
 		GitOpsManifestPushService:              GitOpsManifestPushService,
-		variableSnapshotHistoryService:         variableSnapshotHistoryService,
-		scopedVariableService:                  scopedVariableService,
-		variableEntityMappingService:           variableEntityMappingService,
-		variableTemplateParser:                 variableTemplateParser,
 		argoClientWrapperService:               argoClientWrapperService,
+		scopedVariableManager:                  scopedVariableManager,
 	}
 	return appServiceImpl
 }
@@ -1122,7 +1113,7 @@ func (impl *AppServiceImpl) autoHealChartLocationInChart(ctx context.Context, en
 }
 
 // FIXME tmp workaround
-func (impl *AppServiceImpl) GetCmSecretNew(appId int, envId int, isJob bool) (*bean.ConfigMapJson, *bean.ConfigSecretJson, error) {
+func (impl *AppServiceImpl) GetCmSecretNew(appId int, envId int, isJob bool, scope resourceQualifiers.Scope) (*bean.ConfigMapJson, *bean.ConfigSecretJson, error) {
 	var configMapJson string
 	var secretDataJson string
 	var configMapJsonApp string
@@ -1187,7 +1178,13 @@ func (impl *AppServiceImpl) GetCmSecretNew(appId int, envId int, isJob bool) (*b
 			return nil, nil, err
 		}
 	}
-	return &configResponse, &secretResponse, nil
+
+	resolvedConfigResponse, resolvedSecretResponse, err := impl.scopedVariableManager.ResolveForPrePostStageTrigger(scope, configResponse, secretResponse, configMapA.Id, configMapE.Id)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return resolvedConfigResponse, resolvedSecretResponse, nil
 }
 
 // depricated

pkg/bean/configSecretData.go

@@ -0,0 +1,96 @@
+package bean
+
+import (
+	"encoding/json"
+	"github.com/devtron-labs/devtron/util"
+)
+
+type ConfigList struct {
+	ConfigData []*ConfigData `json:"maps"`
+}
+
+type SecretList struct {
+	ConfigData []*ConfigData `json:"secrets"`
+}
+
+type ConfigData struct {
+	Name                  string           `json:"name"`
+	Type                  string           `json:"type"`
+	External              bool             `json:"external"`
+	MountPath             string           `json:"mountPath,omitempty"`
+	Data                  json.RawMessage  `json:"data"`
+	DefaultData           json.RawMessage  `json:"defaultData,omitempty"`
+	DefaultMountPath      string           `json:"defaultMountPath,omitempty"`
+	Global                bool             `json:"global"`
+	ExternalSecretType    string           `json:"externalType"`
+	ExternalSecret        []ExternalSecret `json:"secretData"`
+	DefaultExternalSecret []ExternalSecret `json:"defaultSecretData,omitempty"`
+	ESOSecretData         ESOSecretData    `json:"esoSecretData"`
+	DefaultESOSecretData  ESOSecretData    `json:"defaultESOSecretData,omitempty"`
+	RoleARN               string           `json:"roleARN"`
+	SubPath               bool             `json:"subPath"`
+	FilePermission        string           `json:"filePermission"`
+}
+
+type ExternalSecret struct {
+	Key      string `json:"key"`
+	Name     string `json:"name"`
+	Property string `json:"property,omitempty"`
+	IsBinary bool   `json:"isBinary"`
+}
+
+type ESOSecretData struct {
+	SecretStore     json.RawMessage `json:"secretStore,omitempty"`
+	SecretStoreRef  json.RawMessage `json:"secretStoreRef,omitempty"`
+	EsoData         []ESOData       `json:"esoData"`
+	RefreshInterval string          `json:"refreshInterval,omitempty"`
+}
+
+type ESOData struct {
+	SecretKey string `json:"secretKey"`
+	Key       string `json:"key"`
+	Property  string `json:"property,omitempty"`
+}
+
+func (ConfigData) GetTransformedDataForSecretData(data string, mode util.SecretTransformMode) (string, error) {
+	secretDataMap := make(map[string]*ConfigData)
+	err := json.Unmarshal([]byte(data), &secretDataMap)
+	if err != nil {
+		return "", err
+	}
+
+	for _, configData := range secretDataMap {
+		data, err := util.GetDecodedAndEncodedData(configData.Data, mode)
+		if err != nil {
+			return "", err
+		}
+		configData.Data = data
+
+	}
+	resolvedTemplate, err := json.Marshal(secretDataMap)
+	if err != nil {
+		return "", err
+	}
+	return string(resolvedTemplate), nil
+}
+
+func (SecretList) GetTransformedDataForSecret(data string, mode util.SecretTransformMode) (string, error) {
+	secretsList := SecretList{}
+	err := json.Unmarshal([]byte(data), &secretsList)
+	if err != nil {
+		return "", err
+	}
+
+	for _, configData := range secretsList.ConfigData {
+		configData.Data, err = util.GetDecodedAndEncodedData(configData.Data, mode)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	marshal, err := json.Marshal(secretsList)
+	if err != nil {
+		return "", err
+	}
+	return string(marshal), nil
+}

pkg/bulkAction/BulkUpdateService.go

@@ -28,7 +28,6 @@ import (
 	"github.com/devtron-labs/devtron/pkg/pipeline/history"
 	repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
 	"github.com/devtron-labs/devtron/pkg/variables"
-	"github.com/devtron-labs/devtron/pkg/variables/parsers"
 	repository5 "github.com/devtron-labs/devtron/pkg/variables/repository"
 	"github.com/devtron-labs/devtron/util/argo"
 	"github.com/devtron-labs/devtron/util/rbac"
@@ -92,8 +91,7 @@ type BulkUpdateServiceImpl struct {
 	appWorkflowService               appWorkflow2.AppWorkflowService
 	pubsubClient                     *pubsub.PubSubClientServiceImpl
 	argoUserService                  argo.ArgoUserService
-	variableEntityMappingService     variables.VariableEntityMappingService
-	variableTemplateParser           parsers.VariableTemplateParser
+	scopedVariableManager            variables.ScopedVariableManager
 }
 
 func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository,
@@ -124,8 +122,8 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito
 	appWorkflowService appWorkflow2.AppWorkflowService,
 	pubsubClient *pubsub.PubSubClientServiceImpl,
 	argoUserService argo.ArgoUserService,
-	variableEntityMappingService variables.VariableEntityMappingService,
-	variableTemplateParser parsers.VariableTemplateParser) (*BulkUpdateServiceImpl, error) {
+	scopedVariableManager variables.ScopedVariableManager,
+) (*BulkUpdateServiceImpl, error) {
 	impl := &BulkUpdateServiceImpl{
 		bulkUpdateRepository:             bulkUpdateRepository,
 		chartRepository:                  chartRepository,
@@ -159,8 +157,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito
 		appWorkflowService:               appWorkflowService,
 		pubsubClient:                     pubsubClient,
 		argoUserService:                  argoUserService,
-		variableTemplateParser:           variableTemplateParser,
-		variableEntityMappingService:     variableEntityMappingService,
+		scopedVariableManager:            scopedVariableManager,
 	}
 
 	err := impl.SubscribeToCdBulkTriggerTopic()
@@ -465,7 +462,7 @@ func (impl BulkUpdateServiceImpl) BulkUpdateDeploymentTemplate(bulkUpdatePayload
 							}
 							//VARIABLE_MAPPING_UPDATE
 							//NOTE: this flow is doesn't have the user info, therefore updated by is being set to the last updated by
-							err = impl.extractAndMapVariables(chart.GlobalOverride, chart.Id, repository5.EntityTypeDeploymentTemplateAppLevel, chart.UpdatedBy)
+							err = impl.scopedVariableManager.ExtractAndMapVariables(chart.GlobalOverride, chart.Id, repository5.EntityTypeDeploymentTemplateAppLevel, chart.UpdatedBy, nil)
 							if err != nil {
 								return nil
 							}
@@ -538,7 +535,7 @@ func (impl BulkUpdateServiceImpl) BulkUpdateDeploymentTemplate(bulkUpdatePayload
 								impl.logger.Errorw("error in creating entry for env deployment template history", "err", err, "envOverride", chartEnv)
 							}
 							//VARIABLE_MAPPING_UPDATE
-							err = impl.extractAndMapVariables(chartEnv.EnvOverrideValues, chartEnv.Id, repository5.EntityTypeDeploymentTemplateEnvLevel, chartEnv.UpdatedBy)
+							err = impl.scopedVariableManager.ExtractAndMapVariables(chartEnv.EnvOverrideValues, chartEnv.Id, repository5.EntityTypeDeploymentTemplateEnvLevel, chartEnv.UpdatedBy, nil)
 							if err != nil {
 								return nil
 							}
@@ -554,21 +551,6 @@ func (impl BulkUpdateServiceImpl) BulkUpdateDeploymentTemplate(bulkUpdatePayload
 	return deploymentTemplateBulkUpdateResponse
 }
 
-func (impl BulkUpdateServiceImpl) extractAndMapVariables(template string, entityId int, entityType repository5.EntityType, userId int32) error {
-	usedVariables, err := impl.variableTemplateParser.ExtractVariables(template, parsers.JsonVariableTemplate)
-	if err != nil {
-		return err
-	}
-	err = impl.variableEntityMappingService.UpdateVariablesForEntity(usedVariables, repository5.Entity{
-		EntityType: entityType,
-		EntityId:   entityId,
-	}, userId, nil)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func (impl BulkUpdateServiceImpl) BulkUpdateConfigMap(bulkUpdatePayload *BulkUpdatePayload) *CmAndSecretBulkUpdateResponse {
 	configMapBulkUpdateResponse := &CmAndSecretBulkUpdateResponse{}
 	var appNameIncludes []string