Merge pull request #6666 from devtron-labs/feat-ext-flux-linking

feat: ext flux linking

Author: iamayushm

api/fluxApplication/FluxApplicationRestHandler.go

@@ -45,6 +45,8 @@ func (handler *FluxApplicationRestHandlerImpl) ListFluxApplications(w http.Respo
 	var clusterIds []int
 	var err error
 
+	noStream := v.Get("noStream") == "true"
+
 	//handling when the clusterIds string is empty ,it will not support the
 	if len(clusterIdString) == 0 {
 		handler.logger.Errorw("error in getting cluster ids", "error", err, "clusterIds", clusterIds)
@@ -57,7 +59,7 @@ func (handler *FluxApplicationRestHandlerImpl) ListFluxApplications(w http.Respo
 		return
 	}
 	handler.logger.Debugw("extracted ClusterIds successfully ", "clusterIds", clusterIds)
-	handler.fluxApplicationService.ListFluxApplications(r.Context(), clusterIds, w)
+	handler.fluxApplicationService.ListFluxApplications(r.Context(), clusterIds, noStream, w)
 }
 
 func (handler *FluxApplicationRestHandlerImpl) GetApplicationDetail(w http.ResponseWriter, r *http.Request) {

api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go

@@ -2472,6 +2472,11 @@ func (handler *PipelineConfigRestHandlerImpl) ValidateExternalAppLinkRequest(w h
 		common.WriteJsonResp(w, err, response, http.StatusOK)
 		return
 		// handle helm deployment types
+	} else if request.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_FLUX {
+		response := handler.pipelineBuilder.ValidateLinkFluxAppRequest(ctx, &request)
+		common.WriteJsonResp(w, err, response, http.StatusOK)
+		return
+		// handle helm deployment types
 	}
 	common.WriteJsonResp(w, errors.New("invalid deployment app type in request"), nil, http.StatusBadRequest)
 	return

client/fluxcd/fluxDeploymentService.go

@@ -135,11 +135,11 @@ func (impl *DeploymentServiceImpl) upsertGitRepoSecret(ctx context.Context, flux
 		"providerId": gitOpsConfig.Provider,
 	}
 
-	namespace := fluxCdSpec.Namespace
+	namespace := fluxCdSpec.GitRepositoryNamespace
 	secret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      fluxCdSpec.GitOpsSecretName,
-			Namespace: fluxCdSpec.Namespace,
+			Namespace: fluxCdSpec.GitRepositoryNamespace,
 			Labels:    labels,
 		},
 		Type: v1.SecretTypeBasicAuth,
@@ -223,7 +223,7 @@ func (impl *DeploymentServiceImpl) updateFluxCdApp(ctx context.Context, fluxCdSp
 }
 
 func (impl *DeploymentServiceImpl) CreateGitRepository(ctx context.Context, fluxCdSpec bean.FluxCDSpec, secretName string, apiClient client.Client) (*sourcev1.GitRepository, error) {
-	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.Namespace
+	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.GitRepositoryNamespace
 	gitRepo := &sourcev1.GitRepository{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -250,7 +250,7 @@ func (impl *DeploymentServiceImpl) CreateGitRepository(ctx context.Context, flux
 
 func (impl *DeploymentServiceImpl) UpdateGitRepository(ctx context.Context, fluxCdSpec bean.FluxCDSpec,
 	secretName string, apiClient client.Client) (*sourcev1.GitRepository, error) {
-	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.Namespace
+	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.GitRepositoryNamespace
 	key := types.NamespacedName{Name: name, Namespace: namespace}
 	existing := &sourcev1.GitRepository{}
 
@@ -274,7 +274,7 @@ func (impl *DeploymentServiceImpl) UpdateGitRepository(ctx context.Context, flux
 
 func (impl *DeploymentServiceImpl) CreateHelmRelease(ctx context.Context, fluxCdSpec bean.FluxCDSpec,
 	apiClient client.Client) (*helmv2.HelmRelease, error) {
-	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.Namespace
+	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.HelmReleaseNamespace
 	helmRelease := &helmv2.HelmRelease{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -313,7 +313,7 @@ func (impl *DeploymentServiceImpl) CreateHelmRelease(ctx context.Context, fluxCd
 
 func (impl *DeploymentServiceImpl) UpdateHelmRelease(ctx context.Context, fluxCdSpec bean.FluxCDSpec,
 	apiClient client.Client) (*helmv2.HelmRelease, error) {
-	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.Namespace
+	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.HelmReleaseNamespace
 	key := types.NamespacedName{Name: name, Namespace: namespace}
 	existing := &helmv2.HelmRelease{}
 
@@ -350,7 +350,6 @@ func (impl *DeploymentServiceImpl) UpdateHelmRelease(ctx context.Context, fluxCd
 func (impl *DeploymentServiceImpl) DeleteFluxDeploymentApp(ctx context.Context, request *DeploymentAppDeleteRequest) error {
 	fluxCdSpec := request.DeploymentConfig.ReleaseConfiguration.FluxCDSpec
 	clusterId := fluxCdSpec.ClusterId
-	namespace := fluxCdSpec.Namespace
 	restConfig, err := impl.K8sUtil.GetRestConfigByCluster(request.ClusterConfig)
 	if err != nil {
 		impl.logger.Errorw("error in getting rest config", "clusterId", clusterId, "err", err)
@@ -361,7 +360,7 @@ func (impl *DeploymentServiceImpl) DeleteFluxDeploymentApp(ctx context.Context,
 		impl.logger.Errorw("error in creating k8s client", "clusterId", clusterId, "err", err)
 		return err
 	}
-	name, namespace := fluxCdSpec.GitRepositoryName, fluxCdSpec.Namespace
+	name, namespace := fluxCdSpec.HelmReleaseName, fluxCdSpec.HelmReleaseNamespace
 	key := types.NamespacedName{Name: name, Namespace: namespace}
 
 	existingHelmRelease := &helmv2.HelmRelease{}
@@ -376,6 +375,7 @@ func (impl *DeploymentServiceImpl) DeleteFluxDeploymentApp(ctx context.Context,
 		return err
 	}
 
+	key = types.NamespacedName{Name: fluxCdSpec.GitRepositoryName, Namespace: fluxCdSpec.GitRepositoryNamespace}
 	existingGitRepository := &sourcev1.GitRepository{}
 	err = apiClient.Get(ctx, key, existingGitRepository)
 	if err != nil {

cmd/external-app/wire_gen.go

@@ -13,6 +13,7 @@
  | DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT | int |6 | Context timeout for no gitops concurrent async deployments |  | false |
  | EXPOSE_CD_METRICS | bool |false |  |  | false |
  | FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE | bool |false | enable migration of external argocd application to devtron pipeline |  | false |
+ | FEATURE_MIGRATE_FLUX_APPLICATION_ENABLE | bool |false | enable flux application services |  | false |
  | FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | eligible time for checking flux app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle. |  | false |
  | HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle. |  | false |
  | IS_INTERNAL_USE | bool |true | If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops. |  | false |
@@ -25,6 +26,7 @@
  | RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS | bool |false |  |  | false |
  | SHOULD_CHECK_NAMESPACE_ON_CLONE | bool |false | should we check if namespace exists or not while cloning app |  | false |
  | USE_DEPLOYMENT_CONFIG_DATA | bool |false | use deployment config data from deployment_config table |  | true |
+ | VALIDATE_EXT_APP_CHART_TYPE | bool |false | validate external flux app chart |  | false |
 
 
 ## CI_RUNNER Related Environment Variables

env_gen.json

@@ -283,7 +283,8 @@ func (chart *InstallAppVersionDTO) GetFluxDeploymentConfig() *bean2.DeploymentCo
 			Version: bean2.Version,
 			FluxCDSpec: bean2.FluxCDSpec{
 				ClusterId:              chart.ClusterId,
-				Namespace:              chart.Namespace,
+				HelmReleaseNamespace:   chart.Namespace,
+				GitRepositoryNamespace: chart.Namespace,
 				GitRepositoryName:      util.BuildDeployedAppName(chart.AppName, chart.EnvironmentName),
 				HelmReleaseName:        util.BuildDeployedAppName(chart.AppName, chart.EnvironmentName),
 				GitOpsSecretName:       fmt.Sprintf("devtron-flux-secret-%d", chart.GitOpsId),

env_gen.md

@@ -28,7 +28,7 @@ import (
 type InstalledAppDBExtendedService interface {
 	EAMode.InstalledAppDBService
 	UpdateInstalledAppVersionStatus(application *v1alpha1.Application) (bool, error)
-	IsGitOpsRepoAlreadyRegistered(repoUrl string) (bool, error)
+	IsGitOpsRepoAlreadyRegistered(repoUrl string, appId int) (bool, error)
 }
 
 type InstalledAppDBExtendedServiceImpl struct {
@@ -91,9 +91,9 @@ func (impl *InstalledAppDBExtendedServiceImpl) UpdateInstalledAppVersionStatus(a
 	return true, nil
 }
 
-func (impl *InstalledAppDBExtendedServiceImpl) IsGitOpsRepoAlreadyRegistered(repoUrl string) (bool, error) {
+func (impl *InstalledAppDBExtendedServiceImpl) IsGitOpsRepoAlreadyRegistered(repoUrl string, appId int) (bool, error) {
 
-	urlPresent, err := impl.InstalledAppDBServiceImpl.DeploymentConfigService.CheckIfURLAlreadyPresent(repoUrl)
+	urlPresent, err := impl.InstalledAppDBServiceImpl.DeploymentConfigService.CheckIfURLAlreadyPresent(repoUrl, appId)
 	if err != nil && !util.IsErrNoRows(err) {
 		impl.Logger.Errorw("error in checking url in deployment configs", "repoUrl", repoUrl, "err", err)
 		return false, err
@@ -111,6 +111,9 @@ func (impl *InstalledAppDBExtendedServiceImpl) IsGitOpsRepoAlreadyRegistered(rep
 	if util.IsErrNoRows(err) {
 		return false, nil
 	}
+	if installedAppModel != nil && installedAppModel.AppId == appId {
+		return false, nil
+	}
 	impl.Logger.Warnw("repository is already in use for helm app", "repoUrl", repoUrl, "appId", installedAppModel.AppId)
 	return true, nil
 }

pkg/appStore/bean/bean.go

@@ -72,7 +72,7 @@ type ChartService interface {
 	ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) (*bean2.DeploymentConfig, error)
 
 	IsGitOpsRepoConfiguredForDevtronApp(appId int) (bool, error)
-	IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error)
+	IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string, appId int) (bool, error)
 
 	GetDeploymentTemplateDataByAppIdAndCharRefId(appId, chartRefId int) (map[string]interface{}, error)
 
@@ -1047,9 +1047,9 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c
 //	return nil
 //}
 
-func (impl *ChartServiceImpl) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error) {
+func (impl *ChartServiceImpl) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string, appId int) (bool, error) {
 
-	isURLPresent, err := impl.deploymentConfigService.CheckIfURLAlreadyPresent(gitOpsRepoUrl)
+	isURLPresent, err := impl.deploymentConfigService.CheckIfURLAlreadyPresent(gitOpsRepoUrl, appId)
 	if err != nil {
 		impl.logger.Errorw("error in checking if gitOps repo url is already present", "error", err)
 		return false, err
@@ -1065,6 +1065,9 @@ func (impl *ChartServiceImpl) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string
 	} else if util.IsErrNoRows(err) {
 		return false, nil
 	}
+	if chartModel != nil && chartModel.AppId == appId {
+		return false, nil
+	}
 	impl.logger.Errorw("repository is already in use for devtron app", "repoUrl", gitOpsRepoUrl, "appId", chartModel.AppId)
 	return true, nil
 }

pkg/appStore/installedApp/service/FullMode/InstalledAppDBExtendedService.go

@@ -46,6 +46,10 @@ func (impl *CommonBaseServiceImpl) isGitOpsEnable() (*FeatureGitOpsVariables, er
 	featureGitOpsFlags := &FeatureGitOpsVariables{
 		IsFeatureArgoCdMigrationEnabled: impl.globalEnvVariables.DeploymentServiceTypeConfig.IsFeatureMigrateArgoCdApplicationEnable(),
 	}
+	if impl.globalEnvVariables.DeploymentServiceTypeConfig.FeatureMigrateFluxApplicationEnable {
+		featureGitOpsFlags.IsFeatureGitOpsEnabled = true
+		featureGitOpsFlags.IsFeatureUserDefinedGitOpsEnabled = true
+	}
 	argoModule, err := impl.moduleReadService.GetModuleInfoByName(bean.ModuleNameArgoCd)
 	if err != nil && !errors.Is(err, moduleErr.ModuleNotFoundError) {
 		impl.logger.Errorw("error in getting argo module", "error", err)