chore: Refactoring deployment template GitOps (#4615)

* removed registerInArgo multiple impls

* extracted app metrics code

* migrated envLevel app metrics code to new service

* chore: Removed unused jira and migration  integration (#4498)

* removed unsued jira integration

* removed test-suite-code

* db migration conf removal

* chore: removed unused injection

* chore: removed dead code

* added: migration script

---------

Co-authored-by: Ash-exp <[email protected]>

* chore: App store dead code cleanup and restructuring  (#4497)

* moved chart-group in seperate code

* removed unused dependency

* removed dead code

* extracted resource tree

* moved notes

* resource movement

* removed unused code

* removed unused dependency

* commit methods

* extracted status update

* chore: clean up unused dead code

* updated: EA mode docker file

* updated: migration number

---------

Co-authored-by: Ash-exp <[email protected]>

* chart ref refactoring

* removed infra metrics db calls

* moved app metrics repositories from /internal to /pkg

* moved: const and types to bean

* removed: unused const

* review comments

* migrated some methods from chartService to chartRefService

* added dt validation service interface

* minor refactoring

* moved validation method - 1

* wip

* removed redundant appMetrics req obj

* moved app metrics bindings  to wireset

* removed multiple dead code

* remove redundant dependency

* moved ChartGroup router and rest handler to respective folder

* gitOps refactoring

* moved gitClient code to a common wrapper service

* review changes

* wip

* fix for unsupported charts

* minor cleanup

* renamed remote package to git

* renamed gitOpsRemoteOpService If and impl

* migrated usages of gitService to gitOperationService

* shifted git service and all gitOps clients to pkg

* gitops repository usages refactor

* refactored gitOpsRepository usages

* gitlab client creation refactoring

* renamed util/ChartService

* reverted renaming changes

* reverted renaming changes

* reverted renaming changes

* wip

* wip

* removed typo

* changes

* changes

---------

Co-authored-by: Nishant <[email protected]>
Co-authored-by: Ash-exp <[email protected]>

Author: 3 people

Wire.go

@@ -87,7 +87,6 @@ import (
 	"github.com/devtron-labs/devtron/pkg/appClone"
 	"github.com/devtron-labs/devtron/pkg/appClone/batch"
 	"github.com/devtron-labs/devtron/pkg/appStatus"
-	appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean"
 	"github.com/devtron-labs/devtron/pkg/appStore/chartGroup"
 	repository4 "github.com/devtron-labs/devtron/pkg/appStore/chartGroup/repository"
 	appStoreDeploymentFullMode "github.com/devtron-labs/devtron/pkg/appStore/deployment/fullMode"
@@ -100,6 +99,8 @@ import (
 	chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository"
 	"github.com/devtron-labs/devtron/pkg/commonService"
 	delete2 "github.com/devtron-labs/devtron/pkg/delete"
+	deployment2 "github.com/devtron-labs/devtron/pkg/deployment"
+	git2 "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git"
 	"github.com/devtron-labs/devtron/pkg/deploymentGroup"
 	"github.com/devtron-labs/devtron/pkg/devtronResource"
 	repository9 "github.com/devtron-labs/devtron/pkg/devtronResource/repository"
@@ -161,6 +162,8 @@ func InitializeApp() (*App, error) {
 		apiToken.ApiTokenWireSet,
 		webhookHelm.WebhookHelmWireSet,
 		terminal.TerminalWireSet,
+		deployment2.DeploymentWireSet,
+
 		// -------wireset end ----------
 		//-------
 		gitSensor.GetConfig,
@@ -174,10 +177,6 @@ func InitializeApp() (*App, error) {
 		//sql.NewDbConnection,
 		//app.GetACDAuthConfig,
 		util3.GetACDAuthConfig,
-		wire.Value(chartRepoRepository.RefChartDir("scripts/devtron-reference-helm-charts")),
-		wire.Value(appStoreBean.RefChartProxyDir("scripts/devtron-reference-helm-charts")),
-		wire.Value(chart.DefaultChart("reference-app-rolling")),
-		wire.Value(util.ChartWorkingDir("/tmp/charts/")),
 		connection.SettingsManager,
 		//auth.GetConfig,
 
@@ -265,8 +264,6 @@ func InitializeApp() (*App, error) {
 
 		util.NewChartTemplateServiceImpl,
 		wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)),
-		util.NewChartDeploymentServiceImpl,
-		wire.Bind(new(util.ChartDeploymentService), new(*util.ChartDeploymentServiceImpl)),
 
 		//scoped variables start
 		variables.NewScopedVariableServiceImpl,
@@ -335,8 +332,6 @@ func InitializeApp() (*App, error) {
 		eClient.NewEventRESTClientImpl,
 		wire.Bind(new(eClient.EventClient), new(*eClient.EventRESTClientImpl)),
 
-		util3.NewTokenCache,
-
 		eClient.NewEventSimpleFactoryImpl,
 		wire.Bind(new(eClient.EventFactory), new(*eClient.EventSimpleFactoryImpl)),
 
@@ -353,7 +348,7 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(pipelineConfig.CiPipelineRepository), new(*pipelineConfig.CiPipelineRepositoryImpl)),
 		pipelineConfig.NewCiPipelineMaterialRepositoryImpl,
 		wire.Bind(new(pipelineConfig.CiPipelineMaterialRepository), new(*pipelineConfig.CiPipelineMaterialRepositoryImpl)),
-		util.NewGitFactory,
+		git2.NewGitFactory,
 
 		application.NewApplicationClientImpl,
 		wire.Bind(new(application.ServiceClient), new(*application.ServiceClientImpl)),
@@ -527,11 +522,6 @@ func InitializeApp() (*App, error) {
 
 		restHandler.NewExternalCiRestHandlerImpl,
 		wire.Bind(new(restHandler.ExternalCiRestHandler), new(*restHandler.ExternalCiRestHandlerImpl)),
-		repository.NewAppLevelMetricsRepositoryImpl,
-		wire.Bind(new(repository.AppLevelMetricsRepository), new(*repository.AppLevelMetricsRepositoryImpl)),
-
-		repository.NewEnvLevelAppMetricsRepositoryImpl,
-		wire.Bind(new(repository.EnvLevelAppMetricsRepository), new(*repository.EnvLevelAppMetricsRepositoryImpl)),
 
 		grafana.GetGrafanaClientConfig,
 		grafana.NewGrafanaClientImpl,
@@ -649,8 +639,6 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(restHandler.GitOpsConfigRestHandler), new(*restHandler.GitOpsConfigRestHandlerImpl)),
 		gitops.NewGitOpsConfigServiceImpl,
 		wire.Bind(new(gitops.GitOpsConfigService), new(*gitops.GitOpsConfigServiceImpl)),
-		repository.NewGitOpsConfigRepositoryImpl,
-		wire.Bind(new(repository.GitOpsConfigRepository), new(*repository.GitOpsConfigRepositoryImpl)),
 
 		router.NewAttributesRouterImpl,
 		wire.Bind(new(router.AttributesRouter), new(*router.AttributesRouterImpl)),
@@ -671,7 +659,7 @@ func InitializeApp() (*App, error) {
 		scopedVariable.NewScopedVariableRestHandlerImpl,
 		wire.Bind(new(scopedVariable.ScopedVariableRestHandler), new(*scopedVariable.ScopedVariableRestHandlerImpl)),
 
-		util.NewGitCliUtil,
+		git2.NewGitCliUtil,
 
 		router.NewTelemetryRouterImpl,
 		wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)),

api/chartRepo/ChartRepositoryRestHandler.go

@@ -30,7 +30,6 @@ import (
 	"github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin"
 	"github.com/devtron-labs/devtron/pkg/auth/user"
 	"github.com/devtron-labs/devtron/pkg/chartRepo"
-	chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository"
 	delete2 "github.com/devtron-labs/devtron/pkg/delete"
 	"github.com/gorilla/mux"
 	"go.uber.org/zap"
@@ -64,23 +63,19 @@ type ChartRepositoryRestHandlerImpl struct {
 	enforcer               casbin.Enforcer
 	validator              *validator.Validate
 	deleteService          delete2.DeleteService
-	chartRefRepository     chartRepoRepository.ChartRefRepository
-	refChartDir            chartRepoRepository.RefChartDir
 	attributesService      attributes.AttributesService
 }
 
 func NewChartRepositoryRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, chartRepositoryService chartRepo.ChartRepositoryService,
 	enforcer casbin.Enforcer, validator *validator.Validate, deleteService delete2.DeleteService,
-	chartRefRepository chartRepoRepository.ChartRefRepository, refChartDir chartRepoRepository.RefChartDir, attributesService attributes.AttributesService) *ChartRepositoryRestHandlerImpl {
+	attributesService attributes.AttributesService) *ChartRepositoryRestHandlerImpl {
 	return &ChartRepositoryRestHandlerImpl{
 		Logger:                 Logger,
 		chartRepositoryService: chartRepositoryService,
 		userAuthService:        userAuthService,
 		enforcer:               enforcer,
 		validator:              validator,
 		deleteService:          deleteService,
-		chartRefRepository:     chartRefRepository,
-		refChartDir:            refChartDir,
 		attributesService:      attributesService,
 	}
 }

api/deployment/DeploymentConfigRestHandler.go

@@ -3,6 +3,8 @@ package deployment
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef"
+	"github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -15,12 +17,10 @@ import (
 	"github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin"
 	"github.com/devtron-labs/devtron/pkg/auth/user"
 	"github.com/devtron-labs/devtron/pkg/chart"
-	chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository"
 	"github.com/devtron-labs/devtron/pkg/sql"
 	"github.com/gorilla/mux"
 	"github.com/juju/errors"
 	"go.uber.org/zap"
-	"gopkg.in/go-playground/validator.v9"
 )
 
 type DeploymentConfigRestHandler interface {
@@ -31,13 +31,11 @@ type DeploymentConfigRestHandler interface {
 }
 
 type DeploymentConfigRestHandlerImpl struct {
-	Logger             *zap.SugaredLogger
-	userAuthService    user.UserService
-	enforcer           casbin.Enforcer
-	validator          *validator.Validate
-	refChartDir        chartRepoRepository.RefChartDir
-	chartService       chart.ChartService
-	chartRefRepository chartRepoRepository.ChartRefRepository
+	Logger          *zap.SugaredLogger
+	userAuthService user.UserService
+	enforcer        casbin.Enforcer
+	chartService    chart.ChartService
+	chartRefService chartRef.ChartRefService
 }
 
 type DeploymentChartInfo struct {
@@ -49,16 +47,14 @@ type DeploymentChartInfo struct {
 	Message      string `json:"message"`
 }
 
-func NewDeploymentConfigRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer, validator *validator.Validate,
-	refChartDir chartRepoRepository.RefChartDir, chartService chart.ChartService, chartRefRepository chartRepoRepository.ChartRefRepository) *DeploymentConfigRestHandlerImpl {
+func NewDeploymentConfigRestHandlerImpl(Logger *zap.SugaredLogger, userAuthService user.UserService, enforcer casbin.Enforcer,
+	chartService chart.ChartService, chartRefService chartRef.ChartRefService) *DeploymentConfigRestHandlerImpl {
 	return &DeploymentConfigRestHandlerImpl{
-		Logger:             Logger,
-		userAuthService:    userAuthService,
-		enforcer:           enforcer,
-		validator:          validator,
-		refChartDir:        refChartDir,
-		chartService:       chartService,
-		chartRefRepository: chartRefRepository,
+		Logger:          Logger,
+		userAuthService: userAuthService,
+		enforcer:        enforcer,
+		chartService:    chartService,
+		chartRefService: chartRefService,
 	}
 }
 
@@ -88,7 +84,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo
 		return
 	}
 
-	err = handler.chartService.ValidateUploadedFileFormat(fileHeader.Filename)
+	err = handler.chartRefService.ValidateCustomChartUploadedFileFormat(fileHeader.Filename)
 	if err != nil {
 		handler.Logger.Errorw("request err, Unsupported format", "err", err, "payload", file)
 		common.WriteJsonResp(w, errors.New("Unsupported format file is uploaded, please upload file with .tgz extension"), nil, http.StatusBadRequest)
@@ -102,7 +98,7 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo
 		return
 	}
 
-	chartInfo, err := handler.chartService.ExtractChartIfMissing(fileBytes, string(handler.refChartDir), "")
+	chartInfo, err := handler.chartRefService.ExtractChartIfMissing(fileBytes, bean.RefChartDirPath, "")
 
 	if err != nil {
 		if chartInfo != nil && chartInfo.TemporaryFolder != "" {
@@ -111,15 +107,15 @@ func (handler *DeploymentConfigRestHandlerImpl) CreateChartFromFile(w http.Respo
 				handler.Logger.Errorw("error in deleting temp dir ", "err", err1)
 			}
 		}
-		if err.Error() == chart.CHART_ALREADY_EXISTS_INTERNAL_ERROR || err.Error() == chart.CHART_NAME_RESERVED_INTERNAL_ERROR {
+		if err.Error() == bean.ChartAlreadyExistsInternalError || err.Error() == bean.ChartNameReservedInternalError {
 			common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 			return
 		}
 		common.WriteJsonResp(w, fmt.Errorf(err.Error()), nil, http.StatusBadRequest)
 		return
 	}
 
-	chartRefs := &chartRepoRepository.ChartRef{
+	chartRefs := &bean.CustomChartRefDto{
 		Name:             chartInfo.ChartName,
 		Version:          chartInfo.ChartVersion,
 		Location:         chartInfo.ChartLocation,
@@ -178,25 +174,25 @@ func (handler *DeploymentConfigRestHandlerImpl) SaveChart(w http.ResponseWriter,
 		return
 	}
 
-	location := filepath.Join(string(handler.refChartDir), request.FileId)
+	location := filepath.Join(bean.RefChartDirPath, request.FileId)
 	if request.Action == "Save" {
 		file, err := ioutil.ReadFile(filepath.Join(location, "output.json"))
 		if err != nil {
 			handler.Logger.Errorw("Error reading output.json", "err", err)
 			common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 			return
 		}
-		chartRefs := &chartRepoRepository.ChartRef{}
-		err = json.Unmarshal(file, &chartRefs)
+		customChartRefDto := &bean.CustomChartRefDto{}
+		err = json.Unmarshal(file, &customChartRefDto)
 		if err != nil {
 			handler.Logger.Errorw("unmarshall err", "err", err)
 			common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
 			return
 		}
-		chartRefs.ChartDescription = request.Description
-		err = handler.chartRefRepository.Save(chartRefs)
+		customChartRefDto.ChartDescription = request.Description
+		err = handler.chartRefService.SaveCustomChart(customChartRefDto)
 		if err != nil {
-			handler.Logger.Errorw("error in saving Chart", "err", err)
+			handler.Logger.Errorw("error in saving Chart", "err", err, "request", customChartRefDto)
 			common.WriteJsonResp(w, err, "Chart couldn't be saved", http.StatusInternalServerError)
 			return
 		}
@@ -234,7 +230,7 @@ func (handler *DeploymentConfigRestHandlerImpl) DownloadChart(w http.ResponseWri
 		common.WriteJsonResp(w, fmt.Errorf("error in parsing chartRefId : %s must be integer", chartRefId), nil, http.StatusBadRequest)
 		return
 	}
-	manifestByteArr, err := handler.chartService.GetCustomChartInBytes(chartRefId)
+	manifestByteArr, err := handler.chartRefService.GetChartInBytes(chartRefId)
 	if err != nil {
 		handler.Logger.Errorw("error in converting chart to bytes", "err", err)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
@@ -258,7 +254,7 @@ func (handler *DeploymentConfigRestHandlerImpl) GetUploadedCharts(w http.Respons
 		return
 	}
 
-	charts, err := handler.chartService.FetchCustomChartsInfo()
+	charts, err := handler.chartRefService.FetchCustomChartsInfo()
 	if err != nil {
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
 		return

api/restHandler/BatchOperationRestHandler.go

@@ -145,15 +145,3 @@ func validatePipeline(pipeline *v1.Pipeline, props v1.InheritedProps) error {
 	}
 	return nil
 }
-
-func executePipeline(pipeline *v1.Pipeline, props v1.InheritedProps) error {
-	if pipeline.Build == nil && pipeline.Deployment == nil {
-		return nil
-	} else if pipeline.Build != nil {
-		pipeline.Build.UpdateMissingProps(props)
-		return validation.ValidateBuild(pipeline.Build)
-	} else if pipeline.Deployment != nil {
-		//return batch.ExecuteDeployment(pipeline.Deployment, props)
-	}
-	return nil
-}

api/restHandler/ChartRefRestHandler.go

@@ -19,7 +19,8 @@ package restHandler
 
 import (
 	"github.com/devtron-labs/devtron/api/restHandler/common"
-	"github.com/devtron-labs/devtron/pkg/chart"
+	chartService "github.com/devtron-labs/devtron/pkg/chart"
+	"github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef"
 	"github.com/gorilla/mux"
 	"go.uber.org/zap"
 	"net/http"
@@ -33,17 +34,19 @@ type ChartRefRestHandler interface {
 }
 
 type ChartRefRestHandlerImpl struct {
-	chartService chart.ChartService
-	logger       *zap.SugaredLogger
+	logger          *zap.SugaredLogger
+	chartRefService chartRef.ChartRefService
+	chartService    chartService.ChartService
 }
 
-func NewChartRefRestHandlerImpl(chartService chart.ChartService, logger *zap.SugaredLogger) *ChartRefRestHandlerImpl {
-	handler := &ChartRefRestHandlerImpl{chartService: chartService, logger: logger}
+func NewChartRefRestHandlerImpl(logger *zap.SugaredLogger, chartRefService chartRef.ChartRefService,
+	chartService chartService.ChartService) *ChartRefRestHandlerImpl {
+	handler := &ChartRefRestHandlerImpl{logger: logger, chartRefService: chartRefService, chartService: chartService}
 	return handler
 }
 
 func (handler ChartRefRestHandlerImpl) ChartRefAutocomplete(w http.ResponseWriter, r *http.Request) {
-	result, err := handler.chartService.ChartRefAutocomplete()
+	result, err := handler.chartRefService.ChartRefAutocomplete()
 	if err != nil {
 		handler.logger.Errorw("service err, ChartRefAutocomplete", "err", err)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/restHandler/CommonRestHanlder.go

@@ -21,38 +21,29 @@ import (
 	"net/http"
 
 	"github.com/devtron-labs/devtron/api/restHandler/common"
-	"github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin"
 	"github.com/devtron-labs/devtron/pkg/auth/user"
 	"github.com/devtron-labs/devtron/pkg/commonService"
-	"github.com/devtron-labs/devtron/pkg/gitops"
 	"go.uber.org/zap"
-	"gopkg.in/go-playground/validator.v9"
 )
 
 type CommonRestHanlder interface {
 	GlobalChecklist(w http.ResponseWriter, r *http.Request)
 }
 
 type CommonRestHanlderImpl struct {
-	logger              *zap.SugaredLogger
-	gitOpsConfigService gitops.GitOpsConfigService
-	userAuthService     user.UserService
-	validator           *validator.Validate
-	enforcer            casbin.Enforcer
-	commonService       commonService.CommonService
+	logger          *zap.SugaredLogger
+	userAuthService user.UserService
+	commonService   commonService.CommonService
 }
 
 func NewCommonRestHanlderImpl(
 	logger *zap.SugaredLogger,
-	gitOpsConfigService gitops.GitOpsConfigService, userAuthService user.UserService,
-	validator *validator.Validate, enforcer casbin.Enforcer, commonService commonService.CommonService) *CommonRestHanlderImpl {
+	userAuthService user.UserService,
+	commonService commonService.CommonService) *CommonRestHanlderImpl {
 	return &CommonRestHanlderImpl{
-		logger:              logger,
-		gitOpsConfigService: gitOpsConfigService,
-		userAuthService:     userAuthService,
-		validator:           validator,
-		enforcer:            enforcer,
-		commonService:       commonService,
+		logger:          logger,
+		userAuthService: userAuthService,
+		commonService:   commonService,
 	}
 }