From 8fb75a69f2fea83b8decbbe09fad2736243b326c Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Tue, 1 Nov 2022 13:29:44 +0530 Subject: [PATCH 01/13] added app worfklow api clone --- api/appbean/AppDetail.go | 6 +++ api/restHandler/CoreAppRestHandler.go | 75 +++++++++++++++++++++++++++ api/router/CoreAppRouter.go | 2 + 3 files changed, 83 insertions(+) diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index c000afe747..e04a48cff6 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -16,6 +16,12 @@ type AppDetail struct { EnvironmentOverrides map[string]*EnvironmentOverride `json:"environmentOverride"` } +type AppWorkflowCloneDto struct { + AppId int `json:"appId"` + AppWorkflows []*AppWorkflow `json:"workflows"` + EnvironmentOverrides map[string]*EnvironmentOverride `json:"environmentOverride"` +} + type AppMetadata struct { AppName string `json:"appName" validate:"required"` ProjectName string `json:"projectName" validate:"required"` diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 131c15ce44..dfa62017a3 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -64,6 +64,7 @@ const ( type CoreAppRestHandler interface { GetAppAllDetail(w http.ResponseWriter, r *http.Request) CreateApp(w http.ResponseWriter, r *http.Request) + CreateAppWorkflow(w http.ResponseWriter, r *http.Request) } type CoreAppRestHandlerImpl struct { @@ -1975,3 +1976,77 @@ func ExtractErrorType(err error) int { return 0 } } + + + +func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r *http.Request) { + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + token := r.Header.Get("token") + acdToken, err := handler.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + handler.logger.Errorw("error in getting acd token", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + ctx := context.WithValue(r.Context(), "token", acdToken) + var createAppRequest appBean.AppWorkflowCloneDto + err = decoder.Decode(&createAppRequest) + if err != nil { + handler.logger.Errorw("request err, CreateApp by API", "err", err, "CreateApp", createAppRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + //to add more validations here + handler.logger.Infow("request payload, CreateApp by API", "CreateApp", createAppRequest) + err = handler.validator.Struct(createAppRequest) + if err != nil { + handler.logger.Errorw("validation err, CreateApp by API", "err", err, "CreateApp", createAppRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + object:=handler.enforcerUtil.GetAppRBACNameByAppId(createAppRequest.AppId) + // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, object); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + //rbac ends + + handler.logger.Infow("creating app v2", "createAppRequest", createAppRequest) + var errResp *multierror.Error + var statusCode int + appId:= 1 + //creating workflow starts + if createAppRequest.AppWorkflows != nil { + err, statusCode = handler.createWorkflows(ctx, appId, userId, createAppRequest.AppWorkflows, token, createAppRequest.Metadata.AppName) + if err != nil { + common.WriteJsonResp(w, errResp, nil, statusCode) + return + } + } + //creating workflow ends + + //creating environment override starts + if createAppRequest.EnvironmentOverrides != nil { + err, statusCode = handler.createEnvOverrides(ctx, appId, userId, createAppRequest.EnvironmentOverrides, token) + if err != nil { + errResp = multierror.Append(errResp, err) + errInAppDelete := handler.deleteApp(ctx, appId, userId) + if errInAppDelete != nil { + errResp = multierror.Append(errResp, fmt.Errorf("%s : %w", APP_DELETE_FAILED_RESP, errInAppDelete)) + } + common.WriteJsonResp(w, errResp, nil, statusCode) + return + } + } + //creating environment override ends + + common.WriteJsonResp(w, nil, APP_CREATE_SUCCESSFUL_RESP, http.StatusOK) +} diff --git a/api/router/CoreAppRouter.go b/api/router/CoreAppRouter.go index 03aafbcb25..bd55f19976 100644 --- a/api/router/CoreAppRouter.go +++ b/api/router/CoreAppRouter.go @@ -38,4 +38,6 @@ func NewCoreAppRouterImpl(restHandler restHandler.CoreAppRestHandler) *CoreAppRo func (router CoreAppRouterImpl) initCoreAppRouter(configRouter *mux.Router) { configRouter.Path("/v1beta1/application").HandlerFunc(router.restHandler.CreateApp).Methods("POST") configRouter.Path("/v1beta1/application/{appId}").HandlerFunc(router.restHandler.GetAppAllDetail).Methods("GET") + configRouter.Path("/v1beta1/application/workflow").HandlerFunc(router.restHandler.CreateAppWorkflow).Methods("POST") + } From eff9e2260964b689c17d35b934bec970c9cdf51b Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Tue, 1 Nov 2022 14:08:02 +0530 Subject: [PATCH 02/13] added app workflow fetch and create clone api --- api/restHandler/CoreAppRestHandler.go | 66 ++++++++++++++++++++++----- api/router/CoreAppRouter.go | 2 +- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index dfa62017a3..654e867b33 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -65,6 +65,7 @@ type CoreAppRestHandler interface { GetAppAllDetail(w http.ResponseWriter, r *http.Request) CreateApp(w http.ResponseWriter, r *http.Request) CreateAppWorkflow(w http.ResponseWriter, r *http.Request) + GetAppWorkflow(w http.ResponseWriter, r *http.Request) } type CoreAppRestHandlerImpl struct { @@ -1977,8 +1978,6 @@ func ExtractErrorType(err error) int { } } - - func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) @@ -2011,7 +2010,7 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r return } - object:=handler.enforcerUtil.GetAppRBACNameByAppId(createAppRequest.AppId) + object := handler.enforcerUtil.GetAppRBACNameByAppId(createAppRequest.AppId) // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, object); !ok { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) @@ -2022,10 +2021,16 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r handler.logger.Infow("creating app v2", "createAppRequest", createAppRequest) var errResp *multierror.Error var statusCode int - appId:= 1 + + app, err := handler.pipelineBuilder.GetApp(createAppRequest.AppId) + if err != nil { + common.WriteJsonResp(w, errResp, nil, statusCode) + return + } + appName := app.AppName //creating workflow starts if createAppRequest.AppWorkflows != nil { - err, statusCode = handler.createWorkflows(ctx, appId, userId, createAppRequest.AppWorkflows, token, createAppRequest.Metadata.AppName) + err, statusCode = handler.createWorkflows(ctx, createAppRequest.AppId, userId, createAppRequest.AppWorkflows, token, appName) if err != nil { common.WriteJsonResp(w, errResp, nil, statusCode) return @@ -2035,13 +2040,8 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r //creating environment override starts if createAppRequest.EnvironmentOverrides != nil { - err, statusCode = handler.createEnvOverrides(ctx, appId, userId, createAppRequest.EnvironmentOverrides, token) + err, statusCode = handler.createEnvOverrides(ctx, createAppRequest.AppId, userId, createAppRequest.EnvironmentOverrides, token) if err != nil { - errResp = multierror.Append(errResp, err) - errInAppDelete := handler.deleteApp(ctx, appId, userId) - if errInAppDelete != nil { - errResp = multierror.Append(errResp, fmt.Errorf("%s : %w", APP_DELETE_FAILED_RESP, errInAppDelete)) - } common.WriteJsonResp(w, errResp, nil, statusCode) return } @@ -2050,3 +2050,47 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r common.WriteJsonResp(w, nil, APP_CREATE_SUCCESSFUL_RESP, http.StatusOK) } + +func (handler CoreAppRestHandlerImpl) GetAppWorkflow(w http.ResponseWriter, r *http.Request) { + + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + + vars := mux.Vars(r) + appId, err := strconv.Atoi(vars["appId"]) + if err != nil { + handler.logger.Errorw("request err, GetAppWorkflow", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + token := r.Header.Get("token") + //get/build app workflows starts + appWorkflows, err, statusCode := handler.buildAppWorkflows(appId) + if err != nil { + common.WriteJsonResp(w, err, nil, statusCode) + return + } + //get/build app workflows ends + + //get/build environment override starts + environmentOverrides, err, statusCode := handler.buildEnvironmentOverrides(appId, token) + if err != nil { + common.WriteJsonResp(w, err, nil, statusCode) + return + } + //get/build environment override ends + + //build full object for response + appDetail := &appBean.AppWorkflowCloneDto{ + AppId: appId, + AppWorkflows: appWorkflows, + EnvironmentOverrides: environmentOverrides, + } + //end + + common.WriteJsonResp(w, nil, appDetail, http.StatusOK) +} diff --git a/api/router/CoreAppRouter.go b/api/router/CoreAppRouter.go index bd55f19976..b63eaed84a 100644 --- a/api/router/CoreAppRouter.go +++ b/api/router/CoreAppRouter.go @@ -39,5 +39,5 @@ func (router CoreAppRouterImpl) initCoreAppRouter(configRouter *mux.Router) { configRouter.Path("/v1beta1/application").HandlerFunc(router.restHandler.CreateApp).Methods("POST") configRouter.Path("/v1beta1/application/{appId}").HandlerFunc(router.restHandler.GetAppAllDetail).Methods("GET") configRouter.Path("/v1beta1/application/workflow").HandlerFunc(router.restHandler.CreateAppWorkflow).Methods("POST") - + configRouter.Path("/v1beta1/application/workflow/{appId}").HandlerFunc(router.restHandler.GetAppWorkflow).Methods("GET") } From b10fc9db5e49d73c48b3c1e0298a979232308d8f Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Nov 2022 17:32:20 +0530 Subject: [PATCH 03/13] added support for wf and ci pipeline deletion --- Wire.go | 5 +- api/restHandler/AppWorkflowRestHandler.go | 2 +- api/restHandler/BulkUpdateRestHandler.go | 104 +++++- api/restHandler/CoreAppRestHandler.go | 2 +- .../app/DeploymentPipelineRestHandler.go | 78 ----- api/router/BulkUpdateRouter.go | 1 + api/router/PipelineConfigRouter.go | 1 - internal/sql/repository/app/AppRepository.go | 21 +- .../appWorkflow/AppWorkflowRepository.go | 35 ++ pkg/appWorkflow/AppWorkflowService.go | 6 +- pkg/bean/app.go | 22 -- .../BulkUpdateService.go | 302 +++++++++++------- pkg/bulkAction/bean.go | 132 ++++++++ pkg/pipeline/PipelineBuilder.go | 102 +----- specs/bulk-env-delete.yaml | 2 +- tests/pipeline/BulkUpdateService_test.go | 22 +- tests/pipeline/ChartService_test.go | 18 +- wire_gen.go | 10 +- 18 files changed, 520 insertions(+), 345 deletions(-) rename pkg/{pipeline => bulkAction}/BulkUpdateService.go (88%) create mode 100644 pkg/bulkAction/bean.go diff --git a/Wire.go b/Wire.go index 4848d05aa9..1c3f833a6d 100644 --- a/Wire.go +++ b/Wire.go @@ -79,6 +79,7 @@ import ( appStoreDeploymentGitopsTool "github.com/devtron-labs/devtron/pkg/appStore/deployment/tool/gitops" "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" + "github.com/devtron-labs/devtron/pkg/bulkAction" "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/commonService" @@ -219,8 +220,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), chart.NewChartServiceImpl, wire.Bind(new(chart.ChartService), new(*chart.ChartServiceImpl)), - pipeline.NewBulkUpdateServiceImpl, - wire.Bind(new(pipeline.BulkUpdateService), new(*pipeline.BulkUpdateServiceImpl)), + bulkAction.NewBulkUpdateServiceImpl, + wire.Bind(new(bulkAction.BulkUpdateService), new(*bulkAction.BulkUpdateServiceImpl)), repository.NewGitProviderRepositoryImpl, wire.Bind(new(repository.GitProviderRepository), new(*repository.GitProviderRepositoryImpl)), diff --git a/api/restHandler/AppWorkflowRestHandler.go b/api/restHandler/AppWorkflowRestHandler.go index dd606fce08..56227871bb 100644 --- a/api/restHandler/AppWorkflowRestHandler.go +++ b/api/restHandler/AppWorkflowRestHandler.go @@ -132,7 +132,7 @@ func (handler AppWorkflowRestHandlerImpl) DeleteAppWorkflow(w http.ResponseWrite } //rback block ends here - err = handler.appWorkflowService.DeleteAppWorkflow(appId, appWorkflowId, userId) + err = handler.appWorkflowService.DeleteAppWorkflow(appWorkflowId, userId) if err != nil { if _, ok := err.(*util.ApiError); ok { handler.Logger.Warnw("error on deleting", "err", err) diff --git a/api/restHandler/BulkUpdateRestHandler.go b/api/restHandler/BulkUpdateRestHandler.go index ee507696d8..945f35e534 100644 --- a/api/restHandler/BulkUpdateRestHandler.go +++ b/api/restHandler/BulkUpdateRestHandler.go @@ -11,6 +11,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/pkg/appClone" "github.com/devtron-labs/devtron/pkg/appWorkflow" + "github.com/devtron-labs/devtron/pkg/bulkAction" "github.com/devtron-labs/devtron/pkg/chart" request "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/pipeline" @@ -24,6 +25,7 @@ import ( "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" "net/http" + "strconv" "strings" ) @@ -36,13 +38,15 @@ type BulkUpdateRestHandler interface { BulkUnHibernate(w http.ResponseWriter, r *http.Request) BulkDeploy(w http.ResponseWriter, r *http.Request) BulkBuildTrigger(w http.ResponseWriter, r *http.Request) + + HandleCdPipelineBulkAction(w http.ResponseWriter, r *http.Request) } type BulkUpdateRestHandlerImpl struct { pipelineBuilder pipeline.PipelineBuilder ciPipelineRepository pipelineConfig.CiPipelineRepository ciHandler pipeline.CiHandler logger *zap.SugaredLogger - bulkUpdateService pipeline.BulkUpdateService + bulkUpdateService bulkAction.BulkUpdateService chartService chart.ChartService propertiesConfigService pipeline.PropertiesConfigService dbMigrationService pipeline.DbMigrationService @@ -67,7 +71,7 @@ type BulkUpdateRestHandlerImpl struct { } func NewBulkUpdateRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, logger *zap.SugaredLogger, - bulkUpdateService pipeline.BulkUpdateService, + bulkUpdateService bulkAction.BulkUpdateService, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, dbMigrationService pipeline.DbMigrationService, @@ -129,7 +133,7 @@ func (handler BulkUpdateRestHandlerImpl) FindBulkUpdateReadme(w http.ResponseWri return } //auth free, only login required - var responseArr []*pipeline.BulkUpdateSeeExampleResponse + var responseArr []*bulkAction.BulkUpdateSeeExampleResponse responseArr = append(responseArr, response) common.WriteJsonResp(w, nil, responseArr, http.StatusOK) } @@ -151,7 +155,7 @@ func (handler BulkUpdateRestHandlerImpl) CheckAuthForImpactedObjects(AppId int, } func (handler BulkUpdateRestHandlerImpl) GetImpactedAppsName(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var script pipeline.BulkUpdateScript + var script bulkAction.BulkUpdateScript err := decoder.Decode(&script) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -206,7 +210,7 @@ func (handler BulkUpdateRestHandlerImpl) CheckAuthForBulkUpdate(AppId int, EnvId } func (handler BulkUpdateRestHandlerImpl) BulkUpdate(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var script pipeline.BulkUpdateScript + var script bulkAction.BulkUpdateScript err := decoder.Decode(&script) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -255,7 +259,7 @@ func (handler BulkUpdateRestHandlerImpl) BulkHibernate(w http.ResponseWriter, r return } decoder := json.NewDecoder(r.Body) - var request pipeline.BulkApplicationForEnvironmentPayload + var request bulkAction.BulkApplicationForEnvironmentPayload err = decoder.Decode(&request) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -292,7 +296,7 @@ func (handler BulkUpdateRestHandlerImpl) BulkUnHibernate(w http.ResponseWriter, return } decoder := json.NewDecoder(r.Body) - var request pipeline.BulkApplicationForEnvironmentPayload + var request bulkAction.BulkApplicationForEnvironmentPayload err = decoder.Decode(&request) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -329,7 +333,7 @@ func (handler BulkUpdateRestHandlerImpl) BulkDeploy(w http.ResponseWriter, r *ht return } decoder := json.NewDecoder(r.Body) - var request pipeline.BulkApplicationForEnvironmentPayload + var request bulkAction.BulkApplicationForEnvironmentPayload err = decoder.Decode(&request) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -365,7 +369,7 @@ func (handler BulkUpdateRestHandlerImpl) BulkBuildTrigger(w http.ResponseWriter, return } decoder := json.NewDecoder(r.Body) - var request pipeline.BulkApplicationForEnvironmentPayload + var request bulkAction.BulkApplicationForEnvironmentPayload err = decoder.Decode(&request) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -403,3 +407,85 @@ func (handler BulkUpdateRestHandlerImpl) checkAuthForBulkActions(token string, a } return true } + +func (handler BulkUpdateRestHandlerImpl) HandleCdPipelineBulkAction(w http.ResponseWriter, r *http.Request) { + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var cdPipelineBulkActionReq bulkAction.CdBulkActionRequestDto + err = decoder.Decode(&cdPipelineBulkActionReq) + cdPipelineBulkActionReq.UserId = userId + if err != nil { + handler.logger.Errorw("request err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + v := r.URL.Query() + forceDelete := false + forceDeleteParam := v.Get("forceDelete") + if len(forceDeleteParam) > 0 { + forceDelete, err = strconv.ParseBool(forceDeleteParam) + if err != nil { + handler.logger.Errorw("request err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + } + cdPipelineBulkActionReq.ForceDelete = forceDelete + + dryRun := false + dryRunParam := v.Get("dryRun") + if len(dryRunParam) > 0 { + dryRun, err = strconv.ParseBool(dryRunParam) + if err != nil { + handler.logger.Errorw("request err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + } + handler.logger.Infow("request payload, HandleCdPipelineBulkAction", "payload", cdPipelineBulkActionReq) + impactedPipelines, impactedAppWfIds, impactedCiPipelineIds, err := handler.bulkUpdateService.GetBulkActionImpactedPipelinesAndWfs(&cdPipelineBulkActionReq) + if err != nil { + handler.logger.Errorw("service err, GetBulkActionImpactedPipelinesAndWfs", "err", err, "payload", cdPipelineBulkActionReq) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + token := r.Header.Get("token") + + appsHavingRbacChecked := make(map[string]bool) + for _, impactedPipeline := range impactedPipelines { + //check to avoid same rbac matching multiple times + if _, ok := appsHavingRbacChecked[impactedPipeline.App.AppName]; !ok { + resourceName := handler.enforcerUtil.GetAppRBACName(impactedPipeline.App.AppName) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, resourceName); !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } else { + appsHavingRbacChecked[impactedPipeline.App.AppName] = true + } + } + object := handler.enforcerUtil.GetAppRBACByAppNameAndEnvId(impactedPipeline.App.AppName, impactedPipeline.EnvironmentId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, object); !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } + } + acdToken, err := handler.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + handler.logger.Errorw("error in getting acd token", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + ctx := context.WithValue(r.Context(), "token", acdToken) + resp, err := handler.bulkUpdateService.PerformBulkActionOnCdPipelines(&cdPipelineBulkActionReq, impactedPipelines, ctx, dryRun, impactedAppWfIds, impactedCiPipelineIds) + if err != nil { + handler.logger.Errorw("service err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) +} diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 131c15ce44..828663b466 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -1167,7 +1167,7 @@ func (handler CoreAppRestHandlerImpl) deleteApp(ctx context.Context, appId int, // delete all workflows for app starts for _, workflow := range workflowsList { - err = handler.appWorkflowService.DeleteAppWorkflow(appId, workflow.Id, userId) + err = handler.appWorkflowService.DeleteAppWorkflow(workflow.Id, userId) if err != nil { handler.logger.Errorw("service err, DeleteAppWorkflow ") return err diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 0898843be1..9567d5ec14 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -34,8 +34,6 @@ type DevtronAppDeploymentRestHandler interface { IsReadyToTrigger(w http.ResponseWriter, r *http.Request) FetchCdWorkflowDetails(w http.ResponseWriter, r *http.Request) - - HandleCdPipelineBulkAction(w http.ResponseWriter, r *http.Request) } type DevtronAppDeploymentConfigRestHandler interface { @@ -1832,79 +1830,3 @@ func (handler PipelineConfigRestHandlerImpl) UpgradeForAllApps(w http.ResponseWr response["failed"] = failedIds common.WriteJsonResp(w, err, response, http.StatusOK) } - -func (handler PipelineConfigRestHandlerImpl) HandleCdPipelineBulkAction(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var cdPipelineBulkActionReq bean.CdBulkActionRequestDto - err = decoder.Decode(&cdPipelineBulkActionReq) - cdPipelineBulkActionReq.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - v := r.URL.Query() - forceDelete := false - forceDeleteParam := v.Get("forceDelete") - if len(forceDeleteParam) > 0 { - forceDelete, err = strconv.ParseBool(forceDeleteParam) - if err != nil { - handler.Logger.Errorw("request err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - } - cdPipelineBulkActionReq.ForceDelete = forceDelete - - dryRun := false - dryRunParam := v.Get("dryRun") - if len(dryRunParam) > 0 { - dryRun, err = strconv.ParseBool(dryRunParam) - if err != nil { - handler.Logger.Errorw("request err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - } - handler.Logger.Infow("request payload, HandleCdPipelineBulkAction", "payload", cdPipelineBulkActionReq) - impactedPipelines, err := handler.pipelineBuilder.GetBulkActionImpactedPipelines(&cdPipelineBulkActionReq) - if err != nil { - handler.Logger.Errorw("service err, GetBulkActionImpactedPipelines", "err", err, "payload", cdPipelineBulkActionReq) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - token := r.Header.Get("token") - for _, impactedPipeline := range impactedPipelines { - resourceName := handler.enforcerUtil.GetAppRBACName(impactedPipeline.App.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionUpdate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - - object := handler.enforcerUtil.GetAppRBACByAppNameAndEnvId(impactedPipeline.App.AppName, impactedPipeline.EnvironmentId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, object); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - } - acdToken, err := handler.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - handler.Logger.Errorw("error in getting acd token", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - ctx := context.WithValue(r.Context(), "token", acdToken) - resp, err := handler.pipelineBuilder.PerformBulkActionOnCdPipelines(&cdPipelineBulkActionReq, impactedPipelines, ctx, dryRun) - if err != nil { - handler.Logger.Errorw("service err, HandleCdPipelineBulkAction", "err", err, "payload", cdPipelineBulkActionReq) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, nil, resp, http.StatusOK) -} diff --git a/api/router/BulkUpdateRouter.go b/api/router/BulkUpdateRouter.go index 83aede3618..dab0da5165 100644 --- a/api/router/BulkUpdateRouter.go +++ b/api/router/BulkUpdateRouter.go @@ -28,5 +28,6 @@ func (router BulkUpdateRouterImpl) initBulkUpdateRouter(bulkRouter *mux.Router) bulkRouter.Path("/v1beta1/unhibernate").HandlerFunc(router.restHandler.BulkUnHibernate).Methods("POST") bulkRouter.Path("/v1beta1/deploy").HandlerFunc(router.restHandler.BulkDeploy).Methods("POST") bulkRouter.Path("/v1beta1/build").HandlerFunc(router.restHandler.BulkBuildTrigger).Methods("POST") + bulkRouter.Path("/v1beta1/cd-pipeline").HandlerFunc(router.restHandler.HandleCdPipelineBulkAction).Methods("POST") } diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index a43df26a6b..e26c112bfa 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -70,7 +70,6 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu configRouter.Path("/cd-pipeline/patch").HandlerFunc(router.restHandler.PatchCdPipeline).Methods("POST") configRouter.Path("/cd-pipeline/{appId}").HandlerFunc(router.restHandler.GetCdPipelines).Methods("GET") configRouter.Path("/cd-pipeline/{appId}/env/{envId}").HandlerFunc(router.restHandler.GetCdPipelinesForAppAndEnv).Methods("GET") - configRouter.Path("/cd-pipeline/bulk-action").HandlerFunc(router.restHandler.HandleCdPipelineBulkAction).Methods("POST") //save environment specific override configRouter.Path("/env/{appId}/{environmentId}").HandlerFunc(router.restHandler.EnvConfigOverrideCreate).Methods("POST") configRouter.Path("/env").HandlerFunc(router.restHandler.EnvConfigOverrideUpdate).Methods("PUT") diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 410c8d79de..0878e59552 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -21,6 +21,7 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/team" "github.com/go-pg/pg" + "go.uber.org/zap" ) type App struct { @@ -57,6 +58,7 @@ type AppRepository interface { FindAppAndProjectByAppName(appName string) (*App, error) GetConnection() *pg.DB FindAllMatchesByAppName(appName string) ([]*App, error) + FindIdsByTeamIds(teamIds []int) ([]int, error) } const DevtronApp = "DevtronApp" @@ -65,10 +67,14 @@ const ExternalApp = "ExternalApp" type AppRepositoryImpl struct { dbConnection *pg.DB + logger *zap.SugaredLogger } -func NewAppRepositoryImpl(dbConnection *pg.DB) *AppRepositoryImpl { - return &AppRepositoryImpl{dbConnection: dbConnection} +func NewAppRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *AppRepositoryImpl { + return &AppRepositoryImpl{ + dbConnection: dbConnection, + logger: logger, + } } func (repo AppRepositoryImpl) GetConnection() *pg.DB { @@ -243,3 +249,14 @@ func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string) ([]*App, e err := repo.dbConnection.Model(&apps).Where("app_name ILIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", false).Select() return apps, err } + +func (repo AppRepositoryImpl) FindIdsByTeamIds(teamIds []int) ([]int, error) { + var ids []int + query := "select id from app where team_id in (?) and active = ?;" + _, err := repo.dbConnection.Query(ids, query, pg.In(teamIds), true) + if err != nil { + repo.logger.Errorw("error in getting appIds by teamIds", "err", err, "teamIds", teamIds) + return nil, err + } + return ids, err +} diff --git a/internal/sql/repository/appWorkflow/AppWorkflowRepository.go b/internal/sql/repository/appWorkflow/AppWorkflowRepository.go index 5584c78c15..304df66cc3 100644 --- a/internal/sql/repository/appWorkflow/AppWorkflowRepository.go +++ b/internal/sql/repository/appWorkflow/AppWorkflowRepository.go @@ -28,6 +28,8 @@ type AppWorkflowRepository interface { SaveAppWorkflow(wf *AppWorkflow) (*AppWorkflow, error) UpdateAppWorkflow(wf *AppWorkflow) (*AppWorkflow, error) FindByIdAndAppId(id int, appId int) (*AppWorkflow, error) + FindById(id int) (*AppWorkflow, error) + FindByIds(ids []int) (*AppWorkflow, error) FindByAppId(appId int) (appWorkflow []*AppWorkflow, err error) DeleteAppWorkflow(appWorkflow *AppWorkflow, tx *pg.Tx) error @@ -47,6 +49,7 @@ type AppWorkflowRepository interface { FindWFCDMappingByCIPipelineIds(ciPipelineIds []int) ([]*AppWorkflowMapping, error) FindWFCDMappingByParentCDPipelineId(cdPipelineId int) ([]*AppWorkflowMapping, error) FindAllWFMappingsByAppId(appId int) ([]*AppWorkflowMapping, error) + FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(envIds []int, appIds []int) ([]*AppWorkflowMapping, error) } type AppWorkflowRepositoryImpl struct { @@ -111,6 +114,24 @@ func (impl AppWorkflowRepositoryImpl) FindByIdAndAppId(id int, appId int) (*AppW return appWorkflow, err } +func (impl AppWorkflowRepositoryImpl) FindById(id int) (*AppWorkflow, error) { + appWorkflow := &AppWorkflow{} + err := impl.dbConnection.Model(appWorkflow). + Where("id = ?", id). + Where("active = ?", true). + Select() + return appWorkflow, err +} + +func (impl AppWorkflowRepositoryImpl) FindByIds(ids []int) (*AppWorkflow, error) { + appWorkflow := &AppWorkflow{} + err := impl.dbConnection.Model(appWorkflow). + Where("id in (?)", pg.In(ids)). + Where("active = ?", true). + Select() + return appWorkflow, err +} + func (impl AppWorkflowRepositoryImpl) DeleteAppWorkflow(appWorkflow *AppWorkflow, tx *pg.Tx) error { appWorkflowMappings, err := impl.FindWFCIMappingByWorkflowId(appWorkflow.Id) if err != nil && pg.ErrNoRows != err { @@ -300,3 +321,17 @@ func (impl AppWorkflowRepositoryImpl) DeleteAppWorkflowMappingsByCdPipelineId(pi } return nil } + +func (impl AppWorkflowRepositoryImpl) FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(envIds []int, appIds []int) ([]*AppWorkflowMapping, error) { + var models []*AppWorkflowMapping + query := `select * from app_workflow_mapping + where app_id in (?) and app_workflow_id not in + (select app_workflow_id from app_workflow_mapping awm inner join pipeline p on p.id=awm.component_id + and p.environment_id not in (?) and p.app_id in (?) and p.deleted = ? and awm.active = ?); ` + _, err := impl.dbConnection.Query(&models, query, pg.In(appIds), CDPIPELINE, pg.In(envIds), pg.In(appIds), false, true) + if err != nil { + impl.Logger.Errorw("error, FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly", "err", err) + return nil, err + } + return models, nil +} diff --git a/pkg/appWorkflow/AppWorkflowService.go b/pkg/appWorkflow/AppWorkflowService.go index 5e07a67a98..160d3cf2cf 100644 --- a/pkg/appWorkflow/AppWorkflowService.go +++ b/pkg/appWorkflow/AppWorkflowService.go @@ -38,7 +38,7 @@ type AppWorkflowService interface { CreateAppWorkflow(req AppWorkflowDto) (AppWorkflowDto, error) FindAppWorkflows(appId int) ([]AppWorkflowDto, error) FindAppWorkflowById(Id int, appId int) (AppWorkflowDto, error) - DeleteAppWorkflow(appId, appWorkflowId int, userId int32) error + DeleteAppWorkflow(appWorkflowId int, userId int32) error SaveAppWorkflowMapping(wf AppWorkflowMappingDto) (AppWorkflowMappingDto, error) FindAppWorkflowMapping(workflowId int) ([]AppWorkflowMappingDto, error) @@ -172,9 +172,9 @@ func (impl AppWorkflowServiceImpl) FindAppWorkflowById(Id int, appId int) (AppWo return *appWorkflowDto, err } -func (impl AppWorkflowServiceImpl) DeleteAppWorkflow(appId, appWorkflowId int, userId int32) error { +func (impl AppWorkflowServiceImpl) DeleteAppWorkflow(appWorkflowId int, userId int32) error { impl.Logger.Debugw("Deleting app-workflow: ", "appWorkflowId", appWorkflowId) - wf, err := impl.appWorkflowRepository.FindByIdAndAppId(appWorkflowId, appId) + wf, err := impl.appWorkflowRepository.FindById(appWorkflowId) if err != nil { impl.Logger.Errorw("err", err) return err diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 4520d21ddc..f3bc4e9c0e 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -598,25 +598,3 @@ type UpdateProjectBulkAppsRequest struct { TeamId int `json:"teamId"` UserId int32 `json:"-"` } - -type CdBulkAction int - -const ( - CD_BULK_DELETE CdBulkAction = iota -) - -type CdBulkActionRequestDto struct { - Action CdBulkAction `json:"action"` - EnvIds []int `json:"envIds"` - AppIds []int `json:"appIds"` - ProjectIds []int `json:"projectIds"` - ForceDelete bool `json:"forceDelete"` - UserId int32 `json:"-"` -} - -type CdBulkActionResponseDto struct { - PipelineName string `json:"pipelineName"` - AppName string `json:"appName"` - EnvironmentName string `json:"environmentName"` - DeletionResult string `json:"deletionResult,omitempty"` -} diff --git a/pkg/pipeline/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go similarity index 88% rename from pkg/pipeline/BulkUpdateService.go rename to pkg/bulkAction/BulkUpdateService.go index 078310140b..416370c108 100644 --- a/pkg/pipeline/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -1,4 +1,4 @@ -package pipeline +package bulkAction import ( "context" @@ -7,122 +7,33 @@ import ( "fmt" "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/client/argocdServer/repository" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/pipeline" + pipeline1 "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository4 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/util/rbac" - "github.com/go-pg/pg" - "net/http" - - "github.com/devtron-labs/devtron/client/argocdServer/repository" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" - "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" jsonpatch "github.com/evanphx/json-patch" + "github.com/go-pg/pg" "github.com/tidwall/gjson" "github.com/tidwall/sjson" "go.uber.org/zap" + "net/http" + "sort" ) -type NameIncludesExcludes struct { - Names []string `json:"names"` -} - -type DeploymentTemplateSpec struct { - PatchJson string `json:"patchJson"` -} -type DeploymentTemplateTask struct { - Spec *DeploymentTemplateSpec `json:"spec"` -} -type CmAndSecretSpec struct { - Names []string `json:"names"` - PatchJson string `json:"patchJson"` -} -type CmAndSecretTask struct { - Spec *CmAndSecretSpec `json:"spec"` -} -type BulkUpdatePayload struct { - Includes *NameIncludesExcludes `json:"includes"` - Excludes *NameIncludesExcludes `json:"excludes"` - EnvIds []int `json:"envIds"` - Global bool `json:"global"` - DeploymentTemplate *DeploymentTemplateTask `json:"deploymentTemplate"` - ConfigMap *CmAndSecretTask `json:"configMap"` - Secret *CmAndSecretTask `json:"secret"` -} -type BulkUpdateScript struct { - ApiVersion string `json:"apiVersion" validate:"required"` - Kind string `json:"kind" validate:"required"` - Spec *BulkUpdatePayload `json:"spec" validate:"required"` -} -type BulkUpdateSeeExampleResponse struct { - Operation string `json:"operation"` - Script *BulkUpdateScript `json:"script" validate:"required"` - ReadMe string `json:"readme"` -} -type ImpactedObjectsResponse struct { - DeploymentTemplate []*DeploymentTemplateImpactedObjectsResponseForOneApp `json:"deploymentTemplate"` - ConfigMap []*CmAndSecretImpactedObjectsResponseForOneApp `json:"configMap"` - Secret []*CmAndSecretImpactedObjectsResponseForOneApp `json:"secret"` -} -type DeploymentTemplateImpactedObjectsResponseForOneApp struct { - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvId int `json:"envId"` -} -type CmAndSecretImpactedObjectsResponseForOneApp struct { - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvId int `json:"envId"` - Names []string `json:"names"` -} -type DeploymentTemplateBulkUpdateResponseForOneApp struct { - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvId int `json:"envId"` - Message string `json:"message"` -} -type CmAndSecretBulkUpdateResponseForOneApp struct { - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvId int `json:"envId"` - Names []string `json:"names"` - Message string `json:"message"` -} -type BulkUpdateResponse struct { - DeploymentTemplate *DeploymentTemplateBulkUpdateResponse `json:"deploymentTemplate"` - ConfigMap *CmAndSecretBulkUpdateResponse `json:"configMap"` - Secret *CmAndSecretBulkUpdateResponse `json:"secret"` -} -type DeploymentTemplateBulkUpdateResponse struct { - Message []string `json:"message"` - Failure []*DeploymentTemplateBulkUpdateResponseForOneApp `json:"failure"` - Successful []*DeploymentTemplateBulkUpdateResponseForOneApp `json:"successful"` -} -type CmAndSecretBulkUpdateResponse struct { - Message []string `json:"message"` - Failure []*CmAndSecretBulkUpdateResponseForOneApp `json:"failure"` - Successful []*CmAndSecretBulkUpdateResponseForOneApp `json:"successful"` -} - -type BulkApplicationForEnvironmentPayload struct { - AppIdIncludes []int `json:"appIdIncludes,omitempty"` - AppIdExcludes []int `json:"appIdExcludes,omitempty"` - EnvId int `json:"envId"` - UserId int32 `json:"-"` -} - -type BulkApplicationForEnvironmentResponse struct { - BulkApplicationForEnvironmentPayload - Response map[string]map[string]bool `json:"response"` -} - type BulkUpdateService interface { FindBulkUpdateReadme(operation string) (response *BulkUpdateSeeExampleResponse, err error) GetBulkAppName(bulkUpdateRequest *BulkUpdatePayload) (*ImpactedObjectsResponse, error) @@ -136,6 +47,9 @@ type BulkUpdateService interface { BulkUnHibernate(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) BulkDeploy(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) BulkBuildTrigger(request *BulkApplicationForEnvironmentPayload, ctx context.Context, w http.ResponseWriter, token string, checkAuthForBulkActions func(token string, appObject string, envObject string) bool) (*BulkApplicationForEnvironmentResponse, error) + + GetBulkActionImpactedPipelinesAndWfs(dto *CdBulkActionRequestDto) ([]*pipelineConfig.Pipeline, []int, []int, error) + PerformBulkActionOnCdPipelines(dto *CdBulkActionRequestDto, impactedPipelines []*pipelineConfig.Pipeline, ctx context.Context, dryRun bool, impactedAppWfIds []int, impactedCiPipelineIds []int) (*PipelineAndWfBulkActionResponseDto, error) } type BulkUpdateServiceImpl struct { @@ -159,14 +73,16 @@ type BulkUpdateServiceImpl struct { appRepository app.AppRepository deploymentTemplateHistoryService history.DeploymentTemplateHistoryService configMapHistoryService history.ConfigMapHistoryService - workflowDagExecutor WorkflowDagExecutor + workflowDagExecutor pipeline.WorkflowDagExecutor cdWorkflowRepository pipelineConfig.CdWorkflowRepository - pipelineBuilder PipelineBuilder + pipelineBuilder pipeline.PipelineBuilder helmAppService client.HelmAppService enforcerUtil rbac.EnforcerUtil enforcerUtilHelm rbac.EnforcerUtilHelm - ciHandler CiHandler + ciHandler pipeline.CiHandler ciPipelineRepository pipelineConfig.CiPipelineRepository + appWorkflowRepository appWorkflow.AppWorkflowRepository + appWorkflowService appWorkflow2.AppWorkflowService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -188,10 +104,13 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito client *http.Client, appRepository app.AppRepository, deploymentTemplateHistoryService history.DeploymentTemplateHistoryService, - configMapHistoryService history.ConfigMapHistoryService, workflowDagExecutor WorkflowDagExecutor, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineBuilder PipelineBuilder, + configMapHistoryService history.ConfigMapHistoryService, workflowDagExecutor pipeline.WorkflowDagExecutor, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineBuilder pipeline.PipelineBuilder, helmAppService client.HelmAppService, enforcerUtil rbac.EnforcerUtil, - enforcerUtilHelm rbac.EnforcerUtilHelm, ciHandler CiHandler, ciPipelineRepository pipelineConfig.CiPipelineRepository) *BulkUpdateServiceImpl { + enforcerUtilHelm rbac.EnforcerUtilHelm, ciHandler pipeline.CiHandler, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + appWorkflowRepository appWorkflow.AppWorkflowRepository, + appWorkflowService appWorkflow2.AppWorkflowService) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, chartRepository: chartRepository, @@ -221,6 +140,8 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito enforcerUtilHelm: enforcerUtilHelm, ciHandler: ciHandler, ciPipelineRepository: ciPipelineRepository, + appWorkflowRepository: appWorkflowRepository, + appWorkflowService: appWorkflowService, } } @@ -1079,11 +1000,11 @@ func (impl BulkUpdateServiceImpl) BulkHibernate(request *BulkApplicationForEnvir } if pipeline.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { - stopRequest := &StopAppRequest{ + stopRequest := &pipeline1.StopAppRequest{ AppId: pipeline.AppId, EnvironmentId: pipeline.EnvironmentId, UserId: request.UserId, - RequestType: STOP, + RequestType: pipeline1.STOP, } _, err := impl.workflowDagExecutor.StopStartApp(stopRequest, ctx) if err != nil { @@ -1144,11 +1065,11 @@ func (impl BulkUpdateServiceImpl) BulkUnHibernate(request *BulkApplicationForEnv } if pipeline.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { - stopRequest := &StopAppRequest{ + stopRequest := &pipeline1.StopAppRequest{ AppId: pipeline.AppId, EnvironmentId: pipeline.EnvironmentId, UserId: request.UserId, - RequestType: START, + RequestType: pipeline1.START, } _, err := impl.workflowDagExecutor.StopStartApp(stopRequest, ctx) if err != nil { @@ -1360,3 +1281,158 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *BulkApplicationForEn bulkOperationResponse.Response = response return bulkOperationResponse, nil } + +func (impl BulkUpdateServiceImpl) GetBulkActionImpactedPipelinesAndWfs(dto *CdBulkActionRequestDto) ([]*pipelineConfig.Pipeline, []int, []int, error) { + var err error + if len(dto.EnvIds) == 0 || (len(dto.AppIds) == 0 && len(dto.ProjectIds) == 0) { + //invalid payload, envIds are must and either of appIds or projectIds are must + return nil, nil, nil, &util.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "invalid payload, can not get pipelines for this filter"} + } + + if len(dto.ProjectIds) > 0 { + appIdsInProjects, err := impl.appRepository.FindIdsByTeamIds(dto.ProjectIds) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting appIds by projectIds", "err", err, "projectIds", dto.ProjectIds) + return nil, nil, nil, err + } + dto.AppIds = append(dto.AppIds, appIdsInProjects...) + } + var impactedWfIds []int + var impactedPipelineIds []int + var impactedCiPipelineIds []int + if len(dto.AppIds) > 0 && len(dto.EnvIds) > 0 { + if !dto.DeleteWfAndCiPipeline { + //getting pipeline IDs for app level deletion request + impactedPipelineIds, err = impl.pipelineRepository.FindIdsByAppIdsAndEnvironmentIds(dto.AppIds, dto.EnvIds) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting cd pipelines by appIds and envIds", "err", err) + return nil, nil, nil, err + } + + } else { + //getting all workflows in given apps which do not have pipelines of other than given environments + appWfs, err := impl.appWorkflowRepository.FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(dto.EnvIds, dto.AppIds) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting wfs having cd pipelines from specific env only", "err", err) + return nil, nil, nil, err + } + + for _, appWf := range appWfs { + if appWf.Type == appWorkflow.CDPIPELINE { + impactedPipelineIds = append(impactedPipelineIds, appWf.ComponentId) + } else if appWf.Type == appWorkflow.CIPIPELINE { + impactedCiPipelineIds = append(impactedCiPipelineIds, appWf.ComponentId) + } + impactedWfIds = append(impactedWfIds, appWf.AppWorkflowId) + } + } + } + var pipelines []*pipelineConfig.Pipeline + if len(impactedPipelineIds) > 0 { + pipelines, err = impl.pipelineRepository.FindByIdsIn(impactedPipelineIds) + if err != nil { + impl.logger.Errorw("error in getting cd pipelines by ids", "err", err, "ids", impactedPipelineIds) + return nil, nil, nil, err + } + } + return pipelines, impactedWfIds, impactedCiPipelineIds, nil +} + +func (impl BulkUpdateServiceImpl) PerformBulkActionOnCdPipelines(dto *CdBulkActionRequestDto, impactedPipelines []*pipelineConfig.Pipeline, + ctx context.Context, dryRun bool, impactedAppWfIds []int, impactedCiPipelineIds []int) (*PipelineAndWfBulkActionResponseDto, error) { + switch dto.Action { + case CD_BULK_DELETE: + bulkDeleteResp, err := impl.PerformBulkDeleteActionOnCdPipelines(impactedPipelines, ctx, dryRun, dto.ForceDelete, false, impactedAppWfIds, impactedCiPipelineIds, dto.UserId) + if err != nil { + impl.logger.Errorw("error in cd pipelines bulk deletion") + } + return bulkDeleteResp, nil + default: + return nil, &util.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "this action is not supported"} + } +} + +func (impl BulkUpdateServiceImpl) PerformBulkDeleteActionOnCdPipelines(impactedPipelines []*pipelineConfig.Pipeline, ctx context.Context, + dryRun, forceDelete, deleteWfAndCiPipeline bool, impactedAppWfIds, impactedCiPipelineIds []int, userId int32) (*PipelineAndWfBulkActionResponseDto, error) { + var cdPipelineRespDtos []*CdBulkActionResponseDto + var wfRespDtos []*WfBulkActionResponseDto + var ciPipelineRespDtos []*CiBulkActionResponseDto + //sorting pipelines in decreasing order to tackle problem of sequential pipelines + //here we are assuming that for now sequential pipelines can only be made through UI and pipelines can not be moved + //(thus ids in decreasing order should not create problems when deleting) + //also sorting does not guarantee deletion because impacted pipelines can be sequential but not necessarily linked to each other + //TODO: implement stack type solution to order pipeline by index in appWfs if pipeline moving is introduced + if impactedPipelines != nil { + sort.SliceStable(impactedPipelines, func(i, j int) bool { + return impactedPipelines[i].Id > impactedPipelines[j].Id + }) + } + for _, pipeline := range impactedPipelines { + respDto := &CdBulkActionResponseDto{ + PipelineName: pipeline.Name, + AppName: pipeline.App.AppName, + EnvironmentName: pipeline.Environment.Name, + } + if !dryRun { + err := impl.pipelineBuilder.DeleteCdPipeline(pipeline, ctx, forceDelete) + if err != nil { + impl.logger.Errorw("error in deleting cd pipeline", "err", err, "pipelineId", pipeline.Id) + respDto.DeletionResult = fmt.Sprintf("Not able to delete pipeline, %v", err) + } else { + respDto.DeletionResult = "Pipeline deleted successfully." + } + } + cdPipelineRespDtos = append(cdPipelineRespDtos, respDto) + } + if deleteWfAndCiPipeline { + for _, impactedCiPipelineId := range impactedCiPipelineIds { + ciPipeline, err := impl.pipelineBuilder.GetCiPipelineById(impactedCiPipelineId) + if err != nil { + impl.logger.Errorw("error in getting ciPipeline by id", "err", err, "id", impactedCiPipelineId) + return nil, err + } + respDto := &CiBulkActionResponseDto{ + PipelineName: ciPipeline.Name, + } + if !dryRun { + deleteReq := &bean2.CiPatchRequest{ + Action: 2, //delete + CiPipeline: ciPipeline, + AppId: ciPipeline.ParentAppId, + } + _, err = impl.pipelineBuilder.DeleteCiPipeline(deleteReq) + if err != nil { + impl.logger.Errorw("error in deleting ci pipeline", "err", err, "pipelineId", impactedCiPipelineId) + respDto.DeletionResult = fmt.Sprintf("Not able to delete pipeline, %v", err) + } else { + respDto.DeletionResult = "Pipeline deleted successfully." + } + } + ciPipelineRespDtos = append(ciPipelineRespDtos, respDto) + } + + for _, impactedAppWfId := range impactedAppWfIds { + respDto := &WfBulkActionResponseDto{ + WorkflowId: impactedAppWfId, + } + if !dryRun { + err := impl.appWorkflowService.DeleteAppWorkflow(impactedAppWfId, userId) + if err != nil { + impl.logger.Errorw("error in deleting appWf", "err", err, "appWfId", impactedAppWfId) + respDto.DeletionResult = fmt.Sprintf("Not able to delete workflow, %v", err) + } else { + respDto.DeletionResult = "Workflow deleted successfully." + } + } + wfRespDtos = append(wfRespDtos, respDto) + } + + } + respDto := &PipelineAndWfBulkActionResponseDto{ + CdPipelinesRespDtos: cdPipelineRespDtos, + CiPipelineRespDtos: ciPipelineRespDtos, + AppWfRespDtos: wfRespDtos, + } + return respDto, nil + +} diff --git a/pkg/bulkAction/bean.go b/pkg/bulkAction/bean.go new file mode 100644 index 0000000000..732e04b976 --- /dev/null +++ b/pkg/bulkAction/bean.go @@ -0,0 +1,132 @@ +package bulkAction + +type NameIncludesExcludes struct { + Names []string `json:"names"` +} + +type DeploymentTemplateSpec struct { + PatchJson string `json:"patchJson"` +} +type DeploymentTemplateTask struct { + Spec *DeploymentTemplateSpec `json:"spec"` +} +type CmAndSecretSpec struct { + Names []string `json:"names"` + PatchJson string `json:"patchJson"` +} +type CmAndSecretTask struct { + Spec *CmAndSecretSpec `json:"spec"` +} +type BulkUpdatePayload struct { + Includes *NameIncludesExcludes `json:"includes"` + Excludes *NameIncludesExcludes `json:"excludes"` + EnvIds []int `json:"envIds"` + Global bool `json:"global"` + DeploymentTemplate *DeploymentTemplateTask `json:"deploymentTemplate"` + ConfigMap *CmAndSecretTask `json:"configMap"` + Secret *CmAndSecretTask `json:"secret"` +} +type BulkUpdateScript struct { + ApiVersion string `json:"apiVersion" validate:"required"` + Kind string `json:"kind" validate:"required"` + Spec *BulkUpdatePayload `json:"spec" validate:"required"` +} +type BulkUpdateSeeExampleResponse struct { + Operation string `json:"operation"` + Script *BulkUpdateScript `json:"script" validate:"required"` + ReadMe string `json:"readme"` +} +type ImpactedObjectsResponse struct { + DeploymentTemplate []*DeploymentTemplateImpactedObjectsResponseForOneApp `json:"deploymentTemplate"` + ConfigMap []*CmAndSecretImpactedObjectsResponseForOneApp `json:"configMap"` + Secret []*CmAndSecretImpactedObjectsResponseForOneApp `json:"secret"` +} +type DeploymentTemplateImpactedObjectsResponseForOneApp struct { + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvId int `json:"envId"` +} +type CmAndSecretImpactedObjectsResponseForOneApp struct { + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvId int `json:"envId"` + Names []string `json:"names"` +} +type DeploymentTemplateBulkUpdateResponseForOneApp struct { + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvId int `json:"envId"` + Message string `json:"message"` +} +type CmAndSecretBulkUpdateResponseForOneApp struct { + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvId int `json:"envId"` + Names []string `json:"names"` + Message string `json:"message"` +} +type BulkUpdateResponse struct { + DeploymentTemplate *DeploymentTemplateBulkUpdateResponse `json:"deploymentTemplate"` + ConfigMap *CmAndSecretBulkUpdateResponse `json:"configMap"` + Secret *CmAndSecretBulkUpdateResponse `json:"secret"` +} +type DeploymentTemplateBulkUpdateResponse struct { + Message []string `json:"message"` + Failure []*DeploymentTemplateBulkUpdateResponseForOneApp `json:"failure"` + Successful []*DeploymentTemplateBulkUpdateResponseForOneApp `json:"successful"` +} +type CmAndSecretBulkUpdateResponse struct { + Message []string `json:"message"` + Failure []*CmAndSecretBulkUpdateResponseForOneApp `json:"failure"` + Successful []*CmAndSecretBulkUpdateResponseForOneApp `json:"successful"` +} + +type BulkApplicationForEnvironmentPayload struct { + AppIdIncludes []int `json:"appIdIncludes,omitempty"` + AppIdExcludes []int `json:"appIdExcludes,omitempty"` + EnvId int `json:"envId"` + UserId int32 `json:"-"` +} + +type BulkApplicationForEnvironmentResponse struct { + BulkApplicationForEnvironmentPayload + Response map[string]map[string]bool `json:"response"` +} + +type CdBulkAction int + +const ( + CD_BULK_DELETE CdBulkAction = iota +) + +type CdBulkActionRequestDto struct { + Action CdBulkAction `json:"action"` + EnvIds []int `json:"envIds"` + AppIds []int `json:"appIds"` + ProjectIds []int `json:"projectIds"` + DeleteWfAndCiPipeline bool `json:"deleteWfAndCiPipeline"` + ForceDelete bool `json:"forceDelete"` + UserId int32 `json:"-"` +} + +type CdBulkActionResponseDto struct { + PipelineName string `json:"pipelineName"` + AppName string `json:"appName"` + EnvironmentName string `json:"environmentName"` + DeletionResult string `json:"deletionResult,omitempty"` +} + +type CiBulkActionResponseDto struct { + PipelineName string `json:"pipelineName"` + DeletionResult string `json:"deletionResult,omitempty"` +} +type WfBulkActionResponseDto struct { + WorkflowId int `json:"workflowId"` + DeletionResult string `json:"deletionResult,omitempty"` +} + +type PipelineAndWfBulkActionResponseDto struct { + CdPipelinesRespDtos []*CdBulkActionResponseDto `json:"cdPipelines"` + CiPipelineRespDtos []*CiBulkActionResponseDto `json:"ciPipelines"` + AppWfRespDtos []*WfBulkActionResponseDto `json:"appWorkflows"` +} diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 1488e5b829..f1163aa628 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -85,6 +85,7 @@ type PipelineBuilder interface { CreateCdPipelines(cdPipelines *bean.CdPipelines, ctx context.Context) (*bean.CdPipelines, error) GetApp(appId int) (application *bean.CreateAppDTO, err error) PatchCdPipelines(cdPipelines *bean.CDPatchRequest, ctx context.Context) (*bean.CdPipelines, error) + DeleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, forceDelete bool) (err error) GetCdPipelinesForApp(appId int) (cdPipelines *bean.CdPipelines, err error) GetCdPipelinesForAppAndEnv(appId int, envId int) (cdPipelines *bean.CdPipelines, err error) /* CreateCdPipelines(cdPipelines bean.CdPipelines) (*bean.CdPipelines, error)*/ @@ -108,9 +109,7 @@ type PipelineBuilder interface { FindAllMatchesByAppName(appName string) ([]*AppBean, error) GetEnvironmentByCdPipelineId(pipelineId int) (int, error) PatchRegexCiPipeline(request *bean.CiRegexPatchRequest) (err error) - - GetBulkActionImpactedPipelines(dto *bean.CdBulkActionRequestDto) ([]*pipelineConfig.Pipeline, error) - PerformBulkActionOnCdPipelines(dto *bean.CdBulkActionRequestDto, impactedPipelines []*pipelineConfig.Pipeline, ctx context.Context, dryRun bool) ([]*bean.CdBulkActionResponseDto, error) + DeleteCiPipeline(request *bean.CiPatchRequest) (*bean.CiPipeline, error) } type PipelineBuilderImpl struct { @@ -928,7 +927,7 @@ func (impl PipelineBuilderImpl) PatchCiPipeline(request *bean.CiPatchRequest) (c case bean.UPDATE_SOURCE: return impl.patchCiPipelineUpdateSource(ciConfig, request.CiPipeline) case bean.DELETE: - pipeline, err := impl.deletePipeline(request) + pipeline, err := impl.DeleteCiPipeline(request) if err != nil { return nil, err } @@ -991,10 +990,10 @@ func (impl PipelineBuilderImpl) PatchRegexCiPipeline(request *bean.CiRegexPatchR } return nil } -func (impl PipelineBuilderImpl) deletePipeline(request *bean.CiPatchRequest) (*bean.CiPipeline, error) { - +func (impl PipelineBuilderImpl) DeleteCiPipeline(request *bean.CiPatchRequest) (*bean.CiPipeline, error) { + ciPipelineId := request.CiPipeline.Id //wf validation - workflowMapping, err := impl.appWorkflowRepository.FindWFCDMappingByCIPipelineId(request.CiPipeline.Id) + workflowMapping, err := impl.appWorkflowRepository.FindWFCDMappingByCIPipelineId(ciPipelineId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching workflow mapping for ci validation", "err", err) return nil, err @@ -1006,12 +1005,13 @@ func (impl PipelineBuilderImpl) deletePipeline(request *bean.CiPatchRequest) (*b UserMessage: fmt.Sprintf("cd pipeline exists for this CI")} } - pipeline, err := impl.ciPipelineRepository.FindById(request.CiPipeline.Id) + pipeline, err := impl.ciPipelineRepository.FindById(ciPipelineId) if err != nil { - impl.logger.Errorw("pipeline fetch err", "id", request.CiPipeline.Id, "err", err) + impl.logger.Errorw("pipeline fetch err", "id", ciPipelineId, "err", err) } - if pipeline.AppId != request.AppId { - return nil, fmt.Errorf("invalid appid: %d pipelineId: %d mapping", request.AppId, request.CiPipeline.Id) + appId := request.AppId + if pipeline.AppId != appId { + return nil, fmt.Errorf("invalid appid: %d pipelineId: %d mapping", appId, ciPipelineId) } dbConnection := impl.pipelineRepository.GetConnection() @@ -1047,7 +1047,7 @@ func (impl PipelineBuilderImpl) deletePipeline(request *bean.CiPatchRequest) (*b } if request.CiPipeline.PostBuildStage != nil && request.CiPipeline.PostBuildStage.Id > 0 { //deleting post stage - err = impl.pipelineStageService.DeleteCiStage(request.CiPipeline.PreBuildStage, request.UserId, tx) + err = impl.pipelineStageService.DeleteCiStage(request.CiPipeline.PostBuildStage, request.UserId, tx) if err != nil { impl.logger.Errorw("error in deleting post stage", "err", err, "postBuildStage", request.CiPipeline.PostBuildStage) return nil, err @@ -1233,14 +1233,14 @@ func (impl PipelineBuilderImpl) PatchCdPipelines(cdPipelines *bean.CDPatchReques impl.logger.Errorw("error in getting cd pipeline by id", "err", err, "id", cdPipelines.Pipeline.Id) return pipelineRequest, err } - err = impl.deleteCdPipeline(pipeline, ctx, cdPipelines.ForceDelete) + err = impl.DeleteCdPipeline(pipeline, ctx, cdPipelines.ForceDelete) return pipelineRequest, err default: return nil, &util.ApiError{Code: "404", HttpStatusCode: 404, UserMessage: "operation not supported"} } } -func (impl PipelineBuilderImpl) deleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, forceDelete bool) (err error) { +func (impl PipelineBuilderImpl) DeleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, forceDelete bool) (err error) { //getting children CD pipeline details appWorkflowMapping, err := impl.appWorkflowRepository.FindWFCDMappingByParentCDPipelineId(pipeline.Id) if err != nil && err != pg.ErrNoRows { @@ -2516,77 +2516,3 @@ func (impl PipelineBuilderImpl) updateGitRepoUrlInCharts(appId int, chartGitAttr } return nil } - -func (impl PipelineBuilderImpl) PerformBulkActionOnCdPipelines(dto *bean.CdBulkActionRequestDto, impactedPipelines []*pipelineConfig.Pipeline, ctx context.Context, dryRun bool) ([]*bean.CdBulkActionResponseDto, error) { - switch dto.Action { - case bean.CD_BULK_DELETE: - bulkDeleteResp := impl.BulkDeleteCdPipelines(impactedPipelines, ctx, dryRun, dto.ForceDelete) - return bulkDeleteResp, nil - default: - return nil, &util.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "this action is not supported"} - } -} - -func (impl PipelineBuilderImpl) BulkDeleteCdPipelines(impactedPipelines []*pipelineConfig.Pipeline, ctx context.Context, dryRun, forceDelete bool) []*bean.CdBulkActionResponseDto { - var respDtos []*bean.CdBulkActionResponseDto - for _, pipeline := range impactedPipelines { - respDto := &bean.CdBulkActionResponseDto{ - PipelineName: pipeline.Name, - AppName: pipeline.App.AppName, - EnvironmentName: pipeline.Environment.Name, - } - if !dryRun { - err := impl.deleteCdPipeline(pipeline, ctx, forceDelete) - if err != nil { - impl.logger.Errorw("error in deleting cd pipeline", "err", err, "pipelineId", pipeline.Id) - respDto.DeletionResult = fmt.Sprintf("Not able to delete pipeline, %v", err) - } else { - respDto.DeletionResult = "Pipeline deleted successfully." - } - } - respDtos = append(respDtos, respDto) - } - return respDtos - -} - -func (impl PipelineBuilderImpl) GetBulkActionImpactedPipelines(dto *bean.CdBulkActionRequestDto) ([]*pipelineConfig.Pipeline, error) { - if len(dto.EnvIds) == 0 || (len(dto.AppIds) == 0 && len(dto.ProjectIds) == 0) { - //invalid payload, envIds are must and either of appIds or projectIds are must - return nil, &util.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "invalid payload, can not get pipelines for this filter"} - } - var pipelineIdsByAppLevel []int - var pipelineIdsByProjectLevel []int - var err error - if len(dto.AppIds) > 0 && len(dto.EnvIds) > 0 { - //getting pipeline IDs for app level deletion request - pipelineIdsByAppLevel, err = impl.pipelineRepository.FindIdsByAppIdsAndEnvironmentIds(dto.AppIds, dto.EnvIds) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting cd pipelines by appIds and envIds", "err", err) - return nil, err - } - } - if len(dto.ProjectIds) > 0 && len(dto.EnvIds) > 0 { - //getting pipeline IDs for project level deletion request - pipelineIdsByProjectLevel, err = impl.pipelineRepository.FindIdsByProjectIdsAndEnvironmentIds(dto.ProjectIds, dto.EnvIds) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting cd pipelines by projectIds and envIds", "err", err) - return nil, err - } - } - var pipelineIdsMerged []int - //it might be possible that pipelineIdsByAppLevel & pipelineIdsByProjectLevel have some same values - //we are still appending them to save operation cost of checking same ids as we will get pipelines from - //in clause which gives correct results even if some values are repeating - pipelineIdsMerged = append(pipelineIdsMerged, pipelineIdsByAppLevel...) - pipelineIdsMerged = append(pipelineIdsMerged, pipelineIdsByProjectLevel...) - var pipelines []*pipelineConfig.Pipeline - if len(pipelineIdsMerged) > 0 { - pipelines, err = impl.pipelineRepository.FindByIdsIn(pipelineIdsMerged) - if err != nil { - impl.logger.Errorw("error in getting cd pipelines by ids", "err", err, "ids", pipelineIdsMerged) - return nil, err - } - } - return pipelines, nil -} diff --git a/specs/bulk-env-delete.yaml b/specs/bulk-env-delete.yaml index a3bc0da0cd..93bf5e4668 100644 --- a/specs/bulk-env-delete.yaml +++ b/specs/bulk-env-delete.yaml @@ -3,7 +3,7 @@ info: version: 1.0.0 title: Devtron Labs paths: - /cd-pipeline/bulk-action: + /orchestrtor/batch/v1beta1/cd-pipeline: post: description: perform bulk actions on cd pipelines parameters: diff --git a/tests/pipeline/BulkUpdateService_test.go b/tests/pipeline/BulkUpdateService_test.go index 00f25932d3..73a29e4bd9 100644 --- a/tests/pipeline/BulkUpdateService_test.go +++ b/tests/pipeline/BulkUpdateService_test.go @@ -4,7 +4,7 @@ import ( "encoding/csv" "encoding/json" "fmt" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/bulkAction" "io" "log" "os" @@ -18,7 +18,7 @@ func TestBulkUpdate(t *testing.T) { type test struct { ApiVersion string Kind string - Payload *pipeline.BulkUpdatePayload + Payload *bulkAction.BulkUpdatePayload deploymentTemplateWant string configMapWant string secretWant string @@ -49,28 +49,28 @@ func TestBulkUpdate(t *testing.T) { } namesIncludes := strings.Fields(record[2]) namesExcludes := strings.Fields(record[3]) - includes := &pipeline.NameIncludesExcludes{Names: namesIncludes} - excludes := &pipeline.NameIncludesExcludes{Names: namesExcludes} - deploymentTemplateSpec := &pipeline.DeploymentTemplateSpec{ + includes := &bulkAction.NameIncludesExcludes{Names: namesIncludes} + excludes := &bulkAction.NameIncludesExcludes{Names: namesExcludes} + deploymentTemplateSpec := &bulkAction.DeploymentTemplateSpec{ PatchJson: record[6]} - deploymentTemplateTask := &pipeline.DeploymentTemplateTask{ + deploymentTemplateTask := &bulkAction.DeploymentTemplateTask{ Spec: deploymentTemplateSpec, } - configMapSpec := &pipeline.CmAndSecretSpec{ + configMapSpec := &bulkAction.CmAndSecretSpec{ Names: strings.Fields(record[7]), PatchJson: record[8], } - configMapTask := &pipeline.CmAndSecretTask{ + configMapTask := &bulkAction.CmAndSecretTask{ Spec: configMapSpec, } - secretSpec := &pipeline.CmAndSecretSpec{ + secretSpec := &bulkAction.CmAndSecretSpec{ Names: strings.Fields(record[9]), PatchJson: record[10], } - secretTask := &pipeline.CmAndSecretTask{ + secretTask := &bulkAction.CmAndSecretTask{ Spec: secretSpec, } - payload := &pipeline.BulkUpdatePayload{ + payload := &bulkAction.BulkUpdatePayload{ Includes: includes, Excludes: excludes, EnvIds: envId, diff --git a/tests/pipeline/ChartService_test.go b/tests/pipeline/ChartService_test.go index cf875fc61f..8fa0b7134d 100644 --- a/tests/pipeline/ChartService_test.go +++ b/tests/pipeline/ChartService_test.go @@ -6,8 +6,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/bulkAction" "github.com/devtron-labs/devtron/pkg/chart" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/sql" jsonpatch "github.com/evanphx/json-patch" "io" @@ -18,7 +18,7 @@ import ( "testing" ) -var bulkUpdateService *pipeline.BulkUpdateServiceImpl +var bulkUpdateService *bulkAction.BulkUpdateServiceImpl var bulkUpdateRepository bulkUpdate.BulkUpdateRepositoryImpl func setup() { @@ -26,7 +26,7 @@ func setup() { logger, _ := util.NewSugardLogger() dbConnection, _ := sql.NewDbConnection(config, logger) bulkUpdateRepository := bulkUpdate.NewBulkUpdateRepository(dbConnection, logger) - bulkUpdateService = pipeline.NewBulkUpdateServiceImpl(bulkUpdateRepository, nil, nil, nil, nil, "", + bulkUpdateService = bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepository, nil, nil, nil, nil, "", chart.DefaultChart(""), util.MergeUtil{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) } @@ -36,7 +36,7 @@ func TestBulkUpdateDeploymentTemplate(t *testing.T) { type test struct { ApiVersion string Kind string - Payload *pipeline.BulkUpdatePayload + Payload *bulkAction.BulkUpdatePayload want string } TestsCsvFile, err := os.Open("ChartService_test.csv") @@ -65,14 +65,14 @@ func TestBulkUpdateDeploymentTemplate(t *testing.T) { } namesIncludes := strings.Fields(record[2]) namesExcludes := strings.Fields(record[3]) - includes := &pipeline.NameIncludesExcludes{Names: namesIncludes} - excludes := &pipeline.NameIncludesExcludes{Names: namesExcludes} - spec := &pipeline.DeploymentTemplateSpec{ + includes := &bulkAction.NameIncludesExcludes{Names: namesIncludes} + excludes := &bulkAction.NameIncludesExcludes{Names: namesExcludes} + spec := &bulkAction.DeploymentTemplateSpec{ PatchJson: record[6]} - task := &pipeline.DeploymentTemplateTask{ + task := &bulkAction.DeploymentTemplateTask{ Spec: spec, } - payload := &pipeline.BulkUpdatePayload{ + payload := &bulkAction.BulkUpdatePayload{ Includes: includes, Excludes: excludes, EnvIds: envId, diff --git a/wire_gen.go b/wire_gen.go index 2563a374d1..4702072e74 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate wire -//+build !wireinject +//go:generate go run github.com/google/wire/cmd/wire +//go:build !wireinject +// +build !wireinject package main @@ -74,6 +75,7 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/values/service" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" "github.com/devtron-labs/devtron/pkg/attributes" + "github.com/devtron-labs/devtron/pkg/bulkAction" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chartRepo" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -255,7 +257,7 @@ func InitializeApp() (*App, error) { tokenCache := util2.NewTokenCache(sugaredLogger, acdAuthConfig, userAuthServiceImpl) syncedEnforcer := casbin.Create() enforcerImpl := casbin.NewEnforcerImpl(syncedEnforcer, sessionManager, sugaredLogger) - appRepositoryImpl := app.NewAppRepositoryImpl(db) + appRepositoryImpl := app.NewAppRepositoryImpl(db, sugaredLogger) enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl) appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(sugaredLogger) appListingRepositoryImpl := repository.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder) @@ -570,7 +572,7 @@ func InitializeApp() (*App, error) { telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - bulkUpdateServiceImpl := pipeline.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl) + bulkUpdateServiceImpl := bulkAction.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, defaultChart, utilMergeUtil, repositoryServiceClientImpl, chartRefRepositoryImpl, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, httpClient, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, workflowDagExecutorImpl, cdWorkflowRepositoryImpl, pipelineBuilderImpl, helmAppServiceImpl, enforcerUtilImpl, enforcerUtilHelmImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, dbMigrationServiceImpl, applicationServiceClientImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, gitSensorClientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, argoUserServiceImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := git.NewWebhookSecretValidatorImpl(sugaredLogger) From 482776f55f5bde33854061937789087e8cb66d62 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Nov 2022 18:17:52 +0530 Subject: [PATCH 04/13] fixed pointer error in appRepository --- internal/sql/repository/app/AppRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 0878e59552..4f4033dc87 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -253,7 +253,7 @@ func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string) ([]*App, e func (repo AppRepositoryImpl) FindIdsByTeamIds(teamIds []int) ([]int, error) { var ids []int query := "select id from app where team_id in (?) and active = ?;" - _, err := repo.dbConnection.Query(ids, query, pg.In(teamIds), true) + _, err := repo.dbConnection.Query(&ids, query, pg.In(teamIds), true) if err != nil { repo.logger.Errorw("error in getting appIds by teamIds", "err", err, "teamIds", teamIds) return nil, err From ca28825997bd2cdb15c45a06bdc9b30d6cb92301 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Nov 2022 18:55:22 +0530 Subject: [PATCH 05/13] fixed query for fetching workflows --- internal/sql/repository/app/AppRepository.go | 2 +- .../sql/repository/appWorkflow/AppWorkflowRepository.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 0878e59552..4f4033dc87 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -253,7 +253,7 @@ func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string) ([]*App, e func (repo AppRepositoryImpl) FindIdsByTeamIds(teamIds []int) ([]int, error) { var ids []int query := "select id from app where team_id in (?) and active = ?;" - _, err := repo.dbConnection.Query(ids, query, pg.In(teamIds), true) + _, err := repo.dbConnection.Query(&ids, query, pg.In(teamIds), true) if err != nil { repo.logger.Errorw("error in getting appIds by teamIds", "err", err, "teamIds", teamIds) return nil, err diff --git a/internal/sql/repository/appWorkflow/AppWorkflowRepository.go b/internal/sql/repository/appWorkflow/AppWorkflowRepository.go index 304df66cc3..301ed1a3f3 100644 --- a/internal/sql/repository/appWorkflow/AppWorkflowRepository.go +++ b/internal/sql/repository/appWorkflow/AppWorkflowRepository.go @@ -324,10 +324,11 @@ func (impl AppWorkflowRepositoryImpl) DeleteAppWorkflowMappingsByCdPipelineId(pi func (impl AppWorkflowRepositoryImpl) FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(envIds []int, appIds []int) ([]*AppWorkflowMapping, error) { var models []*AppWorkflowMapping - query := `select * from app_workflow_mapping - where app_id in (?) and app_workflow_id not in + query := `select * from app_workflow_mapping awm inner join app_workflow aw on aw.id=awm.app_workflow_id + where aw.app_id in (?) and awm.app_workflow_id not in (select app_workflow_id from app_workflow_mapping awm inner join pipeline p on p.id=awm.component_id - and p.environment_id not in (?) and p.app_id in (?) and p.deleted = ? and awm.active = ?); ` + and awm.type = ? and p.environment_id not in (?) and p.app_id in (?) and p.deleted = ? and awm.active = ?) + ; ` _, err := impl.dbConnection.Query(&models, query, pg.In(appIds), CDPIPELINE, pg.In(envIds), pg.In(appIds), false, true) if err != nil { impl.Logger.Errorw("error, FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly", "err", err) From b688ba005ca8b1196c2a4b803b96eec3e5b9de6d Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 1 Nov 2022 19:33:40 +0530 Subject: [PATCH 06/13] fix --- .../sql/repository/appWorkflow/AppWorkflowRepository.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/sql/repository/appWorkflow/AppWorkflowRepository.go b/internal/sql/repository/appWorkflow/AppWorkflowRepository.go index 301ed1a3f3..f070ee148c 100644 --- a/internal/sql/repository/appWorkflow/AppWorkflowRepository.go +++ b/internal/sql/repository/appWorkflow/AppWorkflowRepository.go @@ -325,11 +325,10 @@ func (impl AppWorkflowRepositoryImpl) DeleteAppWorkflowMappingsByCdPipelineId(pi func (impl AppWorkflowRepositoryImpl) FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly(envIds []int, appIds []int) ([]*AppWorkflowMapping, error) { var models []*AppWorkflowMapping query := `select * from app_workflow_mapping awm inner join app_workflow aw on aw.id=awm.app_workflow_id - where aw.app_id in (?) and awm.app_workflow_id not in + where aw.app_id in (?) and awm.active = ? and awm.app_workflow_id not in (select app_workflow_id from app_workflow_mapping awm inner join pipeline p on p.id=awm.component_id - and awm.type = ? and p.environment_id not in (?) and p.app_id in (?) and p.deleted = ? and awm.active = ?) - ; ` - _, err := impl.dbConnection.Query(&models, query, pg.In(appIds), CDPIPELINE, pg.In(envIds), pg.In(appIds), false, true) + and awm.type = ? and p.environment_id not in (?) and p.app_id in (?) and p.deleted = ? and awm.active = ?); ` + _, err := impl.dbConnection.Query(&models, query, pg.In(appIds), true, CDPIPELINE, pg.In(envIds), pg.In(appIds), false, true) if err != nil { impl.Logger.Errorw("error, FindAllWfsHavingCdPipelinesFromSpecificEnvsOnly", "err", err) return nil, err From 4702d41eff9680c87cf51142395a034317b97a3a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Nov 2022 12:29:57 +0530 Subject: [PATCH 07/13] fix --- pkg/bulkAction/BulkUpdateService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 416370c108..4b4b644a21 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -1342,7 +1342,7 @@ func (impl BulkUpdateServiceImpl) PerformBulkActionOnCdPipelines(dto *CdBulkActi ctx context.Context, dryRun bool, impactedAppWfIds []int, impactedCiPipelineIds []int) (*PipelineAndWfBulkActionResponseDto, error) { switch dto.Action { case CD_BULK_DELETE: - bulkDeleteResp, err := impl.PerformBulkDeleteActionOnCdPipelines(impactedPipelines, ctx, dryRun, dto.ForceDelete, false, impactedAppWfIds, impactedCiPipelineIds, dto.UserId) + bulkDeleteResp, err := impl.PerformBulkDeleteActionOnCdPipelines(impactedPipelines, ctx, dryRun, dto.ForceDelete, dto.DeleteWfAndCiPipeline, impactedAppWfIds, impactedCiPipelineIds, dto.UserId) if err != nil { impl.logger.Errorw("error in cd pipelines bulk deletion") } From 40249ffd4af66a9c4dfccccbc0a003d9d9aeb77a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Nov 2022 15:32:41 +0530 Subject: [PATCH 08/13] added check for preventing count of same wf ids --- pkg/bulkAction/BulkUpdateService.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 4b4b644a21..9f25527116 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -1316,14 +1316,17 @@ func (impl BulkUpdateServiceImpl) GetBulkActionImpactedPipelinesAndWfs(dto *CdBu impl.logger.Errorw("error in getting wfs having cd pipelines from specific env only", "err", err) return nil, nil, nil, err } - + impactedWfIdsMap := make(map[int]bool) for _, appWf := range appWfs { if appWf.Type == appWorkflow.CDPIPELINE { impactedPipelineIds = append(impactedPipelineIds, appWf.ComponentId) } else if appWf.Type == appWorkflow.CIPIPELINE { impactedCiPipelineIds = append(impactedCiPipelineIds, appWf.ComponentId) } - impactedWfIds = append(impactedWfIds, appWf.AppWorkflowId) + if _, ok := impactedWfIdsMap[appWf.AppWorkflowId]; !ok { + impactedWfIds = append(impactedWfIds, appWf.AppWorkflowId) + impactedWfIdsMap[appWf.AppWorkflowId] = true + } } } } From b107176327071ca9bebb0927f848014f1062f9d1 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 2 Nov 2022 16:14:39 +0530 Subject: [PATCH 09/13] fixed appId issue in ci deletion --- pkg/bean/app.go | 1 + pkg/bulkAction/BulkUpdateService.go | 2 +- pkg/pipeline/PipelineBuilder.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index f3bc4e9c0e..5526f0137b 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -87,6 +87,7 @@ type CiPipeline struct { IsExternal bool `json:"isExternal"` ParentCiPipeline int `json:"parentCiPipeline"` ParentAppId int `json:"parentAppId"` + AppId int `json:"appId"` ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable diff --git a/pkg/bulkAction/BulkUpdateService.go b/pkg/bulkAction/BulkUpdateService.go index 9f25527116..48be088e6b 100644 --- a/pkg/bulkAction/BulkUpdateService.go +++ b/pkg/bulkAction/BulkUpdateService.go @@ -1401,7 +1401,7 @@ func (impl BulkUpdateServiceImpl) PerformBulkDeleteActionOnCdPipelines(impactedP deleteReq := &bean2.CiPatchRequest{ Action: 2, //delete CiPipeline: ciPipeline, - AppId: ciPipeline.ParentAppId, + AppId: ciPipeline.AppId, } _, err = impl.pipelineBuilder.DeleteCiPipeline(deleteReq) if err != nil { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index f1163aa628..14aab4c1ea 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -2416,6 +2416,7 @@ func (impl PipelineBuilderImpl) GetCiPipelineById(pipelineId int) (ciPipeline *b DockerArgs: dockerArgs, IsManual: pipeline.IsManual, IsExternal: pipeline.IsExternal, + AppId: pipeline.AppId, ParentCiPipeline: pipeline.ParentCiPipeline, ParentAppId: parentCiPipeline.AppId, ExternalCiConfig: externalCiConfig, From 72349ef8b845fa85e2486d2f17bf0415fd4a9b55 Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Wed, 2 Nov 2022 17:40:35 +0530 Subject: [PATCH 10/13] added open api spec for override and workflow clon --- specs/app_create_api.yaml | 93 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/specs/app_create_api.yaml b/specs/app_create_api.yaml index 0c2aa4ae22..8dbd8333a5 100644 --- a/specs/app_create_api.yaml +++ b/specs/app_create_api.yaml @@ -38,6 +38,41 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /orchestrator/core/v1beta1/application/workflow/{appId}: + get: + description: Get all details of an app by appId. These details include - metadata(appName, projectNamr, labels), gitMaterials, docker config, templates, workflows, configMaps, secrets, environment override configurations. + operationId: GetAppWorkflowsAndOverrides + parameters: + - name: appId + in: path + required: true + schema: + type: integer + responses: + '200': + description: Successfully return all details of the app. + content: + application/json: + schema: + $ref: '#/components/schemas/AppWorkflowDto' + '400': + description: Bad Request. Input Validation error/wrong request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Unauthorized User + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /orchestrator/core/v1beta1/application: post: description: Creates a new app for the configurations provided. The input json object is the same we get in response of GET method for fetching all details of an app @@ -74,6 +109,42 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /orchestrator/core/v1beta1/application/workflow: + post: + description: Creates a new app for the configurations provided. The input json object is the same we get in response of GET method for fetching all details of an app + operationId: CreateAppWorkflow + requestBody: + description: A JSON object containing the app configuration + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AppWorkflowDto' + responses: + '200': + description: Successfully return a message stating the operation is successful. + content: + application/json: + schema: + type: string + '400': + description: Bad Request. Validation error/wrong request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Unauthorized User + content: + application/json: + schema: + $ref: '#/components/schemas/Error' components: schemas: @@ -582,4 +653,24 @@ components: description: Error code message: type: string - description: Error message \ No newline at end of file + description: Error message + + AppWorkflowDto: + type: object + properties: + appId: + type: integer + workflows: + type: array + items: + $ref: '#/components/schemas/AppWorkflow' + environmentOverrides: + type: array + items: + type: object + properties: + Name: + type: string + description: Name of environment + Values: + $ref: '#/components/schemas/EnvironmentOverride' \ No newline at end of file From 0c628d29dc07a95dad94c582cfc04ef01d880794 Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Thu, 3 Nov 2022 17:01:52 +0530 Subject: [PATCH 11/13] modification in app workflow and overrides clone api --- api/appbean/AppDetail.go | 1 + api/restHandler/CoreAppRestHandler.go | 91 +++++++++++++++++++++++---- api/router/CoreAppRouter.go | 1 + pkg/app/AppCrudOperationService.go | 18 ++++++ specs/app_create_api.yaml | 2 + wire_gen.go | 5 +- 6 files changed, 104 insertions(+), 14 deletions(-) diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index e04a48cff6..daaabc256b 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -18,6 +18,7 @@ type AppDetail struct { type AppWorkflowCloneDto struct { AppId int `json:"appId"` + AppName string `json:"appName"` AppWorkflows []*AppWorkflow `json:"workflows"` EnvironmentOverrides map[string]*EnvironmentOverride `json:"environmentOverride"` } diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 9651aa8fc7..3d140bcc4e 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -59,6 +59,7 @@ import ( const ( APP_DELETE_FAILED_RESP = "App deletion failed, please try deleting from Devtron UI" APP_CREATE_SUCCESSFUL_RESP = "App created successfully." + APP_WORKFLOW_CREATE_SUCCESSFUL_RESP = "App workflow created successfully." ) type CoreAppRestHandler interface { @@ -66,6 +67,7 @@ type CoreAppRestHandler interface { CreateApp(w http.ResponseWriter, r *http.Request) CreateAppWorkflow(w http.ResponseWriter, r *http.Request) GetAppWorkflow(w http.ResponseWriter, r *http.Request) + GetAppWorkflowAndOverridesSample(w http.ResponseWriter, r *http.Request) } type CoreAppRestHandlerImpl struct { @@ -2009,8 +2011,14 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + app, err := handler.appCrudOperationService.GetAppMetaInfoByAppName(createAppRequest.AppName) + if err != nil { + handler.logger.Errorw("service err, GetAppMetaInfo in GetAppAllDetail", "err", err, "appName", createAppRequest.AppName) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } - object := handler.enforcerUtil.GetAppRBACNameByAppId(createAppRequest.AppId) + object := fmt.Sprintf("%s/%s", app.ProjectName, app.AppName) // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, object); !ok { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) @@ -2018,19 +2026,17 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r } //rbac ends - handler.logger.Infow("creating app v2", "createAppRequest", createAppRequest) + handler.logger.Infow("creating app workflow created ", "createAppRequest", createAppRequest) var errResp *multierror.Error var statusCode int - app, err := handler.pipelineBuilder.GetApp(createAppRequest.AppId) - if err != nil { - common.WriteJsonResp(w, errResp, nil, statusCode) - return - } - appName := app.AppName //creating workflow starts if createAppRequest.AppWorkflows != nil { - err, statusCode = handler.createWorkflows(ctx, createAppRequest.AppId, userId, createAppRequest.AppWorkflows, token, appName) + if len(createAppRequest.AppWorkflows) != 1 { + common.WriteJsonResp(w, err, "please provide only one workflow at one time", http.StatusBadRequest) + return + } + err, statusCode = handler.createWorkflows(ctx, createAppRequest.AppId, userId, createAppRequest.AppWorkflows, token, app.AppName) if err != nil { common.WriteJsonResp(w, errResp, nil, statusCode) return @@ -2039,7 +2045,7 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r //creating workflow ends //creating environment override starts - if createAppRequest.EnvironmentOverrides != nil { + if createAppRequest.EnvironmentOverrides != nil && len(createAppRequest.EnvironmentOverrides) > 0 { err, statusCode = handler.createEnvOverrides(ctx, createAppRequest.AppId, userId, createAppRequest.EnvironmentOverrides, token) if err != nil { common.WriteJsonResp(w, errResp, nil, statusCode) @@ -2048,7 +2054,7 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r } //creating environment override ends - common.WriteJsonResp(w, nil, APP_CREATE_SUCCESSFUL_RESP, http.StatusOK) + common.WriteJsonResp(w, nil, APP_WORKFLOW_CREATE_SUCCESSFUL_RESP, http.StatusOK) } func (handler CoreAppRestHandlerImpl) GetAppWorkflow(w http.ResponseWriter, r *http.Request) { @@ -2094,3 +2100,66 @@ func (handler CoreAppRestHandlerImpl) GetAppWorkflow(w http.ResponseWriter, r *h common.WriteJsonResp(w, nil, appDetail, http.StatusOK) } + +func (handler CoreAppRestHandlerImpl) GetAppWorkflowAndOverridesSample(w http.ResponseWriter, r *http.Request) { + + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + + vars := mux.Vars(r) + appId, err := strconv.Atoi(vars["appId"]) + if err != nil { + handler.logger.Errorw("request err, GetAppWorkflow", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + app, err := handler.appCrudOperationService.GetAppMetaInfo(appId) + if err != nil { + handler.logger.Errorw("service err, GetAppMetaInfo in GetAppAllDetail", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + token := r.Header.Get("token") + //get/build app workflows starts + appWorkflows, err, statusCode := handler.buildAppWorkflows(appId) + if err != nil { + common.WriteJsonResp(w, err, nil, statusCode) + return + } + //get/build app workflows ends + + //get/build environment override starts + environmentOverrides, err, statusCode := handler.buildEnvironmentOverrides(appId, token) + if err != nil { + common.WriteJsonResp(w, err, nil, statusCode) + return + } + //get/build environment override ends + + //build full object for response + appDetail := &appBean.AppWorkflowCloneDto{ + AppId: appId, + AppName: app.AppName, + } + if appWorkflows != nil && len(appWorkflows) > 0 { + aw := make([]*appBean.AppWorkflow, 0) + aw = append(aw, appWorkflows[0]) + appDetail.AppWorkflows = aw + } + + if environmentOverrides != nil && len(environmentOverrides) > 0 { + eo := make(map[string]*appBean.EnvironmentOverride) + for k, v := range environmentOverrides { + eo[k] = v + break + } + appDetail.EnvironmentOverrides = eo + } + + //end + + common.WriteJsonResp(w, nil, appDetail, http.StatusOK) +} diff --git a/api/router/CoreAppRouter.go b/api/router/CoreAppRouter.go index b63eaed84a..06c19aebb1 100644 --- a/api/router/CoreAppRouter.go +++ b/api/router/CoreAppRouter.go @@ -40,4 +40,5 @@ func (router CoreAppRouterImpl) initCoreAppRouter(configRouter *mux.Router) { configRouter.Path("/v1beta1/application/{appId}").HandlerFunc(router.restHandler.GetAppAllDetail).Methods("GET") configRouter.Path("/v1beta1/application/workflow").HandlerFunc(router.restHandler.CreateAppWorkflow).Methods("POST") configRouter.Path("/v1beta1/application/workflow/{appId}").HandlerFunc(router.restHandler.GetAppWorkflow).Methods("GET") + configRouter.Path("/v1beta1/application/workflow/{appId}/sample").HandlerFunc(router.restHandler.GetAppWorkflowAndOverridesSample).Methods("GET") } diff --git a/pkg/app/AppCrudOperationService.go b/pkg/app/AppCrudOperationService.go index d176f7a35a..5dcaf3b814 100644 --- a/pkg/app/AppCrudOperationService.go +++ b/pkg/app/AppCrudOperationService.go @@ -38,6 +38,7 @@ type AppCrudOperationService interface { GetLabelsByAppId(appId int) (map[string]string, error) UpdateApp(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) UpdateProjectForApps(request *bean.UpdateProjectBulkAppsRequest) (*bean.UpdateProjectBulkAppsRequest, error) + GetAppMetaInfoByAppName(appName string) (*bean.AppMetaInfoDto, error) } type AppCrudOperationServiceImpl struct { logger *zap.SugaredLogger @@ -321,3 +322,20 @@ func (impl AppCrudOperationServiceImpl) GetLabelsByAppId(appId int) (map[string] } return labelsDto, nil } + +func (impl AppCrudOperationServiceImpl) GetAppMetaInfoByAppName(appName string) (*bean.AppMetaInfoDto, error) { + app, err := impl.appRepository.FindAppAndProjectByAppName(appName) + if err != nil { + impl.logger.Errorw("error in fetching GetAppMetaInfoByAppName", "error", err) + return nil, err + } + info := &bean.AppMetaInfoDto{ + AppId: app.Id, + AppName: app.AppName, + ProjectId: app.TeamId, + ProjectName: app.Team.Name, + CreatedOn: app.CreatedOn, + Active: app.Active, + } + return info, nil +} diff --git a/specs/app_create_api.yaml b/specs/app_create_api.yaml index 8dbd8333a5..a3b2b27ccf 100644 --- a/specs/app_create_api.yaml +++ b/specs/app_create_api.yaml @@ -660,6 +660,8 @@ components: properties: appId: type: integer + appName: + type: string workflows: type: array items: diff --git a/wire_gen.go b/wire_gen.go index 4702072e74..366c6e0baa 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,8 +1,7 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire -//go:build !wireinject -// +build !wireinject +//go:generate wire +//+build !wireinject package main From b7866a7ee1496557632233e7285868ab7befae5a Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Thu, 3 Nov 2022 17:37:02 +0530 Subject: [PATCH 12/13] fix for app workflow clone --- api/restHandler/CoreAppRestHandler.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 3d140bcc4e..959f667acd 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -57,8 +57,8 @@ import ( ) const ( - APP_DELETE_FAILED_RESP = "App deletion failed, please try deleting from Devtron UI" - APP_CREATE_SUCCESSFUL_RESP = "App created successfully." + APP_DELETE_FAILED_RESP = "App deletion failed, please try deleting from Devtron UI" + APP_CREATE_SUCCESSFUL_RESP = "App created successfully." APP_WORKFLOW_CREATE_SUCCESSFUL_RESP = "App workflow created successfully." ) @@ -2017,7 +2017,7 @@ func (handler CoreAppRestHandlerImpl) CreateAppWorkflow(w http.ResponseWriter, r common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - + createAppRequest.AppId = app.AppId object := fmt.Sprintf("%s/%s", app.ProjectName, app.AppName) // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, object); !ok { @@ -2092,9 +2092,9 @@ func (handler CoreAppRestHandlerImpl) GetAppWorkflow(w http.ResponseWriter, r *h //build full object for response appDetail := &appBean.AppWorkflowCloneDto{ - AppId: appId, - AppWorkflows: appWorkflows, - EnvironmentOverrides: environmentOverrides, + AppId: appId, + AppWorkflows: appWorkflows, + EnvironmentOverrides: environmentOverrides, } //end From 3cb0e7818d1718ed207e35ebb190da76fc0cc244 Mon Sep 17 00:00:00 2001 From: vikramdevtron Date: Thu, 3 Nov 2022 18:02:40 +0530 Subject: [PATCH 13/13] api spec for workflow and override and sample api modified --- specs/app_create_api.yaml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/specs/app_create_api.yaml b/specs/app_create_api.yaml index a3b2b27ccf..71726326d2 100644 --- a/specs/app_create_api.yaml +++ b/specs/app_create_api.yaml @@ -40,7 +40,7 @@ paths: $ref: '#/components/schemas/Error' /orchestrator/core/v1beta1/application/workflow/{appId}: get: - description: Get all details of an app by appId. These details include - metadata(appName, projectNamr, labels), gitMaterials, docker config, templates, workflows, configMaps, secrets, environment override configurations. + description: fetch all the workflows and env overrides for an app. operationId: GetAppWorkflowsAndOverrides parameters: - name: appId @@ -73,6 +73,41 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /orchestrator/core/v1beta1/application/workflow/{appId}/sample: + get: + description: get sample workflow and env overrides config of any app id for creating new workflow. + operationId: GetAppWorkflowsAndOverridesSample + parameters: + - name: appId + in: path + required: true + schema: + type: integer + responses: + '200': + description: Successfully return all details of the app. + content: + application/json: + schema: + $ref: '#/components/schemas/AppWorkflowDto' + '400': + description: Bad Request. Input Validation error/wrong request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Unauthorized User + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /orchestrator/core/v1beta1/application: post: description: Creates a new app for the configurations provided. The input json object is the same we get in response of GET method for fetching all details of an app