From 7f9a0c431ac98011afa406ad1aed0583fe7703ec Mon Sep 17 00:00:00 2001 From: ShashwatDevtron Date: Sat, 11 Feb 2023 21:44:46 +0530 Subject: [PATCH 001/118] Made chenges --- .../app/DeploymentPipelineRestHandler.go | 2 +- .../app/PipelineConfigRestHandler.go | 70 ++++++++++++++- api/router/PipelineConfigRouter.go | 2 + .../sql/repository/AppListingRepository.go | 2 +- internal/sql/repository/app/AppRepository.go | 2 +- .../AppListingRepositoryQueryBuilder.go | 2 +- .../service/AppStoreDeploymentService.go | 2 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 90 +++++++++++++++++++ pkg/pipeline/PipelineBuilder.go | 10 +++ pkg/security/ImageScanService.go | 2 +- pkg/security/policyService.go | 17 ++-- scripts/sql/1_insert.up.sql | 2 +- util/version.go | 2 +- 13 files changed, 187 insertions(+), 18 deletions(-) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 7a50fab46b..da2cd84708 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -763,7 +763,7 @@ func (handler PipelineConfigRestHandlerImpl) GetArtifactsByCDPipeline(w http.Res } if len(digests) > 0 { vulnerableMap := make(map[string]bool) - cvePolicy, severityPolicy, err := handler.policyService.GetApplicablePolicy(pipelineModel.Environment.ClusterId, pipelineModel.EnvironmentId, pipelineModel.AppId, pipelineModel.App.AppStore) + cvePolicy, severityPolicy, err := handler.policyService.GetApplicablePolicy(pipelineModel.Environment.ClusterId, pipelineModel.EnvironmentId, pipelineModel.AppId, pipelineModel.App.AppStore == 1) if err != nil { handler.Logger.Errorw("service err, GetArtifactsByCDPipeline", "err", err, "cdPipelineId", cdPipelineId, "stage", stage) } diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 436a4f5c0f..ac561d4ee0 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -59,9 +59,9 @@ import ( type DevtronAppRestHandler interface { CreateApp(w http.ResponseWriter, r *http.Request) + CreateJob(w http.ResponseWriter, r *http.Request) DeleteApp(w http.ResponseWriter, r *http.Request) GetApp(w http.ResponseWriter, r *http.Request) - FindAppsByTeamId(w http.ResponseWriter, r *http.Request) FindAppsByTeamName(w http.ResponseWriter, r *http.Request) } @@ -283,6 +283,74 @@ func (handler PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, createResp, http.StatusOK) } +func (handler PipelineConfigRestHandlerImpl) CreatJob(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("token") + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var createRequest bean.CreateAppDTO + err = decoder.Decode(&createRequest) + createRequest.UserId = userId + if err != nil { + handler.Logger.Errorw("request err, CreateApp", "err", err, "CreateApp", createRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + handler.Logger.Infow("request payload, CreateApp", "CreateApp", createRequest) + err = handler.validator.Struct(createRequest) + if err != nil { + handler.Logger.Errorw("validation err, CreateApp", "err", err, "CreateApp", createRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + project, err := handler.teamService.FetchOne(createRequest.TeamId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, fmt.Sprintf("%s/%s", strings.ToLower(project.Name), "*")); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + var createResp *bean.CreateAppDTO + err = nil + if createRequest.TemplateId == 0 { + createResp, err = handler.pipelineBuilder.CreateJob(&createRequest) + } else { + ctx, cancel := context.WithCancel(r.Context()) + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + var acdToken string + acdToken, err = handler.argoUserService.GetLatestDevtronArgoCdUserToken() + if err != nil { + handler.Logger.Errorw("error in getting acd token", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + ctx = context.WithValue(r.Context(), "token", acdToken) + createResp, err = handler.appCloneService.CloneApp(&createRequest, ctx) + } + if err != nil { + handler.Logger.Errorw("service err, CreateApp", "err", err, "CreateApp", createRequest) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, createResp, http.StatusOK) +} + func (handler PipelineConfigRestHandlerImpl) GetApp(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") vars := mux.Vars(r) diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index 9cc196ab03..79942d3e9a 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -51,6 +51,7 @@ func NewPipelineRouterImpl(restHandler app.PipelineConfigRestHandler, func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mux.Router) { configRouter.Path("").HandlerFunc(router.restHandler.CreateApp).Methods("POST") + configRouter.Path("/jobs").HandlerFunc(router.restHandler.CreateJob).Methods("POST") configRouter.Path("/{appId}").HandlerFunc(router.restHandler.DeleteApp).Methods("DELETE") configRouter.Path("/material").HandlerFunc(router.restHandler.CreateMaterial).Methods("POST") configRouter.Path("/material").HandlerFunc(router.restHandler.UpdateMaterial).Methods("PUT") @@ -94,6 +95,7 @@ func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mu configRouter.Path("/team/by-name/{teamName}").HandlerFunc(router.restHandler.FindAppsByTeamName).Methods("GET") configRouter.Path("/ci-pipeline/trigger").HandlerFunc(router.restHandler.TriggerCiPipeline).Methods("POST") + configRouter.Path("/{appId}/ci-pipeline/min").HandlerFunc(router.restHandler.GetCiPipelineMin).Methods("GET") configRouter.Path("/ci-pipeline/{pipelineId}/material").HandlerFunc(router.restHandler.FetchMaterials).Methods("GET") configRouter.Path("/ci-pipeline/refresh-material/{gitMaterialId}").HandlerFunc(router.restHandler.RefreshMaterials).Methods("GET") diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index ff372bafb9..a6d75c4769 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -313,7 +313,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig " INNER JOIN ci_pipeline cp on cp.id = p.ci_pipeline_id" + " INNER JOIN app a ON a.id = p.app_id" + " INNER JOIN environment env on env.id = p.environment_id" + - " WHERE p.app_id=? and p.deleted=false and a.app_store is false AND env.active = TRUE;" + " WHERE p.app_id=? and p.deleted=false and a.app_store is 0 AND env.active = TRUE;" impl.Logger.Debugw("query", query) _, err := impl.dbConnection.Query(&triggerView, query, appId) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 85511a55e9..c111bd3a3f 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -31,7 +31,7 @@ type App struct { AppName string `sql:"app_name,notnull"` //same as app name Active bool `sql:"active, notnull"` TeamId int `sql:"team_id"` - AppStore bool `sql:"app_store, notnull"` + AppStore int `sql:"app_store, notnull"` AppOfferingMode string `sql:"app_offering_mode,notnull"` Team team.Team sql.AuditLog diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 12293c0594..33c8c896c2 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -94,7 +94,7 @@ func (impl AppListingRepositoryQueryBuilder) buildAppListingSortBy(appListingFil } func (impl AppListingRepositoryQueryBuilder) buildAppListingWhereCondition(appListingFilter AppListingFilter) string { - whereCondition := "WHERE a.active = true and a.app_store is false " + whereCondition := "WHERE a.active = true and a.app_store is 0 " if len(appListingFilter.Environments) > 0 { envIds := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(appListingFilter.Environments)), ","), "[]") whereCondition = whereCondition + "and env.id IN (" + envIds + ") " diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index e0a5b197ea..beadc8a1cf 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -410,7 +410,7 @@ func (impl AppStoreDeploymentServiceImpl) createAppForAppStore(createRequest *be Active: true, AppName: createRequest.AppName, TeamId: createRequest.TeamId, - AppStore: true, + AppStore: 1, AppOfferingMode: appInstallationMode, AuditLog: sql.AuditLog{UpdatedBy: createRequest.UserId, CreatedBy: createRequest.UserId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 8a00358d57..89540859b1 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -57,6 +57,7 @@ import ( type CiCdPipelineOrchestrator interface { CreateApp(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) + CreateJob(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) DeleteApp(appId int, userId int32) error CreateMaterials(createMaterialRequest *bean.CreateMaterialDTO) (*bean.CreateMaterialDTO, error) UpdateMaterial(updateMaterialRequest *bean.UpdateMaterialDTO) (*bean.UpdateMaterialDTO, error) @@ -827,6 +828,43 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp return createRequest, nil } +func (impl CiCdPipelineOrchestratorImpl) CreateJob(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { + dbConnection := impl.appRepository.GetConnection() + tx, err := dbConnection.Begin() + if err != nil { + return nil, err + } + // Rollback tx on error. + defer tx.Rollback() + app, err := impl.createJobGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, tx) + if err != nil { + return nil, err + } + // create labels and tags with app + if app.Active && len(createRequest.AppLabels) > 0 { + for _, label := range createRequest.AppLabels { + request := &bean.AppLabelDto{ + AppId: app.Id, + Key: label.Key, + Value: label.Value, + UserId: createRequest.UserId, + } + _, err := impl.appLabelsService.Create(request, tx) + if err != nil { + impl.logger.Errorw("error on creating labels for app id ", "err", err, "appId", app.Id) + return nil, err + } + } + } + err = tx.Commit() + if err != nil { + impl.logger.Errorw("error in commit repo", "error", err) + return nil, err + } + createRequest.Id = app.Id + return createRequest, nil +} + func (impl CiCdPipelineOrchestratorImpl) DeleteApp(appId int, userId int32) error { // Delete git materials,call git sensor and delete app impl.logger.Debug("deleting materials in orchestrator") @@ -1030,6 +1068,58 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 return pg, nil } +func (impl CiCdPipelineOrchestratorImpl) createJobGroup(name string, userId int32, teamId int, tx *pg.Tx) (*app2.App, error) { + app, err := impl.appRepository.FindActiveByName(name) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if app != nil && app.Id > 0 { + impl.logger.Warnw("job already exists", "name", name) + err = &util.ApiError{ + Code: constants.AppAlreadyExists.Code, + InternalMessage: "job already exists", + UserMessage: constants.AppAlreadyExists.UserMessage(name), + } + return nil, err + } + pg := &app2.App{ + Active: true, + AppName: name, + TeamId: teamId, + AppStore: 2, + AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, + } + err = impl.appRepository.SaveWithTxn(pg, tx) + if err != nil { + impl.logger.Errorw("error in saving entity ", "entity", pg) + return nil, err + } + + apps, err := impl.appRepository.FindActiveListByName(name) + if err != nil { + return nil, err + } + appLen := len(apps) + if appLen > 1 { + firstElement := apps[0] + if firstElement.Id != pg.Id { + pg.Active = false + err = impl.appRepository.UpdateWithTxn(pg, tx) + if err != nil { + impl.logger.Errorw("error in saving entity ", "entity", pg) + return nil, err + } + err = &util.ApiError{ + Code: constants.AppAlreadyExists.Code, + InternalMessage: "job already exists", + UserMessage: constants.AppAlreadyExists.UserMessage(name), + } + return nil, err + } + } + return pg, nil +} + func (impl CiCdPipelineOrchestratorImpl) validateCheckoutPathsForMultiGit(allPaths map[int]string) error { dockerfilePathMap := make(map[string]bool) impl.logger.Debugw("all paths ", "path", allPaths) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index f0a418898d..56c468fc95 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -86,6 +86,7 @@ func GetDeploymentServiceTypeConfig() (*DeploymentServiceTypeConfig, error) { type PipelineBuilder interface { CreateCiPipeline(createRequest *bean.CiConfigRequest) (*bean.PipelineCreateResponse, error) CreateApp(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) + CreateJob(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) CreateMaterialsForApp(request *bean.CreateMaterialDTO) (*bean.CreateMaterialDTO, error) UpdateMaterialsForApp(request *bean.UpdateMaterialDTO) (*bean.UpdateMaterialDTO, error) DeleteMaterial(request *bean.UpdateMaterialDTO) error @@ -309,6 +310,15 @@ func (impl PipelineBuilderImpl) CreateApp(request *bean.CreateAppDTO) (*bean.Cre return res, err } +func (impl PipelineBuilderImpl) CreateJob(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { + impl.logger.Debugw("job create request received", "req", request) + res, err := impl.ciCdPipelineOrchestrator.CreateJob(request) + if err != nil { + impl.logger.Errorw("error in saving create job req", "req", request, "err", err) + } + return res, err +} + func (impl PipelineBuilderImpl) DeleteApp(appId int, userId int32) error { impl.logger.Debugw("app delete request received", "app", appId) err := impl.ciCdPipelineOrchestrator.DeleteApp(appId, userId) diff --git a/pkg/security/ImageScanService.go b/pkg/security/ImageScanService.go index 83a9d78062..eecabf6cfd 100644 --- a/pkg/security/ImageScanService.go +++ b/pkg/security/ImageScanService.go @@ -409,7 +409,7 @@ func (impl ImageScanServiceImpl) FetchExecutionDetailResult(request *ImageScanRe imageScanResponse.EnvId = request.EnvId imageScanResponse.EnvName = env.Environment - blockCveList, err := impl.policyService.GetBlockedCVEList(cveStores, env.ClusterId, env.Id, request.AppId, app.AppStore) + blockCveList, err := impl.policyService.GetBlockedCVEList(cveStores, env.ClusterId, env.Id, request.AppId, app.AppStore == 1) if err != nil { impl.Logger.Errorw("error while fetching env", "err", err) //return nil, err diff --git a/pkg/security/policyService.go b/pkg/security/policyService.go index 0b2591ae9d..8d36c6f4a1 100644 --- a/pkg/security/policyService.go +++ b/pkg/security/policyService.go @@ -190,7 +190,7 @@ func (impl *PolicyServiceImpl) VerifyImage(verifyImageRequest *VerifyImageReques return nil, err } else if app != nil { appId = app.Id - isAppStore = app.AppStore + isAppStore = app.AppStore == 1 } else { //np app do nothing } @@ -327,7 +327,7 @@ func (impl *PolicyServiceImpl) VerifyImage(verifyImageRequest *VerifyImageReques return imageBlockedCves, nil } -//image(cve), appId, envId +// image(cve), appId, envId func (impl *PolicyServiceImpl) enforceCvePolicy(cves []*security.CveStore, cvePolicy map[string]*security.CvePolicy, severityPolicy map[security.Severity]*security.CvePolicy) (blockedCVE []*security.CveStore) { for _, cve := range cves { @@ -497,8 +497,8 @@ func (impl *PolicyServiceImpl) SavePolicy(request bean.CreateVulnerabilityPolicy } /* - 1. policy id - 2. action +1. policy id +2. action */ func (impl *PolicyServiceImpl) UpdatePolicy(updatePolicyParams bean.UpdatePolicyParams, userId int32) (*bean.IdVulnerabilityPolicyResult, error) { policyAction, err := impl.parsePolicyAction(updatePolicyParams.Action) @@ -550,13 +550,12 @@ func (impl *PolicyServiceImpl) DeletePolicy(id int, userId int32) (*bean.IdVulne } /* - global: na - cluster: clusterId - environment: environmentId - application : appId, envId + global: na + cluster: clusterId + environment: environmentId + application : appId, envId res: - */ func (impl *PolicyServiceImpl) GetPolicies(policyLevel security.PolicyLevel, clusterId, environmentId, appId int) (*bean.GetVulnerabilityPolicyResult, error) { diff --git a/scripts/sql/1_insert.up.sql b/scripts/sql/1_insert.up.sql index ffa7476e7a..28013c19ec 100644 --- a/scripts/sql/1_insert.up.sql +++ b/scripts/sql/1_insert.up.sql @@ -32,7 +32,7 @@ CREATE TABLE public.app ( updated_on timestamp with time zone NOT NULL, updated_by integer NOT NULL, team_id integer, - app_store boolean DEFAULT false + app_store integer DEFAULT 0 ); diff --git a/util/version.go b/util/version.go index f64cb32956..417b3fb010 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: ServerMode} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE_FULL} } func IsBaseStack() bool { From 893c803b2c6663151d6323dd6b945d5059f1c19f Mon Sep 17 00:00:00 2001 From: ShashwatDevtron Date: Sat, 11 Feb 2023 22:16:09 +0530 Subject: [PATCH 002/118] Made chenges --- api/restHandler/app/PipelineConfigRestHandler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index ac561d4ee0..99263698c5 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -115,6 +115,11 @@ type PipelineConfigRestHandlerImpl struct { argoUserService argo.ArgoUserService } +func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r *http.Request) { + //TODO implement me + panic("implement me") +} + func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, From c60f6f06611acb3b303297b14e3205a017ab5bab Mon Sep 17 00:00:00 2001 From: ShashwatDevtron Date: Sun, 12 Feb 2023 12:16:33 +0530 Subject: [PATCH 003/118] Made chenges --- api/restHandler/app/PipelineConfigRestHandler.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 99263698c5..a27d5329aa 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -115,11 +115,6 @@ type PipelineConfigRestHandlerImpl struct { argoUserService argo.ArgoUserService } -func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r *http.Request) { - //TODO implement me - panic("implement me") -} - func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, chartService chart.ChartService, propertiesConfigService pipeline.PropertiesConfigService, @@ -288,7 +283,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, createResp, http.StatusOK) } -func (handler PipelineConfigRestHandlerImpl) CreatJob(w http.ResponseWriter, r *http.Request) { +func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) From a69a0062a69895dac2fa0d3cdd07f1c963d2cab9 Mon Sep 17 00:00:00 2001 From: ShashwatDevtron Date: Mon, 13 Feb 2023 00:06:27 +0530 Subject: [PATCH 004/118] Clone job function added --- .../app/PipelineConfigRestHandler.go | 2 +- internal/constants/InternalErrorCode.go | 1 + pkg/appClone/AppCloneService.go | 126 ++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index a27d5329aa..187e9e592a 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -341,7 +341,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r return } ctx = context.WithValue(r.Context(), "token", acdToken) - createResp, err = handler.appCloneService.CloneApp(&createRequest, ctx) + createResp, err = handler.appCloneService.CloneJob(&createRequest, ctx) } if err != nil { handler.Logger.Errorw("service err, CreateApp", "err", err, "CreateApp", createRequest) diff --git a/internal/constants/InternalErrorCode.go b/internal/constants/InternalErrorCode.go index e8dc49384e..7a60d5d29d 100644 --- a/internal/constants/InternalErrorCode.go +++ b/internal/constants/InternalErrorCode.go @@ -82,3 +82,4 @@ const ( ) var AppAlreadyExists = &ErrorCode{"4001", "application %s already exists"} +var AppDoesNotExist = &ErrorCode{"4004", "application %s does not exist"} diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 449ead356a..195081616f 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -20,7 +20,11 @@ package appClone import ( "context" bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/constants" + app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/chart" + "github.com/go-pg/pg" "strings" "fmt" @@ -35,8 +39,10 @@ import ( type AppCloneService interface { CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) + CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) } type AppCloneServiceImpl struct { + appRepository app2.AppRepository logger *zap.SugaredLogger pipelineBuilder pipeline.PipelineBuilder materialRepository pipelineConfig.MaterialRepository @@ -81,6 +87,20 @@ type CloneRequest struct { } func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { + //validate template app + templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { + impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) + err = &util.ApiError{ + Code: constants.AppDoesNotExist.Code, + InternalMessage: "app does not exist", + UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + } + return nil, err + } //create new app cloneReq := &CloneRequest{ RefAppId: createReq.TemplateId, @@ -187,6 +207,112 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context return app, nil } +func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { + //validate template job + templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { + impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) + err = &util.ApiError{ + Code: constants.AppDoesNotExist.Code, + InternalMessage: "job does not exist", + UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + } + return nil, err + } + //create new job + + cloneReq := &CloneRequest{ + RefAppId: createReq.TemplateId, + Name: createReq.AppName, + ProjectId: createReq.TeamId, + AppLabels: createReq.AppLabels, + } + userId := createReq.UserId + appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) + if err != nil { + return nil, err + } + refApp, err := impl.pipelineBuilder.GetApp(cloneReq.RefAppId) + if err != nil { + return nil, err + } + isSmaeProject := refApp.TeamId == cloneReq.ProjectId + /* appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId)) + appStageStatus = append(appStageStatus, impl.makeAppStageStatus(1, "MATERIAL", materialExists)) + appStageStatus = append(appStageStatus, impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId)) + appStageStatus = append(appStageStatus, impl.makeAppStageStatus(3, "CI_PIPELINE", stages.CiPipelineId)) + appStageStatus = append(appStageStatus, impl.makeAppStageStatus(4, "CHART", stages.ChartId)) + appStageStatus = append(appStageStatus, impl.makeAppStageStatus(5, "CD_PIPELINE", stages.PipelineId)) + */ + refAppStatus := make(map[string]bool) + for _, as := range appStatus { + refAppStatus[as.StageName] = as.Status + } + + //TODO check stage of current app + if !refAppStatus["APP"] { + impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) + return nil, nil + } + app, err := impl.CreateApp(cloneReq, userId) + if err != nil { + impl.logger.Errorw("error in creating app", "req", cloneReq, "err", err) + return nil, err + } + newAppId := app.Id + if !refAppStatus["MATERIAL"] { + impl.logger.Errorw("status not", "MATERIAL", cloneReq.RefAppId) + return app, nil + } + _, gitMaerialMap, err := impl.CloneGitRepo(cloneReq.RefAppId, newAppId, userId) + if err != nil { + impl.logger.Errorw("error in cloning git", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) + return nil, err + } + + _, err = impl.CreateCiTemplate(cloneReq.RefAppId, newAppId, userId) + if err != nil { + impl.logger.Errorw("error in cloning docker template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) + return nil, err + } + if !refAppStatus["TEMPLATE"] { + impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) + return app, nil + } + if !refAppStatus["CHART"] { + impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) + return app, nil + } + if isSmaeProject { + _, err = impl.CreateEnvCm(context, cloneReq.RefAppId, newAppId, userId) + if err != nil { + impl.logger.Errorw("error in creating env cm", "err", err) + return nil, err + } + _, err = impl.CreateEnvSecret(context, cloneReq.RefAppId, newAppId, userId) + if err != nil { + impl.logger.Errorw("error in creating env secret", "err", err) + return nil, err + } + _, err = impl.createEnvOverride(cloneReq.RefAppId, newAppId, userId, context) + if err != nil { + impl.logger.Errorw("error in cloning env override", "err", err) + return nil, err + } + } + + _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSmaeProject) + if err != nil { + impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) + return nil, err + } + + return app, nil +} + func (impl *AppCloneServiceImpl) CreateApp(cloneReq *CloneRequest, userId int32) (*bean.CreateAppDTO, error) { createAppReq := &bean.CreateAppDTO{ AppName: cloneReq.Name, From d21dbac89a76f2aa6b2bdbd68f237f1d754d3465 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Feb 2023 00:13:24 +0530 Subject: [PATCH 005/118] fixed typo --- pkg/appClone/AppCloneService.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 195081616f..68aa03fa02 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -117,7 +117,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context if err != nil { return nil, err } - isSmaeProject := refApp.TeamId == cloneReq.ProjectId + isSameProject := refApp.TeamId == cloneReq.ProjectId /* appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId)) appStageStatus = append(appStageStatus, impl.makeAppStageStatus(1, "MATERIAL", materialExists)) appStageStatus = append(appStageStatus, impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId)) @@ -180,7 +180,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context impl.logger.Errorw("error in creating global secret", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err } - if isSmaeProject { + if isSameProject { _, err = impl.CreateEnvCm(context, cloneReq.RefAppId, newAppId, userId) if err != nil { impl.logger.Errorw("error in creating env cm", "err", err) @@ -198,7 +198,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context } } - _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSmaeProject) + _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) if err != nil { impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err @@ -239,7 +239,7 @@ func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context if err != nil { return nil, err } - isSmaeProject := refApp.TeamId == cloneReq.ProjectId + isSameProject := refApp.TeamId == cloneReq.ProjectId /* appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId)) appStageStatus = append(appStageStatus, impl.makeAppStageStatus(1, "MATERIAL", materialExists)) appStageStatus = append(appStageStatus, impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId)) @@ -286,7 +286,7 @@ func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) return app, nil } - if isSmaeProject { + if isSameProject { _, err = impl.CreateEnvCm(context, cloneReq.RefAppId, newAppId, userId) if err != nil { impl.logger.Errorw("error in creating env cm", "err", err) @@ -304,7 +304,7 @@ func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context } } - _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSmaeProject) + _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) if err != nil { impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err @@ -683,7 +683,7 @@ func (impl *AppCloneServiceImpl) CreateGlobalSecret(oldAppId, newAppId int, user return thisCm, err } -func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, gitMaterialMapping map[int]int, ctx context.Context, isSmaeProject bool) (interface{}, error) { +func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, gitMaterialMapping map[int]int, ctx context.Context, isSameProject bool) (interface{}, error) { refAppWFs, err := impl.appWorkflowService.FindAppWorkflows(oldAppId) if err != nil { return nil, err @@ -701,7 +701,7 @@ func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, if err != nil { return nil, err } - err = impl.createWfMappings(refAppWF.AppWorkflowMappingDto, oldAppId, newAppId, userId, thisWf.Id, gitMaterialMapping, ctx, isSmaeProject) + err = impl.createWfMappings(refAppWF.AppWorkflowMappingDto, oldAppId, newAppId, userId, thisWf.Id, gitMaterialMapping, ctx, isSameProject) if err != nil { return nil, err } @@ -709,7 +709,7 @@ func (impl *AppCloneServiceImpl) CreateWf(oldAppId, newAppId int, userId int32, return nil, nil } -func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.AppWorkflowMappingDto, oldAppId, newAppId int, userId int32, thisWfId int, gitMaterialMapping map[int]int, ctx context.Context, isSmaeProject bool) error { +func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.AppWorkflowMappingDto, oldAppId, newAppId int, userId int32, thisWfId int, gitMaterialMapping map[int]int, ctx context.Context, isSameProject bool) error { impl.logger.Debugw("wf mapping cloning", "refWfMappings", refWfMappings) var ciMapping []appWorkflow.AppWorkflowMappingDto var cdMappings []appWorkflow.AppWorkflowMappingDto @@ -759,7 +759,7 @@ func (impl *AppCloneServiceImpl) createWfMappings(refWfMappings []appWorkflow.Ap } impl.logger.Debugw("ci created", "ci", ci) } - if isSmaeProject { + if isSameProject { for _, refCdMapping := range cdMappings { cdCloneReq := &cloneCdPipelineRequest{ refCdPipelineId: refCdMapping.ComponentId, From 777725b7d168cd4b7706154c42ab973b56b81096 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Feb 2023 00:27:53 +0530 Subject: [PATCH 006/118] fixed typo --- api/restHandler/app/PipelineConfigRestHandler.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 187e9e592a..11b8bb23ee 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -291,6 +291,13 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + + //isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + //if !isSuperAdmin || err != nil { + // common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + // return + //} + var createRequest bean.CreateAppDTO err = decoder.Decode(&createRequest) createRequest.UserId = userId From 916e196259a301390a9aec8ff04671e113cd0234 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Mon, 13 Feb 2023 00:42:45 +0530 Subject: [PATCH 007/118] Merging main to job-feature-branch (#2955) * fix: Installed apps rbac optimisation (#2918) * rbac enforced in batch * minor restructuring * minor fix * code comments added * throwing unauthorised error * fix: pvc mounted on pods for cache handling (#2912) * first cut pvc for ci * isPvcMounted flag introduced * cache invalidate and global invalidate handling * cache path made configurable * pvc mounted false * PVC mounted at two new paths for storing build and buildx cache * mnor fix * removed redundant objects from struct * chenged names of env variables for paths * buildx cache path restored after doing POC --------- Co-authored-by: Prashant Ghildiyal Co-authored-by: Kripansh * updated log type for pipeline status methods (#2926) * docs: minor updates ingress (#2931) * minor updates ingress * minor updates ingress * finished on time update fix (#2932) Co-authored-by: Kripansh * fix delete pipeline app status check (#2939) * feat: added option to propagate custom tag to k8s resources for that application (#2841) * wip * sql file renamed * propagation in labels support for create app api * validating labels if propagate is true * refactoring * sql file no. changed * removed required field from bool * sql file no. changed * removed cascade * alter app lable key * db migration seq fix for global tags --------- Co-authored-by: vikramdevtron * excluding inactive ci-pipelin-material from pre-cd stage (#2920) * feat: cluster bearer token hide from dashboard (#2894) * cluster token config removed mandatory * api spec added for cluster update and create, and cluster list api changes for token * fix check config for cluster token * docs: mount pvc (#2941) * added pvc * added pvc * added pvc * added pvc * added pvc * feat: Resource browser child ref (#2913) * child rbac handling init commit * resource manifest validate handling added * handle rbac case * wire gen fix * gvk passed and ap resource handling * code cleaning * dead code cleaning * removed unused func * fix: k8s log stream cpu issue (#2929) * updated buffer size * refactored code for getting logs from k8s * updated Dockerfile * updated Dockerfile * downgraded go version * removed redundant log * sorting_changes (#2935) * support given for cloning linked ci pipelines via workflow cloning API (#2944) * docs: Added preset plugins (#2904) * added preset plugins * added preset plugins * Added semgrep, codacy * updates * updated screenshots * updates --------- Co-authored-by: Gireesh Naidu <111440205+gireesh-devtron@users.noreply.github.com> Co-authored-by: Prakash Kumar Co-authored-by: Prashant Ghildiyal Co-authored-by: Kripansh Co-authored-by: kartik-579 <84493919+kartik-579@users.noreply.github.com> Co-authored-by: SNe789 <113748500+SNe789@users.noreply.github.com> Co-authored-by: Vikram <73224103+vikramdevtron@users.noreply.github.com> Co-authored-by: Manish Agrawal <85211469+manish-agrawal-ai@users.noreply.github.com> Co-authored-by: vikramdevtron Co-authored-by: kripanshdevtron <107392309+kripanshdevtron@users.noreply.github.com> Co-authored-by: Dhananjay Sharma <124036328+dhananjay0601@users.noreply.github.com> --- Dockerfile | 2 +- DockerfileEA | 2 +- api/appStore/InstalledAppRestHandler.go | 84 ++++- api/appbean/AppDetail.go | 15 +- api/cluster/ClusterRestHandler.go | 25 +- api/connector/Connector.go | 52 +-- api/restHandler/CoreAppRestHandler.go | 36 ++- .../pubsub/ApplicationStatusUpdateHandler.go | 2 - .../cron/CdApplicationStatusUpdateHandler.go | 2 - cmd/external-app/wire_gen.go | 11 +- docs/SUMMARY.md | 2 +- ...l-devtron-on-Minikube-Microk8s-K3s-Kind.md | 5 +- docs/setup/install/ingress-setup.md | 15 +- .../install-devtron-with-cicd-with-gitops.md | 8 +- .../install/install-devtron-with-cicd.md | 5 +- docs/setup/install/install-devtron.md | 7 +- .../creating-application/overview.md | 61 +++- .../workflow/ci-build-pre-post-plugins.md | 297 +++++++++++++++--- .../pipelineConfig/AppLabelsRepository.go | 1 + internal/util/K8sUtil.go | 102 +++++- internal/util/K8sUtilBean.go | 29 ++ pkg/app/AppCrudOperationService.go | 70 +++-- pkg/app/AppListingViewBuilder.go | 5 + pkg/app/AppService.go | 2 +- pkg/bean/app.go | 14 +- pkg/cluster/ClusterService.go | 38 ++- pkg/cluster/ClusterServiceExtended.go | 11 + pkg/pipeline/CdHandler.go | 5 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 23 +- pkg/pipeline/CiConfig.go | 3 + pkg/pipeline/CiService.go | 3 +- pkg/pipeline/PipelineBuilder.go | 2 +- pkg/pipeline/WorkflowDagExecutor.go | 6 +- pkg/pipeline/WorkflowService.go | 46 ++- scripts/sql/113_labels_propagate.down.sql | 3 + scripts/sql/113_labels_propagate.up.sql | 3 + scripts/sql/114_global_tags.down.sql | 4 + scripts/sql/114_global_tags.up.sql | 18 ++ specs/app-labels.yaml | 3 + specs/app_create_api.yaml | 3 + specs/application.yaml | 3 + specs/cluster_api_spec.yaml | 118 +++++++ util/K8sUtil.go | 20 ++ util/k8s/k8sApplicationRestHandler.go | 86 ++--- util/k8s/k8sApplicationService.go | 73 ++++- wire_gen.go | 7 +- 46 files changed, 1087 insertions(+), 245 deletions(-) create mode 100644 scripts/sql/113_labels_propagate.down.sql create mode 100644 scripts/sql/113_labels_propagate.up.sql create mode 100644 scripts/sql/114_global_tags.down.sql create mode 100644 scripts/sql/114_global_tags.up.sql create mode 100644 util/K8sUtil.go diff --git a/Dockerfile b/Dockerfile index 20d83e6ce1..643642cc0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19 AS build-env +FROM golang:1.18 AS build-env RUN echo $GOPATH RUN apt update diff --git a/DockerfileEA b/DockerfileEA index 52a4ec3592..f7400f0e05 100644 --- a/DockerfileEA +++ b/DockerfileEA @@ -1,4 +1,4 @@ -FROM golang:1.19 AS build-env +FROM golang:1.18 AS build-env RUN echo $GOPATH RUN apt update diff --git a/api/appStore/InstalledAppRestHandler.go b/api/appStore/InstalledAppRestHandler.go index 9eb163d32e..88b2d81807 100644 --- a/api/appStore/InstalledAppRestHandler.go +++ b/api/appStore/InstalledAppRestHandler.go @@ -95,6 +95,12 @@ func (handler InstalledAppRestHandlerImpl) GetAllInstalledApp(w http.ResponseWri } v := r.URL.Query() token := r.Header.Get("token") + userEmailId, err := handler.userAuthService.GetEmailFromToken(token) + if err != nil { + handler.Logger.Errorw("error in getting user emailId from token", "userId", userId, "token", token) + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } var envIds []int envsQueryParam := v.Get("envIds") if envsQueryParam != "" { @@ -181,32 +187,80 @@ func (handler InstalledAppRestHandlerImpl) GetAllInstalledApp(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } + isActionUserSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if err != nil { + handler.Logger.Errorw("request err, GetAllInstalledApp", "err", err, "userId", userId) + common.WriteJsonResp(w, err, "Failed to check is super admin", http.StatusInternalServerError) + return + } + if isActionUserSuperAdmin { + common.WriteJsonResp(w, err, res, http.StatusOK) + return + } + + appIdToAppMap := make(map[string]openapi.HelmApp) + + //the value of this map is array of strings because the GetHelmObjectByAppNameAndEnvId method may return "//" for error cases + //so different apps may contain same object, to handle that we are using (map[string] []string) + rbacObjectToAppIdMap1 := make(map[string][]string) + rbacObjectToAppIdMap2 := make(map[string][]string) + + objectArray1 := make([]string, 0) + objectArray2 := make([]string, 0) - authorizedApp := make([]openapi.HelmApp, 0) for _, app := range *res.HelmApps { + + appIdToAppMap[*app.AppId] = app appName := *app.AppName envId := (*app.EnvironmentDetail).EnvironmentId - //rbac block starts from here - object, object2 := handler.enforcerUtil.GetHelmObjectByAppNameAndEnvId(appName, int(*envId)) + object1, object2 := handler.enforcerUtil.GetHelmObjectByAppNameAndEnvId(appName, int(*envId)) + objectArray1 = append(objectArray1, object1) + _, ok := rbacObjectToAppIdMap1[object1] + if !ok { + rbacObjectToAppIdMap1[object1] = make([]string, 0) + } + rbacObjectToAppIdMap1[object1] = append(rbacObjectToAppIdMap1[object1], *app.AppId) + if object2 != "" { + _, ok := rbacObjectToAppIdMap2[object2] + if !ok { + rbacObjectToAppIdMap2[object2] = make([]string, 0) + } + rbacObjectToAppIdMap2[object2] = append(rbacObjectToAppIdMap2[object2], *app.AppId) + objectArray2 = append(objectArray2, object2) + } - var ok bool + } - if object2 == "" { - ok = handler.enforcer.Enforce(token, casbin.ResourceHelmApp, casbin.ActionGet, object) - } else { - // futuristic case - ok = handler.enforcer.Enforce(token, casbin.ResourceHelmApp, casbin.ActionGet, object) || handler.enforcer.Enforce(token, casbin.ResourceHelmApp, casbin.ActionGet, object2) - } + resultObjectMap1 := handler.enforcer.EnforceByEmailInBatch(userEmailId, casbin.ResourceHelmApp, casbin.ActionGet, objectArray1) + resultObjectMap2 := handler.enforcer.EnforceByEmailInBatch(userEmailId, casbin.ResourceHelmApp, casbin.ActionGet, objectArray2) - if !ok { - continue + authorizedAppIdSet := make(map[string]bool) + //O(n) time loop , at max we will only iterate through all the apps + for obj, ok := range resultObjectMap1 { + if ok { + appIds := rbacObjectToAppIdMap1[obj] + for _, appId := range appIds { + authorizedAppIdSet[appId] = true + } + + } + } + for obj, ok := range resultObjectMap2 { + if ok { + appIds := rbacObjectToAppIdMap2[obj] + for _, appId := range appIds { + authorizedAppIdSet[appId] = true + } } + } - authorizedApp = append(authorizedApp, app) - //rback block ends here + authorizedApps := make([]openapi.HelmApp, 0) + for appId, _ := range authorizedAppIdSet { + authorizedApp := appIdToAppMap[appId] + authorizedApps = append(authorizedApps, authorizedApp) } - res.HelmApps = &authorizedApp + res.HelmApps = &authorizedApps common.WriteJsonResp(w, err, res, http.StatusOK) } diff --git a/api/appbean/AppDetail.go b/api/appbean/AppDetail.go index 069c0ad9e1..6f7966e359 100644 --- a/api/appbean/AppDetail.go +++ b/api/appbean/AppDetail.go @@ -32,8 +32,9 @@ type AppMetadata struct { } type AppLabel struct { - Key string `json:"key,notnull" validate:"required"` - Value string `json:"value,notnull" validate:"required"` + Key string `json:"key,notnull" validate:"required"` + Value string `json:"value,notnull" validate:"required"` + Propagate bool `json:"propagate"` } type GitMaterial struct { @@ -85,12 +86,16 @@ type CiPipelineDetails struct { PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty"` PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty"` IsExternal bool `json:"isExternal"` // true for linked and external + ParentCiPipeline int `json:"parentCiPipeline,omitempty"` + ParentAppId int `json:"parentAppId,omitempty"` + LinkedCount int `json:"linkedCount,omitempty"` } type CiPipelineMaterialConfig struct { - Type pipelineConfig.SourceType `json:"type,omitempty" validate:"oneof=SOURCE_TYPE_BRANCH_FIXED WEBHOOK"` - Value string `json:"value,omitempty" ` - CheckoutPath string `json:"checkoutPath"` + Type pipelineConfig.SourceType `json:"type,omitempty" validate:"oneof=SOURCE_TYPE_BRANCH_FIXED WEBHOOK"` + Value string `json:"value,omitempty" ` + CheckoutPath string `json:"checkoutPath"` + GitMaterialId int `json:"gitMaterialId"` } type BuildScript struct { diff --git a/api/cluster/ClusterRestHandler.go b/api/cluster/ClusterRestHandler.go index d9bf6d0d05..8cb8187a15 100644 --- a/api/cluster/ClusterRestHandler.go +++ b/api/cluster/ClusterRestHandler.go @@ -43,7 +43,6 @@ const CLUSTER_DELETE_SUCCESS_RESP = "Cluster deleted successfully." type ClusterRestHandler interface { Save(w http.ResponseWriter, r *http.Request) - FindOne(w http.ResponseWriter, r *http.Request) FindAll(w http.ResponseWriter, r *http.Request) FindById(w http.ResponseWriter, r *http.Request) @@ -153,29 +152,9 @@ func (impl ClusterRestHandlerImpl) Save(w http.ResponseWriter, r *http.Request) common.WriteJsonResp(w, err, bean, http.StatusOK) } -func (impl ClusterRestHandlerImpl) FindOne(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - cName := vars["cluster_name"] - // RBAC enforcer applying - token := r.Header.Get("token") - if ok := impl.enforcer.Enforce(token, casbin.ResourceCluster, casbin.ActionGet, strings.ToLower(cName)); !ok { - common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) - return - } - //RBAC enforcer Ends - - envBean, err := impl.clusterService.FindOne(cName) - if err != nil { - impl.logger.Errorw("service err, FindOne", "error", err, "cluster name", cName) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, envBean, http.StatusOK) -} - func (impl ClusterRestHandlerImpl) FindAll(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") - clusterList, err := impl.clusterService.FindAll() + clusterList, err := impl.clusterService.FindAllWithoutConfig() if err != nil { impl.logger.Errorw("service err, FindAll", "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -203,7 +182,7 @@ func (impl ClusterRestHandlerImpl) FindById(w http.ResponseWriter, r *http.Reque common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - bean, err := impl.clusterService.FindById(i) + bean, err := impl.clusterService.FindByIdWithoutConfig(i) if err != nil { impl.logger.Errorw("service err, FindById", "err", err, "clusterId", id) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/connector/Connector.go b/api/connector/Connector.go index daecd4f688..eb82518ab1 100644 --- a/api/connector/Connector.go +++ b/api/connector/Connector.go @@ -33,6 +33,7 @@ import ( "net/http" "regexp" "strconv" + "strings" "sync" "time" ) @@ -104,28 +105,39 @@ func (impl PumpImpl) StartK8sStreamWithHeartBeat(w http.ResponseWriter, isReconn done <- true }() - // heartbeat end - for { - sc := bufio.NewScanner(stream) - for sc.Scan() { - log := sc.Text() - a := regexp.MustCompile(" ") - splitLog := a.Split(log, 2) - timeParsed, err := time.Parse(time.RFC3339, splitLog[0]) - if err != nil { - impl.logger.Errorw("error in writing data over sse", "err", err) - return - } - mux.Lock() - err = impl.sendEvent([]byte(strconv.FormatInt(timeParsed.UnixNano(), 10)), nil, []byte(splitLog[1]), w) - mux.Unlock() - if err != nil { - impl.logger.Errorw("error in writing data over sse", "err", err) + bufReader := bufio.NewReader(stream) + eof := false + for !eof { + log, err := bufReader.ReadString('\n') + if err == io.EOF { + eof = true + // stop if we reached end of stream and the next line is empty + if log == "" { return } - f.Flush() + } else if err != nil && err != io.EOF { + impl.logger.Errorw("error in reading buffer string, StartK8sStreamWithHeartBeat", "err", err) + return + } + log = strings.TrimSpace(log) // Remove trailing line ending + a := regexp.MustCompile(" ") + splitLog := a.Split(log, 2) + parsedTime, err := time.Parse(time.RFC3339, splitLog[0]) + if err != nil { + impl.logger.Errorw("error in writing data over sse", "err", err) + return } + eventId := strconv.FormatInt(parsedTime.UnixNano(), 10) + mux.Lock() + err = impl.sendEvent([]byte(eventId), nil, []byte(splitLog[1]), w) + mux.Unlock() + if err != nil { + impl.logger.Errorw("error in writing data over sse", "err", err) + return + } + f.Flush() } + // heartbeat end } func (impl PumpImpl) StartStreamWithHeartBeat(w http.ResponseWriter, isReconnect bool, recv func() (*application.LogEntry, error), err error) { @@ -223,11 +235,9 @@ func (impl *PumpImpl) sendEvent(eventId []byte, eventName []byte, payload []byte res = append(res, payload...) } res = append(res, '\n', '\n') - if i, err := w.Write(res); err != nil { + if _, err := w.Write(res); err != nil { impl.logger.Errorf("Failed to send response chunk: %v", err) return err - } else { - impl.logger.Debugw("msg written", "count", i) } return nil diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 1915fd2997..0550d8d20d 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -437,8 +437,9 @@ func (handler CoreAppRestHandlerImpl) buildAppMetadata(appId int) (*appBean.AppM if len(appMetaInfo.Labels) > 0 { for _, label := range appMetaInfo.Labels { appLabelsRes = append(appLabelsRes, &appBean.AppLabel{ - Key: label.Key, - Value: label.Value, + Key: label.Key, + Value: label.Value, + Propagate: label.Propagate, }) } } @@ -679,6 +680,9 @@ func (handler CoreAppRestHandlerImpl) buildCiPipelineResp(appId int, ciPipeline DockerBuildArgs: ciPipeline.DockerArgs, VulnerabilityScanEnabled: ciPipeline.ScanEnabled, IsExternal: ciPipeline.IsExternal, + ParentCiPipeline: ciPipeline.ParentCiPipeline, + ParentAppId: ciPipeline.ParentAppId, + LinkedCount: ciPipeline.LinkedCount, } //build ciPipelineMaterial resp @@ -690,9 +694,10 @@ func (handler CoreAppRestHandlerImpl) buildCiPipelineResp(appId int, ciPipeline return nil, err } ciPipelineMaterialConfig := &appBean.CiPipelineMaterialConfig{ - Type: ciMaterial.Source.Type, - Value: ciMaterial.Source.Value, - CheckoutPath: gitMaterial.CheckoutPath, + Type: ciMaterial.Source.Type, + Value: ciMaterial.Source.Value, + CheckoutPath: gitMaterial.CheckoutPath, + GitMaterialId: gitMaterial.Id, } ciPipelineMaterialsConfig = append(ciPipelineMaterialsConfig, ciPipelineMaterialConfig) } @@ -1101,8 +1106,9 @@ func (handler CoreAppRestHandlerImpl) createBlankApp(appMetadata *appBean.AppMet var appLabels []*bean.Label for _, requestLabel := range appMetadata.Labels { appLabel := &bean.Label{ - Key: requestLabel.Key, - Value: requestLabel.Value, + Key: requestLabel.Key, + Value: requestLabel.Value, + Propagate: requestLabel.Propagate, } appLabels = append(appLabels, appLabel) } @@ -1532,7 +1538,7 @@ func (handler CoreAppRestHandlerImpl) createWorkflowInDb(workflowName string, ap func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, workflowId int, ciPipelineData *appBean.CiPipelineDetails) (int, error) { // if ci pipeline is of external type, then throw error as we are not supporting it as of now - if ciPipelineData.IsExternal { + if ciPipelineData.ParentCiPipeline == 0 && ciPipelineData.ParentAppId == 0 && ciPipelineData.IsExternal { err := errors.New("external ci pipeline creation is not supported yet") handler.logger.Error("external ci pipeline creation is not supported yet") return 0, err @@ -1541,8 +1547,15 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, // build ci pipeline materials starts var ciMaterialsRequest []*bean.CiMaterial for _, ciMaterial := range ciPipelineData.CiPipelineMaterialsConfig { - //finding gitMaterial by appId and checkoutPath - gitMaterial, err := handler.materialRepository.FindByAppIdAndCheckoutPath(appId, ciMaterial.CheckoutPath) + var gitMaterial *pipelineConfig.GitMaterial + var err error + if ciPipelineData.ParentCiPipeline == 0 && ciPipelineData.ParentAppId == 0 { + //finding gitMaterial by appId and checkoutPath + gitMaterial, err = handler.materialRepository.FindByAppIdAndCheckoutPath(appId, ciMaterial.CheckoutPath) + } else { + //if linkedci find git material by it's id + gitMaterial, err = handler.materialRepository.FindById(ciMaterial.GitMaterialId) + } if err != nil { handler.logger.Errorw("service err, FindByAppIdAndCheckoutPath in CreateWorkflows", "err", err, "appId", appId) return 0, err @@ -1585,6 +1598,9 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, CiMaterial: ciMaterialsRequest, PreBuildStage: ciPipelineData.PreBuildStage, PostBuildStage: ciPipelineData.PostBuildStage, + ParentCiPipeline: ciPipelineData.ParentCiPipeline, + ParentAppId: ciPipelineData.ParentAppId, + LinkedCount: ciPipelineData.LinkedCount, }, } diff --git a/api/router/pubsub/ApplicationStatusUpdateHandler.go b/api/router/pubsub/ApplicationStatusUpdateHandler.go index 53b68a8b1d..f1ad49df1a 100644 --- a/api/router/pubsub/ApplicationStatusUpdateHandler.go +++ b/api/router/pubsub/ApplicationStatusUpdateHandler.go @@ -66,8 +66,6 @@ type ApplicationDetail struct { func (impl *ApplicationStatusUpdateHandlerImpl) Subscribe() error { callback := func(msg *pubsub.PubSubMsg) { - impl.logger.Debug("received app update request") - //defer msg.Ack() impl.logger.Debugw("APP_STATUS_UPDATE_REQ", "stage", "raw", "data", msg.Data) applicationDetail := ApplicationDetail{} err := json.Unmarshal([]byte(msg.Data), &applicationDetail) diff --git a/client/cron/CdApplicationStatusUpdateHandler.go b/client/cron/CdApplicationStatusUpdateHandler.go index 0e13facc23..c79fc2a306 100644 --- a/client/cron/CdApplicationStatusUpdateHandler.go +++ b/client/cron/CdApplicationStatusUpdateHandler.go @@ -93,8 +93,6 @@ func NewCdApplicationStatusUpdateHandlerImpl(logger *zap.SugaredLogger, appServi func (impl *CdApplicationStatusUpdateHandlerImpl) Subscribe() error { callback := func(msg *pubsub.PubSubMsg) { - impl.logger.Debug("received argo pipeline status update request") - //defer msg.Ack() statusUpdateEvent := pipeline.ArgoPipelineStatusSyncEvent{} err := json.Unmarshal([]byte(string(msg.Data)), &statusUpdateEvent) if err != nil { diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index a24c1123af..28838b9eb9 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,8 +1,7 @@ // Code generated by Wire. DO NOT EDIT. //go:generate go run github.com/google/wire/cmd/wire -//go:build !wireinject -// +build !wireinject +//+build !wireinject package main @@ -35,7 +34,6 @@ import ( "github.com/devtron-labs/devtron/client/telemetry" repository4 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/apiToken" @@ -151,8 +149,7 @@ func InitializeApp() (*App, error) { v := informer.NewGlobalMapClusterNamespace() k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, runtimeConfig) clusterServiceImpl := cluster.NewClusterServiceImpl(clusterRepositoryImpl, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) - appStatusRepositoryImpl := appStatus.NewAppStatusRepositoryImpl(db, sugaredLogger) - environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db, sugaredLogger, appStatusRepositoryImpl) + environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db) environmentServiceImpl := cluster.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImpl, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthServiceImpl) chartRepoRepositoryImpl := chartRepoRepository.NewChartRepoRepositoryImpl(db) acdAuthConfig, err := util3.GetACDAuthConfig() @@ -196,7 +193,7 @@ func InitializeApp() (*App, error) { serverDataStoreServerDataStore := serverDataStore.InitServerDataStore() appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) pipelineRepositoryImpl := pipelineConfig.NewPipelineRepositoryImpl(db, sugaredLogger) - helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl) + helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl) attributesRepositoryImpl := repository4.NewAttributesRepositoryImpl(db) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) @@ -211,7 +208,7 @@ func InitializeApp() (*App, error) { terminalSessionHandlerImpl := terminal.NewTerminalSessionHandlerImpl(environmentServiceImpl, clusterServiceImpl, sugaredLogger) ciPipelineRepositoryImpl := pipelineConfig.NewCiPipelineRepositoryImpl(db, sugaredLogger) enforcerUtilImpl := rbac.NewEnforcerUtilImpl(sugaredLogger, teamRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, clusterRepositoryImpl) - k8sApplicationRestHandlerImpl := k8s.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, clusterServiceImpl, helmAppServiceImpl, userServiceImpl) + k8sApplicationRestHandlerImpl := k8s.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl) k8sApplicationRouterImpl := k8s.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) chartRefRepositoryImpl := chartRepoRepository.NewChartRefRepositoryImpl(db) refChartDir := _wireRefChartDirValue diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 251f066bf0..ec72865b5e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -53,7 +53,7 @@ * [Job and Cronjob](user-guide/creating-application/deployment-template/job-and-cronjob.md) * [Workflow Overview](user-guide/creating-application/workflow/README.md) * [CI Pipeline](user-guide/creating-application/workflow/ci-pipeline.md) - * [Pre-Build/Post-Build Tasks](user-guide/creating-application/workflow/ci-build-pre-post-plugins.md) + * [Pre-Build/Post-Build Stages](user-guide/creating-application/workflow/ci-build-pre-post-plugins.md) * [Override Build Configuration](user-guide/creating-application/container-registry-override.md) * [CI Pipeline (Legacy)](user-guide/creating-application/workflow/ci-pipeline-legacy.md) * [CD Pipeline](user-guide/creating-application/workflow/cd-pipeline.md) diff --git a/docs/setup/install/Install-devtron-on-Minikube-Microk8s-K3s-Kind.md b/docs/setup/install/Install-devtron-on-Minikube-Microk8s-K3s-Kind.md index 90d13c5a59..2a0b208b09 100644 --- a/docs/setup/install/Install-devtron-on-Minikube-Microk8s-K3s-Kind.md +++ b/docs/setup/install/Install-devtron-on-Minikube-Microk8s-K3s-Kind.md @@ -79,7 +79,9 @@ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d ``` -### For Devtron version less than v0.6.0 + +
+For Devtron version less than v0.6.0 **Username**: `admin`
**Password**: Run the following command to get the admin password: @@ -88,6 +90,7 @@ kubectl -n devtroncd get secret devtron-secret \ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ACD_PASSWORD}' | base64 -d ``` +
## Install Devtron on Cloud VM (AWS ec2, Azure VM, GCP VM) diff --git a/docs/setup/install/ingress-setup.md b/docs/setup/install/ingress-setup.md index efa7001bae..cdd78890ac 100644 --- a/docs/setup/install/ingress-setup.md +++ b/docs/setup/install/ingress-setup.md @@ -1,19 +1,20 @@ # Ingress Setup After Devtron is installed, Devtron is accessible through service `devtron-service`. -If you want to access devtron through ingress, edit devtron-service and change the loadbalancer to ClusterIP. You can do this using `kubectl patch` command like : +If you want to access Devtron through ingress, edit `devtron-service` and change the loadbalancer to ClusterIP. You can do this using `kubectl patch` command: ```bash kubectl patch -n devtroncd svc devtron-service -p '{"spec": {"ports": [{"port": 80,"targetPort": "devtron","protocol": "TCP","name": "devtron"}],"type": "ClusterIP","selector": {"app": "devtron"}}}' ``` -After that create ingress by applying the ingress yaml file. -You can use [this yaml file](https://github.com/devtron-labs/devtron/blob/main/manifests/yamls/devtron-ingress.yaml) to create ingress to access devtron: +After this, create ingress by applying the ingress yaml file. +You can use [this yaml file](https://github.com/devtron-labs/devtron/blob/main/manifests/yamls/devtron-ingress.yaml) to create ingress to access Devtron: ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: + annotations: nginx.ingress.kubernetes.io/app-root: /dashboard labels: app: devtron release: devtron @@ -47,12 +48,13 @@ spec: pathType: ImplementationSpecific ``` -You can access devtron from any host after applying this yaml. For k8s versions <1.19, [apply this yaml](https://github.com/devtron-labs/devtron/blob/main/manifests/yamls/devtron-ingress-legacy.yaml): +You can access Devtron from any host after applying this yaml. For k8s versions <1.19, [apply this yaml](https://github.com/devtron-labs/devtron/blob/main/manifests/yamls/devtron-ingress-legacy.yaml): ```yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: + annotations: nginx.ingress.kubernetes.io/app-root: /dashboard labels: app: devtron release: devtron @@ -73,12 +75,13 @@ spec: pathType: ImplementationSpecific ``` -Optionally you also can access devtron through a specific host like : +Optionally, you also can access Devtron through a specific host by running the following YAML file: ```yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: + annotations: nginx.ingress.kubernetes.io/app-root: /dashboard labels: app: devtron release: devtron @@ -120,7 +123,7 @@ Once ingress setup for devtron is done and you want to run Devtron over `https`, ### 1. Nginx Ingress Controller -In case of `nginx ingress controller`, add following annotations under `service.annotations` under nginx ingress controller to run devtron over `https`. +In case of `nginx ingress controller`, add the following annotations under `service.annotations` under nginx ingress controller to run devtron over `https`. (i) Amazon Web Services (AWS) diff --git a/docs/setup/install/install-devtron-with-cicd-with-gitops.md b/docs/setup/install/install-devtron-with-cicd-with-gitops.md index 0edd053dae..e1141d48e8 100644 --- a/docs/setup/install/install-devtron-with-cicd-with-gitops.md +++ b/docs/setup/install/install-devtron-with-cicd-with-gitops.md @@ -238,7 +238,9 @@ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d ``` -### For Devtron version less than v0.6.0 + +
+For Devtron version less than v0.6.0 **Username**: `admin`
**Password**: Run the following command to get the admin password: @@ -247,10 +249,12 @@ kubectl -n devtroncd get secret devtron-secret \ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ACD_PASSWORD}' | base64 -d ``` +
+ * If you want to uninstall Devtron or clean Devtron helm installer, refer our [uninstall Devtron](https://docs.devtron.ai/install/uninstall-devtron). -* Related to installaltion, please also refer [FAQ](https://docs.devtron.ai/install/faq-on-installation) section also. +* Related to installation, please also refer [FAQ](https://docs.devtron.ai/install/faq-on-installation) section also. **Note**: If you have questions, please let us know on our discord channel. [![Join Discord](https://img.shields.io/badge/Join%20us%20on-Discord-e01563.svg)](https://discord.gg/jsRG5qx2gp) diff --git a/docs/setup/install/install-devtron-with-cicd.md b/docs/setup/install/install-devtron-with-cicd.md index 6f1e5cb5f7..5d16c956ab 100644 --- a/docs/setup/install/install-devtron-with-cicd.md +++ b/docs/setup/install/install-devtron-with-cicd.md @@ -245,7 +245,8 @@ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d ``` -### For Devtron version less than v0.6.0 +
+For Devtron version less than v0.6.0 **Username**: `admin`
**Password**: Run the following command to get the admin password: @@ -254,6 +255,8 @@ kubectl -n devtroncd get secret devtron-secret \ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ACD_PASSWORD}' | base64 -d ``` +
+ * If you want to uninstall Devtron or clean Devtron helm installer, refer our [uninstall Devtron](https://docs.devtron.ai/install/uninstall-devtron). diff --git a/docs/setup/install/install-devtron.md b/docs/setup/install/install-devtron.md index dee5e555fc..c1dcf080be 100644 --- a/docs/setup/install/install-devtron.md +++ b/docs/setup/install/install-devtron.md @@ -81,8 +81,9 @@ The hostname `aaff16e9760594a92afa0140dbfd99f7-305259315.us-east-1.elb.amazonaws kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ADMIN_PASSWORD}' | base64 -d ``` - -### For Devtron version less than v0.6.0 + +
+For Devtron version less than v0.6.0 **Username**: `admin`
**Password**: Run the following command to get the admin password: @@ -91,6 +92,8 @@ kubectl -n devtroncd get secret devtron-secret \ kubectl -n devtroncd get secret devtron-secret \ -o jsonpath='{.data.ACD_PASSWORD}' | base64 -d ``` +
+ **Note**: If you want to uninstall Devtron or clean Devtron helm installer, refer our [uninstall Devtron](https://docs.devtron.ai/install/uninstall-devtron). diff --git a/docs/user-guide/creating-application/overview.md b/docs/user-guide/creating-application/overview.md index c962180580..0f5ccb0ded 100644 --- a/docs/user-guide/creating-application/overview.md +++ b/docs/user-guide/creating-application/overview.md @@ -47,4 +47,63 @@ Click **Save**. The application will be moved to the selected project. * To remove the tags from propagation, click the symbol again. * Click `Save`. -The changes in the tags will be reflected in the `Tags` on the `Overview` section. \ No newline at end of file +The changes in the tags will be reflected in the `Tags` on the `Overview` section. + + +## Configure PersistentVolumeClaim (PVC) for Build Time Optimization + + A PersistentVolumeClaim (PVC) volume is a request for storage, which is used to mount a PersistentVolume (PV) into a Pod. In order to optimize build time, you can configure PVC in your application. + +If you want to optimize build time for the multiple target platforms (e.g., arm64, amd64), mounting a PVC will provide volume directly to a pod which helps in shorter build time by storing build cache. Mounting a PVC into a pod will provide storage for build cache which will not impact the normal build where the image is built on the basis of architecture and operating system of the K8s node on which CI is running. + +### Create PVC file + +* The following configuration file describes persistent volume claim e.g.,`cache-pvc.yaml`, where you have to define the metadata `name` and `storageClassname`. + +```bash +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cache-pvc # here comes the name of PVC +spec: + accessModes: + - ReadWriteOnce + storageClassName: # here comes storage class name + resources: + requests: + storage: 30Gi +``` + +* Create the PersistentVolumeClaim by running the following command: + +```bash +kubectl apply -f https://k8s.io/examples/pods/storage/pv-claim.yaml -n {namespace} +``` + +For more detail, refer [Kubernetes PVC](https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/#create-a-persistentvolumeclaim). + + +## Configure PVC + +In order to configure PVC: +* Go to the `Overview` section of your application. +* On the right-corner, click `Edit Tags`. + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/overview/pvc-edit-tags.jpg) + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/overview/manage-tags-pvc.jpg) + +* For app level PVC mounting, enter the following:
  • key:`devtron.ai/ci-pvc-all`
  • value: metadata name (e.g., `cache-pvc)` which you define on the [PVC template](#create-pvc-file).
`Note`: This PVC mounting will impact all the build pipilines of the application. +* For pipeline level, enter the following:
  • key:`devtron.ai/ci-pvc-{pipelinename}`
  • value: metadata name which you define on the [PVC template](#create-pvc-file).
`Note`: This PVC mounting will impact only the particular build pipeline. + +To know the `pipelinename` detail, go to the `App Configutation`, click `Workflow Editor` the pipeline name will be on the `Build` pipeline as shown below. + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/images/creating-application/overview/pipeline-name-pvc.jpg) + +* Click `Save`. + + + + + + diff --git a/docs/user-guide/creating-application/workflow/ci-build-pre-post-plugins.md b/docs/user-guide/creating-application/workflow/ci-build-pre-post-plugins.md index 84fe464a09..5cbafd4e01 100644 --- a/docs/user-guide/creating-application/workflow/ci-build-pre-post-plugins.md +++ b/docs/user-guide/creating-application/workflow/ci-build-pre-post-plugins.md @@ -1,73 +1,294 @@ -# Pre-Build and Post-Build stages +# Pre-Build and Post-Build Stages The CI pipeline includes Pre and Post-build steps to validate and introduce checkpoints in the build process. +The pre/post plugins allow you to execute some standard tasks, such as Code analysis, Load testing, Security scanning etc. You can build custom pre-build/post-build tasks or select one of the standard preset plugins provided by Devtron. -The pre/post plugins allow you to execute some standard tasks, such as Code analysis, Load testing, Security scanning, and so on. -You can build custom pre/post tasks or use one from the standard preset plugins provided by Devtron. +Preset plugin is an API resource which you can add within the CI build environment. By integrating the preset plugin in your application, it helps your development cycle to keep track of finding bugs, code duplication, code complexity, load testing, security scanning etc. You can analyze your code easily. + +> Devtron CI pipeline includes the following build stages: +> +> * Pre-Build Stage: The tasks in this stage run before the image is built. +> * Build Stage: In this stage, the build is triggered from the source code (container image) that you provide. +> * Post-Build Stage: The tasks in this stage are triggered once the build is complete. ## Before you begin -Create a [CI build pipeline](./ci-pipeline.md) if you haven't done that already! +Make sure you have [CI build pipeline](./ci-pipeline.md) before you start configuring Pre-Build or Post-Build tasks. -## Configuring Pre/Post-build stages +## Configuring Pre/Post-build Tasks Each Pre/Post-build stage is executed as a series of events called tasks and includes custom scripts. -You could create one or more tasks that are dependent on one another for execution. In other words, the output variable of one task can be used as an input for the next task to build a CI runner. +You can create one or more tasks that are dependent on one another for execution. In other words, the output variable of one task can be used as an input for the next task to build a CI runner. The tasks will run following the execution order. -The tasks will run following the execution order. +The tasks can be re-arranged by drag-and-drop; however, the order of passing the variables must be followed. -> The tasks may be re-arranged by using drag-and-drop; however, the order of passing the variables must be followed. +You can create a task either by selecting one of the available preset plugins or by creating a custom script. | Stage | Task | | :--- | :--- | -| Pre-Build/Post-Build |
  1. Create a task from - [Preset Plugin](#preset-plugins)
    • Sonarqube
    • K6 Load testing
  2. Create a task from - [Execute Custom script](#execute-custom-script)
    • [Custom script - Shell](#custom-script-shell)
    • [Custom script - Container image](#custom-script-container-image)
| +| Pre-Build/Post-Build |
  1. Create a task using one of the [Preset Plugins](#preset-plugins) integrated in Devtron:
    • [K6 Load testing](#k6-load-testing)
    • [Sonarqube](#sonarqube)
    • [Dependency track for Python](#dependency-track-for-python)
    • [Dependency track for NodeJs](#dependency-track-for-nodejs)
    • [Dependency track for Marven and Gradle](#dependency-track-for-maven--gradle)
    • [Semgrep](#semgrep)
    • [Codacy](#codacy)
  2. Create a task from [Execute Custom script](#execute-custom-script) which you can customize your script with:
    • [Custom script - Shell](#custom-script-shell)
    • Or, [Custom script - Container image](#custom-script-container-image)
| -## Creating a task -1. Go to **Applications** and select your application from the **Devtron Apps** tabs. -2. From the **App Configuration** tab select **Workflow Editor**. -3. Select the build pipeline for editing the stages. +## Creating Pre/Post-build Tasks -> Devtron CI pipeline includes the following build stages: -> -> * Pre-build stage: The tasks in this stage run before the image is built. -> * Build stage: In this stage, the build is triggered from the source code that you provide. -> * Post-build stage: The tasks in this stage are triggered once the build is complete. +Lets take `Codacy` as an example and configure it in the Pre-Build stage in the CI pipeline for finding bugs, detecting dependency vulnerabilities, and enforcing code standards. + +* Go to the **Applications** and select your application from the **Devtron Apps** tabs. + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/plugins/plugins-latest/applications-app.jpg) + + +* Go to the **App Configuration** tab, click **Workflow Editor**. + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/plugins/plugins-latest/app-configuration.jpg) + + +* Select the build pipeline for configuring the pre/post-build tasks. +* On the **Edit build pipeline**, in the `Pre-Build Stage`, click **+ Add task**. + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/plugins/plugins-latest/add-task-pre-build-stage.jpg) + + +* Select **Codacy** from **PRESET PLUGINS**. +* Enter a relevant name or codacy in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field.
`Note`: The description is available by default. +* In the **Input Variables**, provide the information in the following fields: + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/plugins/plugins-latest/codacy-details-pre-build-stage.jpg) + + | Variable | Format | Description | +| ---- | ---- | ---- | +| CodacyEndpoint | String | API endpoint for Codacy. | +| GitProvider | String | Git provider for the scanning. | +| CodacyApiToken | String | API token for Codacy. If it is provided, it will be used, otherwise it will be picked from Global secret (CODACY_API_TOKEN). | +| Organisation | String | Your Organisation for Codacy.| +| RepoName | String | Your Repository name. | +| Branch | String | Your branch name. | + +* In `Trigger/Skip Condition`, set the trigger conditions to execute a task or `Set skip conditions`. As an exmple: CodacyEndpoint equal to https://app.codacy.com.
`Note`: You can set more than one condition. + +* In `Pass/Failure Condition` set the conditions to execute pass or fail of your build. As an example: Pass if number of issues equal to zero.
`Note`: You can set more than one condition. + +* Click **Update Pipeline**. + +* Go to the **Build & Deploy**, click the build pipiline and start your build. + +* Click `Details` on the build pipeline and you can view the details on the `Logs`. -You can create a task either by selecting one of the available preset plugins or by creating a custom script. -[Preset plugins](#preset-plugins) | [Execute custom script](#execute-custom-script) ### Preset plugins -**Prerequisite**: Set up Sonarqube, or get the API keys from an admin. +#### K6 Load Testing -The example shows a Post-build stage with a task created using a preset plugin - Sonarqube. +K6 is an open-source tool and cloud service that makes load testing easy for developers and QA engineers. -1. On the **Edit build pipeline** screen, select the **Post-build stage** (or Pre-build). -2. Select **+ Add task**. -3. Select **Sonarqube** from **PRESET PLUGINS**. +**Prerequisite**: Make sure you have set up an account in `k6.io` or get the API keys from an admin. -![Preset plugin - Sonarqube](https://devtron-public-asset.s3.us-east-2.amazonaws.com/plugins/preset-plugin-sonarqube-2.png) +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **K6 Load Testing** from **PRESET PLUGINS**. -| Field name | Required/Optional | Field description | -| --- | --- | --- | -| Task name | Required | A relevant name for the task | -| Description | Optional | A descriptive message for the task | -| Input variables | Optional | VALUE: A value for the input variable. The value may be any of the values from the previous build stages, a global variable, or a custom value | -| Trigger/Skip Condition | Optional | A conditional statement to execute or skip the task | +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
+ + | Variable | Format | Description | +| ---- | ---- | ---- | +| RelativePathToScript | String | Checkout path + script path along with script name. | +| PrometheusUsername | String | Username of Prometheus account. | +| PrometheusApiKey | String | API key of Prometheus account. | +| PrometheusRemoteWriteEndpoint | String | Remote write endpoint of Prometheus account. | +| OutputType | String | `Log` or `Prometheus` | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* Click **Update Pipeline**. + +#### Sonarqube + +Configuring `Sonarqube` in pre-build or post build task enhances your workflow with Continuous Code Quality & Code Security. + +**Prerequisite**: Make sure you have set up an account in `Sonarqube` or get the API keys from an admin. + +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **Sonarqube** from **PRESET PLUGINS**. + +![](https://devtron-public-asset.s3.us-east-2.amazonaws.com/plugins/preset-plugin-sonarqube-2.png) -#### Input Variables +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
-| Variable | Format | Description | + | Variable | Format | Description | | ---- | ---- | ---- | | SonarqubeProjectKey | String | Project key of sonarqube account. | -| SonarqubeApiKey | String | Api key of sonarqube account. | -| SonarqubeEndpoint | String | Api endpoint of sonarqube account. | -| CheckoutPath | String | Checkout path of git material. | +| SonarqubeApiKey | String | API key of Sonarqube account. | +| SonarqubeEndpoint | String | API endpoint of Sonarqube account. | +| CheckoutPath | String | Checkout path of Git material. | +| UsePropertiesFileFromProject | Boolean | Enter either `true` or `false` accordingly whether you want the poll to generate report or not. | +| CheckForSonarAnalysisReport | Boolean | Enter either `true` or `false` accordingly whether you want the poll to generate report or not. | +| AbortPipelineOnPolicyCheckFailed | Boolean | Enter either `true` or `false` accordingly whether you want to check if the policy fails or not. | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* Click **Update Pipeline**. + + +#### Dependency Track for Python + +Configuring `Dependency Track for Python` in pre-build or post build task creates a bill of materials from Python projects and environments and uploads it to D-track for [Component Analysis](https://owasp.org/www-community/Component_Analysis) to identify and reduce risk in the software supply chain. + +**Prerequisite**: Make sure you have set up an account in `dependency track` or get the API keys from an admin. + +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **Dependency track for Python** from **PRESET PLUGINS**. + + + +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
+ + | Variable | Format | Description | +| ---- | ---- | ---- | +| ProjectManifestType | String | Type of your Python project manifest which is used to build cycloneDx [Software Bill of Materials (SBOM)](https://owasp.org/www-community/Component_Analysis#software-bill-of-materials-sbom). E.g., PIP, Poetry etc. | +| RelativePathToPoetryLock | String | Path to your poetry.lock file inside your project. | +| RelativePathToPipfile | String | Path to your Pipfile.lock file inside your project. | +| RelativePathToRequirementTxt | String | Path to your requirements.txt file inside your project. | +| DTrackEndpoint | String | API endpoint of your dependency track account. | +| DTrackProjectName | String | Name of your dependency track project. | +| DTrackProjectVersion | String | Version of dependency track project. | +| DTrackApiKey | String | API key of your dependency track account. | +| CheckoutPath | String | Checkout path of Git material. | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* Click **Update Pipeline**. + + +#### Dependency Track for NodeJs + +Configuring `Dependency Track for NodeJs` in pre-build or post build task creates a bill of materials from NodeJs projects and environments and uploads it to D-track for [Component Analysis](https://owasp.org/www-community/Component_Analysis) to identify and reduce risk in the software supply chain. + + +**Prerequisite**: Make sure you have set up an account in `dependency track` or get the API keys from an admin. + +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **Dependency track for NodeJs** from **PRESET PLUGINS**. + + + +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
+ + | Variable | Format | Description | +| ---- | ---- | ---- | +| DTrackEndpoint | String | API endpoint of your dependency track account. | +| DTrackProjectName | String | Name of your dependency track project. | +| DTrackProjectVersion | String | Version of dependency track project. | +| DTrackApiKey | String | API key of your dependency track account. | +| CheckoutPath | String | Checkout path of Git material. | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* Click **Update Pipeline**. + + + +#### Dependency Track for Maven & Gradle + +Configuring `Dependency Track for NodeJs` in pre-build or post build task creates a bill of materials from Maven & Gradle projects and environments and uploads it to D-track for [Component Analysis](https://owasp.org/www-community/Component_Analysis) to identify and reduce risk in the software supply chain. + + +**Prerequisite**: Make sure you have set up an account in `dependency track` or get the API keys from an admin. + +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **Dependency track for Maven & Gradle** from **PRESET PLUGINS**. + + +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
+ + | Variable | Format | Description | +| ---- | ---- | ---- | +| BuildToolType | String | Type of build tool your project is using. E.g., Maven, or Gradle. | +| DTrackEndpoint | String | API endpoint of your dependency track account. | +| DTrackProjectName | String | Name of your dependency track project. | +| DTrackProjectVersion | String | Version of dependency track project. | +| DTrackApiKey | String | API key of your dependency track account. | +| CheckoutPath | String | Checkout path of Git material. | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* Click **Update Pipeline**. + + +#### Semgrep + +Semgrep is a fast, open source, static analysis engine for finding bugs, detecting dependency vulnerabilities, and enforcing code standards. + +**Prerequisite**: Make sure you have set up an account in `Semgrep` or get the API keys from an admin. + +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **Semgrep** from **PRESET PLUGINS**. + + +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
+ + | Variable | Format | Description | +| ---- | ---- | ---- | +| SemgrepAppToken | String | App token of Semgrep. If it is provided, this token will be used, otherwise it will be picked from Global Secret. | +| PrefixAppNameInSemgrepBranchName | Bool | Enter either `true` or `false` accordingly whether you want app name to be reflected with a branch name. If it is `true`, it will add app name with branch name. E.g., {SemgrepAppName}-{branchName} | +| UseCommitAsSemgrepBranchName | Bool | Enter either `true` or `false` accordingly whether you want app name to be reflected with commit hash. If it is `true`, it will add app name with commit hash. E.g., {SemgrepAppName}-{CommitHash}. | +| SemgrepAppName | String | App name for Semgrep. If it is provided, and `PrefixAppNameInSemgrepBranchName` is true, then this will be prefixed with branch name/commit hash.| +| ExtraCommandArguments | String | Extra command arguments for Semgrep CI command. E.g., Input: --json --dry-run. | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* Click **Update Pipeline**. + + +#### Codacy + +Codacy is an automated code analysis/quality tool that helps developers to ship better software in a faster manner. + +**Prerequisite**: Make sure you have set up an account in `Codacy` or get the API keys from an admin. + +1. On the **Edit build pipeline** page, select the **Pre-Build Stage** (or Post-Build Stage). +2. Click **+ Add task**. +3. Select **Codacy** from **PRESET PLUGINS**. + + +* Enter a relevant name in the `Task name` field. It is a mandatory field. +* Enter a descriptive message for the task in the `Description` field. It is an optional field. +* Provide a value for the input variable.
Note: The value may be any of the values from the previous build stages, a global variable, or a custom value.
+ + | Variable | Format | Description | +| ---- | ---- | ---- | +| CodacyEndpoint | String | API endpoint for Codacy. | +| GitProvider | String | Git provider for the scanning. | +| CodacyApiToken | String | API token for Codacy. If it is provided, it will be used, otherwise it will be picked from Global secret (CODACY_API_TOKEN). | +| Organisation | String | Your Organisation for Codacy.| +| RepoName | String | Your Repository name. | +| Branch | String | Your branch name. | + +* `Trigger/Skip Condition` refers to a conditional statement to execute or skip the task. You can select either:
  • `Set trigger conditions` or
  • `Set skip conditions`
+ +* `Pass/Failure Condition` refers to conditions to execute pass or fail of your build. You can select either:
  • `Set pass conditions` or
  • `Set failure conditions`
+ +* Click **Update Pipeline**. -Select **Update Pipeline**. ### Execute custom script diff --git a/internal/sql/repository/pipelineConfig/AppLabelsRepository.go b/internal/sql/repository/pipelineConfig/AppLabelsRepository.go index af39d379e2..f64b0c5bb1 100644 --- a/internal/sql/repository/pipelineConfig/AppLabelsRepository.go +++ b/internal/sql/repository/pipelineConfig/AppLabelsRepository.go @@ -31,6 +31,7 @@ type AppLabel struct { AppId int `sql:"app_id,notnull"` Key string `sql:"key,notnull"` Value string `sql:"value,notnull"` + Propagate bool `sql:"propagate,notnull"` App app.App sql.AuditLog } diff --git a/internal/util/K8sUtil.go b/internal/util/K8sUtil.go index f60faac829..867dfa8265 100644 --- a/internal/util/K8sUtil.go +++ b/internal/util/K8sUtil.go @@ -23,8 +23,10 @@ import ( error2 "errors" "flag" "fmt" + "github.com/argoproj/gitops-engine/pkg/utils/kube" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "net/http" "os/user" "path/filepath" @@ -546,11 +548,12 @@ func (impl K8sUtil) GetPodByName(namespace string, name string, client *v12.Core } } -func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.UnstructuredList, namespaced bool, kind string, validateResourceAccess func(namespace, resourceName string) bool) (*ClusterResourceListMap, error) { +func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.UnstructuredList, namespaced bool, gvk schema.GroupVersionKind, validateResourceAccess func(namespace string, group string, kind string, resourceName string) bool) (*ClusterResourceListMap, error) { clusterResourceListMap := &ClusterResourceListMap{} // build headers var headers []string columnIndexes := make(map[int]string) + kind := gvk.Kind if kind == "Event" { headers, columnIndexes = impl.getEventKindHeader() } else { @@ -587,13 +590,11 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu // build rows rowsMapping := make([]map[string]interface{}, 0) rowsDataUncast := manifest.Object[K8sClusterResourceRowsKey] - var resourceName string var namespace string var allowed bool if rowsDataUncast != nil { rows := rowsDataUncast.([]interface{}) for _, row := range rows { - resourceName = "" namespace = "" allowed = true rowIndex := make(map[string]interface{}) @@ -614,11 +615,10 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu rowIndex[columnName] = cell } - // set namespace - cellObjUncast := rowMap[K8sClusterResourceObjectKey] + var cellObj map[string]interface{} if cellObjUncast != nil { - cellObj := cellObjUncast.(map[string]interface{}) + cellObj = cellObjUncast.(map[string]interface{}) if cellObj != nil && cellObj[K8sClusterResourceMetadataKey] != nil { metadata := cellObj[K8sClusterResourceMetadataKey].(map[string]interface{}) if metadata[K8sClusterResourceNamespaceKey] != nil { @@ -627,14 +627,9 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu rowIndex[K8sClusterResourceNamespaceKey] = namespace } } - if metadata[K8sClusterResourceMetadataNameKey] != nil { - resourceName = metadata[K8sClusterResourceMetadataNameKey].(string) - } } } - if resourceName != "" { - allowed = validateResourceAccess(namespace, resourceName) - } + allowed = impl.ValidateResource(cellObj, gvk, validateResourceAccess) if allowed { rowsMapping = append(rowsMapping, rowIndex) } @@ -647,6 +642,89 @@ func (impl K8sUtil) BuildK8sObjectListTableData(manifest *unstructured.Unstructu return clusterResourceListMap, nil } +func (impl K8sUtil) ValidateResource(resourceObj map[string]interface{}, gvk schema.GroupVersionKind, validateCallback func(namespace string, group string, kind string, resourceName string) bool) bool { + resKind := gvk.Kind + groupName := gvk.Group + metadata := resourceObj[K8sClusterResourceMetadataKey] + if metadata == nil { + return false + } + metadataMap := metadata.(map[string]interface{}) + var namespace, resourceName string + var ownerReferences []interface{} + if metadataMap[K8sClusterResourceNamespaceKey] != nil { + namespace = metadataMap[K8sClusterResourceNamespaceKey].(string) + } + if metadataMap[K8sClusterResourceMetadataNameKey] != nil { + resourceName = metadataMap[K8sClusterResourceMetadataNameKey].(string) + } + if metadataMap[K8sClusterResourceOwnerReferenceKey] != nil { + ownerReferences = metadataMap[K8sClusterResourceOwnerReferenceKey].([]interface{}) + } + if len(ownerReferences) > 0 { + for _, ownerRef := range ownerReferences { + allowed := impl.validateForResource(namespace, ownerRef, validateCallback) + if allowed { + return allowed + } + } + } + // check current RBAC in case not matched with above one + return validateCallback(namespace, groupName, resKind, resourceName) +} + +func (impl K8sUtil) validateForResource(namespace string, resourceRef interface{}, validateCallback func(namespace string, group string, kind string, resourceName string) bool) bool { + resourceReference := resourceRef.(map[string]interface{}) + resKind := resourceReference[K8sClusterResourceKindKey].(string) + apiVersion := resourceReference[K8sClusterResourceApiVersionKey].(string) + groupName := "" + if strings.Contains(apiVersion, "/") { + groupName = apiVersion[:strings.LastIndex(apiVersion, "/")] // extracting group from this apiVersion + } + resName := "" + if resourceReference["name"] != "" { + resName = resourceReference["name"].(string) + switch resKind { + case kube.ReplicaSetKind: + // check deployment first, then RO and then RS + if strings.Contains(resName, "-") { + deploymentName := resName[:strings.LastIndex(resName, "-")] + allowed := validateCallback(namespace, groupName, kube.DeploymentKind, deploymentName) + if allowed { + return true + } + allowed = validateCallback(namespace, K8sClusterResourceRolloutGroup, K8sClusterResourceRolloutKind, deploymentName) + if allowed { + return true + } + } + allowed := validateCallback(namespace, groupName, resKind, resName) + if allowed { + return true + } + case kube.JobKind: + // check CronJob first, then Job + if strings.Contains(resName, "-") { + cronJobName := resName[:strings.LastIndex(resName, "-")] + allowed := validateCallback(namespace, groupName, K8sClusterResourceCronJobKind, cronJobName) + if allowed { + return true + } + } + allowed := validateCallback(namespace, groupName, resKind, resName) + if allowed { + return true + } + case kube.DeploymentKind, K8sClusterResourceCronJobKind, kube.StatefulSetKind, kube.DaemonSetKind, K8sClusterResourceRolloutKind, K8sClusterResourceReplicationControllerKind: + allowed := validateCallback(namespace, groupName, resKind, resName) + if allowed { + return true + } + } + } + return false +} + func (impl K8sUtil) getEventKindHeader() ([]string, map[int]string) { headers := []string{"type", "message", "namespace", "involved object", "source", "count", "age", "last seen"} columnIndexes := make(map[int]string) diff --git a/internal/util/K8sUtilBean.go b/internal/util/K8sUtilBean.go index 884e2b0c75..942ed2bc98 100644 --- a/internal/util/K8sUtilBean.go +++ b/internal/util/K8sUtilBean.go @@ -1,5 +1,10 @@ package util +import ( + "github.com/argoproj/gitops-engine/pkg/utils/kube" + "k8s.io/apimachinery/pkg/runtime/schema" +) + type ClusterResourceListMap struct { Headers []string `json:"headers"` Data []map[string]interface{} `json:"data"` @@ -10,8 +15,32 @@ const K8sClusterResourcePriorityKey = "priority" const K8sClusterResourceNamespaceKey = "namespace" const K8sClusterResourceMetadataKey = "metadata" const K8sClusterResourceMetadataNameKey = "name" +const K8sClusterResourceOwnerReferenceKey = "ownerReferences" +const K8sClusterResourceCreationTimestampKey = "creationTimestamp" const K8sClusterResourceRowsKey = "rows" const K8sClusterResourceCellKey = "cells" const K8sClusterResourceColumnDefinitionKey = "columnDefinitions" const K8sClusterResourceObjectKey = "object" + +const K8sClusterResourceKindKey = "kind" +const K8sClusterResourceApiVersionKey = "apiVersion" + +const K8sClusterResourceRolloutKind = "Rollout" +const K8sClusterResourceRolloutGroup = "argoproj.io" +const K8sClusterResourceReplicationControllerKind = "ReplicationController" +const K8sClusterResourceCronJobKind = "CronJob" +const V1VERSION = "v1" +const BatchGroup = "batch" +const AppsGroup = "apps" + +var KindVsChildrenGvk = map[string][]schema.GroupVersionKind{ + kube.DeploymentKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Group: AppsGroup, Version: V1VERSION, Kind: kube.ReplicaSetKind}, schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + K8sClusterResourceRolloutKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Group: AppsGroup, Version: V1VERSION, Kind: kube.ReplicaSetKind}, schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + K8sClusterResourceCronJobKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Group: BatchGroup, Version: V1VERSION, Kind: kube.JobKind}, schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + kube.JobKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + kube.ReplicaSetKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + kube.DaemonSetKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + kube.StatefulSetKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), + K8sClusterResourceReplicationControllerKind: append(make([]schema.GroupVersionKind, 0), schema.GroupVersionKind{Version: V1VERSION, Kind: kube.PodKind}), +} diff --git a/pkg/app/AppCrudOperationService.go b/pkg/app/AppCrudOperationService.go index f9c40b9f66..e07aa4bdee 100644 --- a/pkg/app/AppCrudOperationService.go +++ b/pkg/app/AppCrudOperationService.go @@ -25,9 +25,9 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/user/repository" + util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/util/validation" "strconv" "strings" "time" @@ -78,6 +78,19 @@ type TeamAppBean struct { } func (impl AppCrudOperationServiceImpl) UpdateApp(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { + // validate the labels key-value if propagate is true + for _, label := range request.AppLabels { + if !label.Propagate { + continue + } + labelKey := label.Key + labelValue := label.Value + err := util2.CheckIfValidLabel(labelKey, labelValue) + if err != nil { + return nil, err + } + } + dbConnection := impl.appRepository.GetConnection() tx, err := dbConnection.Begin() if err != nil { @@ -154,9 +167,10 @@ func (impl AppCrudOperationServiceImpl) Create(request *bean.AppLabelDto, tx *pg } if err == pg.ErrNoRows { model := &pipelineConfig.AppLabel{ - Key: request.Key, - Value: request.Value, - AppId: request.AppId, + Key: request.Key, + Value: request.Value, + Propagate: request.Propagate, + AppId: request.AppId, } model.CreatedBy = request.UserId model.UpdatedBy = request.UserId @@ -181,20 +195,21 @@ func (impl AppCrudOperationServiceImpl) UpdateLabelsInApp(request *bean.CreateAp } appLabelMap := make(map[string]*pipelineConfig.AppLabel) for _, appLabel := range appLabels { - uniqueLabelExists := fmt.Sprintf("%s:%s", appLabel.Key, appLabel.Value) + uniqueLabelExists := fmt.Sprintf("%s:%s:%t", appLabel.Key, appLabel.Value, appLabel.Propagate) if _, ok := appLabelMap[uniqueLabelExists]; !ok { appLabelMap[uniqueLabelExists] = appLabel } } for _, label := range request.AppLabels { - uniqueLabelRequest := fmt.Sprintf("%s:%s", label.Key, label.Value) + uniqueLabelRequest := fmt.Sprintf("%s:%s:%t", label.Key, label.Value, label.Propagate) if _, ok := appLabelMap[uniqueLabelRequest]; !ok { // create new model := &pipelineConfig.AppLabel{ - Key: label.Key, - Value: label.Value, - AppId: request.Id, + Key: label.Key, + Value: label.Value, + Propagate: label.Propagate, + AppId: request.Id, } model.CreatedBy = request.UserId model.UpdatedBy = request.UserId @@ -230,9 +245,10 @@ func (impl AppCrudOperationServiceImpl) FindById(id int) (*bean.AppLabelDto, err return &bean.AppLabelDto{}, nil } label := &bean.AppLabelDto{ - Key: model.Key, - Value: model.Value, - AppId: model.AppId, + Key: model.Key, + Value: model.Value, + Propagate: model.Propagate, + AppId: model.AppId, } return label, nil } @@ -249,9 +265,10 @@ func (impl AppCrudOperationServiceImpl) FindAll() ([]*bean.AppLabelDto, error) { } for _, model := range models { dto := &bean.AppLabelDto{ - AppId: model.AppId, - Key: model.Key, - Value: model.Value, + AppId: model.AppId, + Key: model.Key, + Value: model.Value, + Propagate: model.Propagate, } results = append(results, dto) } @@ -275,8 +292,9 @@ func (impl AppCrudOperationServiceImpl) GetAppMetaInfo(appId int) (*bean.AppMeta } else { for _, model := range models { dto := &bean.Label{ - Key: model.Key, - Value: model.Value, + Key: model.Key, + Value: model.Value, + Propagate: model.Propagate, } labels = append(labels, dto) } @@ -387,6 +405,11 @@ func (impl AppCrudOperationServiceImpl) GetLabelsByAppIdForDeployment(appId int) labelKey := strings.TrimSpace(label.Key) labelValue := strings.TrimSpace(label.Value) + if !label.Propagate { + impl.logger.Warnw("Ignoring label to propagate to app level as propagation is false", "labelKey", labelKey, "labelValue", labelValue, "appId", appId) + continue + } + // if labelKey or labelValue is empty then don't add in labels if len(labelKey) == 0 || len(labelValue) == 0 { impl.logger.Warnw("Ignoring label to propagate to app level", "labelKey", labelKey, "labelValue", labelValue, "appId", appId) @@ -395,16 +418,9 @@ func (impl AppCrudOperationServiceImpl) GetLabelsByAppIdForDeployment(appId int) // if labelKey is not satisfying the label key criteria don't add in labels // label key must be a 'qualified name' (https://github.com/kubernetes/website/issues/17969) - errs := validation.IsQualifiedName(labelKey) - if len(errs) > 0 { - impl.logger.Warnw("Ignoring label to propagate to app level", "message", fmt.Sprintf("Validation error - label key - %s is not satisfying the label key criteria", labelKey), "appId", appId) - continue - } - - // if labelValue is not satisfying the label value criteria don't add in labels - errs = validation.IsValidLabelValue(labelValue) - if len(errs) > 0 { - impl.logger.Warnw("Ignoring label to propagate to app level", "message", fmt.Sprintf("Validation error - label value - %s is not satisfying the label value criteria", labelValue), "appId", appId) + err = util2.CheckIfValidLabel(labelKey, labelValue) + if err != nil { + impl.logger.Warnw("Ignoring label to propagate to app level", "err", err, "appId", appId) continue } diff --git a/pkg/app/AppListingViewBuilder.go b/pkg/app/AppListingViewBuilder.go index 5beba9156d..287ece62c2 100644 --- a/pkg/app/AppListingViewBuilder.go +++ b/pkg/app/AppListingViewBuilder.go @@ -76,6 +76,11 @@ func (impl *AppListingViewBuilderImpl) BuildView(fetchAppListingRequest FetchApp break } } + + sort.Slice(v, func(i, j int) bool { + return v[i].LastDeployedTime >= v[j].LastDeployedTime + }) + appContainerResponse := &bean.AppContainer{ AppId: appId, AppName: appName, diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 276139e781..b48ec928f9 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -495,7 +495,7 @@ func (impl *AppServiceImpl) UpdateDeploymentStatusForPipeline(app *v1alpha1.Appl } func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(app *v1alpha1.Application, cdWfrId int, statusTime time.Time, triggeredAt time.Time, statusTimeoutDuration int, latestTimelineBeforeUpdate *pipelineConfig.PipelineStatusTimeline) (bool, bool, error) { - impl.logger.Infow("updating pipeline status timeline", "app", app, "pipelineOverride", cdWfrId, "APP_TO_UPDATE", app.Name) + impl.logger.Debugw("updating pipeline status timeline", "app", app, "pipelineOverride", cdWfrId, "APP_TO_UPDATE", app.Name) isTimelineUpdated := false isTimelineTimedOut := false terminalStatusExists, err := impl.pipelineStatusTimelineRepository.CheckIfTerminalStatusTimelinePresentByWfrId(cdWfrId) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 251bbdd83e..f007f52e17 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -582,15 +582,17 @@ type AppLabelsDto struct { } type AppLabelDto struct { - Key string `json:"key,notnull"` - Value string `json:"value,notnull"` - AppId int `json:"appId,omitempty"` - UserId int32 `json:"-"` + Key string `json:"key,notnull"` + Value string `json:"value,notnull"` + Propagate bool `json:"propagate,notnull"` + AppId int `json:"appId,omitempty"` + UserId int32 `json:"-"` } type Label struct { - Key string `json:"key" validate:"required"` - Value string `json:"value" validate:"required"` + Key string `json:"key" validate:"required"` + Value string `json:"value" validate:"required"` + Propagate bool `json:"propagate"` } type AppMetaInfoDto struct { diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 4762e568a1..f7da5bac0c 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -50,7 +50,7 @@ type ClusterBean struct { ServerUrl string `json:"server_url,omitempty" validate:"url,required"` PrometheusUrl string `json:"prometheus_url,omitempty" validate:"validate-non-empty-url"` Active bool `json:"active"` - Config map[string]string `json:"config,omitempty" validate:"required"` + Config map[string]string `json:"config,omitempty"` PrometheusAuth *PrometheusAuth `json:"prometheusAuth,omitempty"` DefaultClusterComponent []*DefaultClusterComponent `json:"defaultClusterComponent"` AgentInstallationStage int `json:"agentInstallationStage,notnull"` // -1=external, 0=not triggered, 1=progressing, 2=success, 3=fails @@ -80,10 +80,12 @@ type ClusterService interface { FindOne(clusterName string) (*ClusterBean, error) FindOneActive(clusterName string) (*ClusterBean, error) FindAll() ([]*ClusterBean, error) + FindAllWithoutConfig() ([]*ClusterBean, error) FindAllActive() ([]ClusterBean, error) DeleteFromDb(bean *ClusterBean, userId int32) error FindById(id int) (*ClusterBean, error) + FindByIdWithoutConfig(id int) (*ClusterBean, error) FindByIds(id []int) ([]ClusterBean, error) Update(ctx context.Context, bean *ClusterBean, userId int32) (*ClusterBean, error) Delete(bean *ClusterBean, userId int32) error @@ -257,6 +259,17 @@ func (impl *ClusterServiceImpl) FindOneActive(clusterName string) (*ClusterBean, return bean, nil } +func (impl *ClusterServiceImpl) FindAllWithoutConfig() ([]*ClusterBean, error) { + models, err := impl.FindAll() + if err != nil { + return nil, err + } + for _, model := range models { + model.Config = map[string]string{"bearer_token": ""} + } + return models, nil +} + func (impl *ClusterServiceImpl) FindAll() ([]*ClusterBean, error) { model, err := impl.clusterRepository.FindAllActive() if err != nil { @@ -326,6 +339,16 @@ func (impl *ClusterServiceImpl) FindById(id int) (*ClusterBean, error) { return bean, nil } +func (impl *ClusterServiceImpl) FindByIdWithoutConfig(id int) (*ClusterBean, error) { + model, err := impl.FindById(id) + if err != nil { + return nil, err + } + //empty bearer token as it will be hidden for user + model.Config = map[string]string{"bearer_token": ""} + return model, nil +} + func (impl *ClusterServiceImpl) FindByIds(ids []int) ([]ClusterBean, error) { models, err := impl.clusterRepository.FindByIds(ids) if err != nil { @@ -349,11 +372,6 @@ func (impl *ClusterServiceImpl) FindByIds(ids []int) ([]ClusterBean, error) { } func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *ClusterBean, userId int32) (*ClusterBean, error) { - //validating config - err := impl.CheckIfConfigIsValid(bean) - if err != nil { - return nil, err - } model, err := impl.clusterRepository.FindById(bean.Id) if err != nil { impl.logger.Error(err) @@ -372,8 +390,16 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *ClusterBean, u // check whether config modified or not, if yes create informer with updated config dbConfig := model.Config["bearer_token"] requestConfig := bean.Config["bearer_token"] + if len(requestConfig) == 0 { + bean.Config = model.Config + } if bean.ServerUrl != model.ServerUrl || dbConfig != requestConfig { bean.HasConfigOrUrlChanged = true + //validating config + err := impl.CheckIfConfigIsValid(bean) + if err != nil { + return nil, err + } } model.ClusterName = bean.ClusterName model.ServerUrl = bean.ServerUrl diff --git a/pkg/cluster/ClusterServiceExtended.go b/pkg/cluster/ClusterServiceExtended.go index 375de22a18..65176ecfb1 100644 --- a/pkg/cluster/ClusterServiceExtended.go +++ b/pkg/cluster/ClusterServiceExtended.go @@ -59,6 +59,17 @@ func NewClusterServiceImplExtended(repository repository.ClusterRepository, envi return clusterServiceExt } +func (impl *ClusterServiceImplExtended) FindAllWithoutConfig() ([]*ClusterBean, error) { + beans, err := impl.FindAll() + if err != nil { + return nil, err + } + for _, bean := range beans { + bean.Config = map[string]string{"bearer_token": ""} + } + return beans, nil +} + func (impl *ClusterServiceImplExtended) FindAll() ([]*ClusterBean, error) { beans, err := impl.ClusterServiceImpl.FindAll() if err != nil { diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index f3d0b0061b..2122acc026 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -208,7 +208,7 @@ func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch impl.Logger.Errorw("error in getting latest cdWfr by cdPipelineId", "err", err, "pipelineId", pipeline.Id) return nil, isTimelineUpdated } - impl.Logger.Infow("ARGO_PIPELINE_STATUS_UPDATE_REQ", "stage", "checkingDeploymentStatus", "argoAppName", pipeline, "cdWfr", cdWfr) + impl.Logger.Debugw("ARGO_PIPELINE_STATUS_UPDATE_REQ", "stage", "checkingDeploymentStatus", "argoAppName", pipeline, "cdWfr", cdWfr) if util3.IsTerminalStatus(cdWfr.Status) { //drop event return nil, isTimelineUpdated @@ -267,7 +267,7 @@ func (impl *CdHandlerImpl) UpdatePipelineTimelineAndStatusByLiveApplicationFetch err = impl.appStatusService.UpdateStatusWithAppIdEnvId(pipeline.AppId, pipeline.EnvironmentId, string(appStatus)) if err != nil { impl.Logger.Errorw("error occurred while updating app-status", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) - impl.Logger.Infow("ignoring the error", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) + impl.Logger.Debugw("ignoring the error, UpdateStatusWithAppIdEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) } } if isSucceeded { @@ -312,6 +312,7 @@ func (impl *CdHandlerImpl) CheckHelmAppStatusPeriodicallyAndUpdateInDb(helmPipel } if helmAppStatus == application.Healthy { wfr.Status = pipelineConfig.WorkflowSucceeded + wfr.FinishedOn = time.Now() } else { wfr.Status = pipelineConfig.WorkflowInProgress } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 89540859b1..d8149521ee 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/user" repository3 "github.com/devtron-labs/devtron/pkg/user/repository" + util2 "github.com/devtron-labs/devtron/util" "path" "regexp" "strconv" @@ -792,6 +793,19 @@ func (impl CiCdPipelineOrchestratorImpl) CheckStringMatchRegex(regex string, val } func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { + // validate the labels key-value if propagate is true + for _, label := range createRequest.AppLabels { + if !label.Propagate { + continue + } + labelKey := label.Key + labelValue := label.Value + err := util2.CheckIfValidLabel(labelKey, labelValue) + if err != nil { + return nil, err + } + } + dbConnection := impl.appRepository.GetConnection() tx, err := dbConnection.Begin() if err != nil { @@ -807,10 +821,11 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp if app.Active && len(createRequest.AppLabels) > 0 { for _, label := range createRequest.AppLabels { request := &bean.AppLabelDto{ - AppId: app.Id, - Key: label.Key, - Value: label.Value, - UserId: createRequest.UserId, + AppId: app.Id, + Key: label.Key, + Value: label.Value, + Propagate: label.Propagate, + UserId: createRequest.UserId, } _, err := impl.appLabelsService.Create(request, tx) if err != nil { diff --git a/pkg/pipeline/CiConfig.go b/pkg/pipeline/CiConfig.go index b9bf752cd2..eed0e29cd1 100644 --- a/pkg/pipeline/CiConfig.go +++ b/pkg/pipeline/CiConfig.go @@ -79,6 +79,9 @@ type CiConfig struct { CiRunnerDockerMTUValue int `env:"CI_RUNNER_DOCKER_MTU_VALUE" envDefault:"-1"` IgnoreDockerCacheForCI bool `env:"CI_IGNORE_DOCKER_CACHE"` VolumeMountsForCiJson string `env:"CI_VOLUME_MOUNTS_JSON"` + BuildPvcCachePath string `env:"PRE_CI_CACHE_PATH" envDefault:"/devtroncd-cache"` + DefaultPvcCachePath string `env:"DOCKER_BUILD_CACHE_PATH" envDefault:"/var/lib/docker"` + BuildxPvcCachePath string `env:"BUILDX_CACHE_PATH" envDefault:"/var/lib/devtron/buildx"` ClusterConfig *rest.Config NodeLabel map[string]string } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 3edfb00588..8e796ae39e 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -480,7 +480,8 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. CiBuildConfig: ciBuildConfigBean, CiBuildDockerMtuValue: impl.ciConfig.CiRunnerDockerMTUValue, IgnoreDockerCachePush: impl.ciConfig.IgnoreDockerCacheForCI, - IgnoreDockerCachePull: impl.ciConfig.IgnoreDockerCacheForCI || trigger.InvalidateCache, + IgnoreDockerCachePull: impl.ciConfig.IgnoreDockerCacheForCI, + CacheInvalidate: trigger.InvalidateCache, } if ciWorkflowConfig.LogsBucket == "" { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 56c468fc95..290ecd38c3 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -1596,7 +1596,7 @@ func (impl PipelineBuilderImpl) DeleteCdPipeline(pipeline *pipelineConfig.Pipeli } // delete entry in app_status table err = impl.appStatusRepository.Delete(tx, pipeline.AppId, pipeline.EnvironmentId) - if err != nil { + if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("err in deleting app_status from db", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return err } diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index 67acac14fa..47d7fc537f 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -600,6 +600,10 @@ func (impl *WorkflowDagExecutorImpl) buildWFRequest(runner *pipelineConfig.CdWor } for _, m := range ciPipeline.CiPipelineMaterials { + // git material should be active in this case + if m == nil || m.GitMaterial == nil || !m.GitMaterial.Active { + continue + } var ciMaterialCurrent repository.CiMaterialInfo for _, ciMaterial := range ciMaterialInfo { if ciMaterial.Material.GitConfiguration.URL == m.GitMaterial.Url { @@ -1161,7 +1165,7 @@ func (impl *WorkflowDagExecutorImpl) updatePreviousDeploymentStatus(currentRunne } impl.logger.Infow("updating cd wf runner status as previous runner status is", "status", previousRunner.Status) previousRunner.FinishedOn = triggeredAt - previousRunner.Message = "triggered new deployment" + previousRunner.Message = "A new deployment was initiated before this deployment completed" previousRunner.Status = pipelineConfig.WorkflowFailed previousRunner.UpdatedOn = time.Now() previousRunner.UpdatedBy = triggeredBy diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 3c10eb2d47..32c1da96a8 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -20,6 +20,7 @@ package pipeline import ( "context" "encoding/json" + "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" @@ -37,6 +38,7 @@ import ( "k8s.io/client-go/rest" "net/url" "strconv" + "strings" ) type WorkflowService interface { @@ -112,6 +114,8 @@ type WorkflowRequest struct { CiBuildDockerMtuValue int `json:"ciBuildDockerMtuValue"` IgnoreDockerCachePush bool `json:"ignoreDockerCachePush"` IgnoreDockerCachePull bool `json:"ignoreDockerCachePull"` + CacheInvalidate bool `json:"cacheInvalidate"` + IsPvcMounted bool `json:"IsPvcMounted"` } const ( @@ -122,6 +126,8 @@ const ( CI_WORKFLOW_NAME = "ci" CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector" + CI_NODE_PVC_ALL_ENV = "devtron.ai/ci-pvc-all" + CI_NODE_PVC_PIPELINE_PREFIX = "devtron.ai/ci-pvc" ) type ContainerResources struct { @@ -194,7 +200,15 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest miniCred := []v12.EnvVar{{Name: "AWS_ACCESS_KEY_ID", Value: impl.ciConfig.BlobStorageS3AccessKey}, {Name: "AWS_SECRET_ACCESS_KEY", Value: impl.ciConfig.BlobStorageS3SecretKey}} containerEnvVariables = append(containerEnvVariables, miniCred...) } - + pvc := appLabels[strings.ToLower(fmt.Sprintf("%s-%s", CI_NODE_PVC_PIPELINE_PREFIX, workflowRequest.PipelineName))] + if len(pvc) == 0 { + pvc = appLabels[CI_NODE_PVC_ALL_ENV] + } + if len(pvc) != 0 { + workflowRequest.IsPvcMounted = true + workflowRequest.IgnoreDockerCachePush = true + workflowRequest.IgnoreDockerCachePull = true + } ciCdTriggerEvent := CiCdTriggerEvent{ Type: ciEvent, CiRequest: workflowRequest, @@ -543,6 +557,36 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *WorkflowRequest } } + // pvc mounting starts + if len(pvc) != 0 { + buildPvcCachePath := impl.ciConfig.BuildPvcCachePath + buildxPvcCachePath := impl.ciConfig.BuildxPvcCachePath + defaultPvcCachePath := impl.ciConfig.DefaultPvcCachePath + + ciTemplate.Volumes = append(ciTemplate.Volumes, v12.Volume{ + Name: "root-vol", + VolumeSource: v12.VolumeSource{ + PersistentVolumeClaim: &v12.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc, + ReadOnly: false, + }, + }, + }) + ciTemplate.Container.VolumeMounts = append(ciTemplate.Container.VolumeMounts, + v12.VolumeMount{ + Name: "root-vol", + MountPath: buildPvcCachePath, + }, + v12.VolumeMount{ + Name: "root-vol", + MountPath: buildxPvcCachePath, + }, + v12.VolumeMount{ + Name: "root-vol", + MountPath: defaultPvcCachePath, + }) + } + // node selector if val, ok := appLabels[CI_NODE_SELECTOR_APP_LABEL_KEY]; ok { var nodeSelectors map[string]string diff --git a/scripts/sql/113_labels_propagate.down.sql b/scripts/sql/113_labels_propagate.down.sql new file mode 100644 index 0000000000..9033dddb30 --- /dev/null +++ b/scripts/sql/113_labels_propagate.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE app_label DROP COLUMN IF EXISTS propagate; + +ALTER TABLE "public"."app_label" ALTER COLUMN "key" SET DATA TYPE varchar(255); \ No newline at end of file diff --git a/scripts/sql/113_labels_propagate.up.sql b/scripts/sql/113_labels_propagate.up.sql new file mode 100644 index 0000000000..fe0707e973 --- /dev/null +++ b/scripts/sql/113_labels_propagate.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE app_label ADD COLUMN IF NOT EXISTS propagate boolean NOT NULL DEFAULT true; + +ALTER TABLE "public"."app_label" ALTER COLUMN "key" SET DATA TYPE varchar(317); \ No newline at end of file diff --git a/scripts/sql/114_global_tags.down.sql b/scripts/sql/114_global_tags.down.sql new file mode 100644 index 0000000000..e94e30ce7f --- /dev/null +++ b/scripts/sql/114_global_tags.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS "public"."global_tag"; + +---- DROP sequence +DROP SEQUENCE IF EXISTS public.id_seq_global_tag; \ No newline at end of file diff --git a/scripts/sql/114_global_tags.up.sql b/scripts/sql/114_global_tags.up.sql new file mode 100644 index 0000000000..5aabd76f58 --- /dev/null +++ b/scripts/sql/114_global_tags.up.sql @@ -0,0 +1,18 @@ +-- Sequence and defined type +CREATE SEQUENCE IF NOT EXISTS id_seq_global_tag; + +-- Table Definition +CREATE TABLE IF NOT EXISTS "public"."global_tag" +( + "id" int4 NOT NULL DEFAULT nextval('id_seq_global_tag'::regclass), + "key" varchar(100) NOT NULL, + "mandatory_project_ids_csv" varchar(100), + "propagate" bool, + "description" TEXT NOT NULL, + "active" bool, + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz, + "updated_by" int4, + PRIMARY KEY ("id") +); \ No newline at end of file diff --git a/specs/app-labels.yaml b/specs/app-labels.yaml index a28454b799..df657b401c 100644 --- a/specs/app-labels.yaml +++ b/specs/app-labels.yaml @@ -114,6 +114,9 @@ components: value: type: string description: label value + propagate: + type: boolean + description: Whether to propagate to kubernetes resources AppLabels: type: object diff --git a/specs/app_create_api.yaml b/specs/app_create_api.yaml index 5366a76358..553508f223 100644 --- a/specs/app_create_api.yaml +++ b/specs/app_create_api.yaml @@ -402,6 +402,9 @@ components: type: string Value: type: string + propagate: + type: boolean + description: Whether to propagate to kubernetes resources GitMaterial: type: object properties: diff --git a/specs/application.yaml b/specs/application.yaml index 48df4e80ed..5fc4434f7f 100644 --- a/specs/application.yaml +++ b/specs/application.yaml @@ -185,6 +185,9 @@ components: value: type: string description: label value + propagate: + type: boolean + description: Whether to propagate to kubernetes resources App: type: object diff --git a/specs/cluster_api_spec.yaml b/specs/cluster_api_spec.yaml index bcde53bc9c..ae79007643 100644 --- a/specs/cluster_api_spec.yaml +++ b/specs/cluster_api_spec.yaml @@ -3,6 +3,77 @@ info: version: 1.0.0 title: Devtron Labs paths: + /orchestrator/cluster: + put: + description: Update Cluster + operationId: UpdateCluster + requestBody: + description: A JSON object containing the cluster config + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterBean' + responses: + '200': + description: Successfully update the cluster + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterBean' + '400': + description: Bad Request. Input Validation(decode) error/wrong request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized User + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + get: + description: Get Cluster + operationId: GetCluster + parameters: + - name: id + in: query + description: cluster id. + required: true + schema: + type: integer + responses: + '200': + description: Successfully get cluster + content: + application/json: + schema: + $ref: '#/components/schemas/ClusterBean' + '400': + description: Bad Request. Input Validation(decode) error/wrong request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: Internal Server Error + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Unauthorized User + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /orchestrator/cluster/auth-list: get: description: list of accessible cluster @@ -34,6 +105,53 @@ paths: # components mentioned below components: schemas: + ClusterBean: + type: object + properties: + id: + type: integer + cluster_name: + type: string + server_url: + type: string + prometheus_url: + type: string + active: + type: boolean + config: + type: object + properties: + bearer_token: + type: string + description: it will be empty while fetching, and if no change while updating + k8sversion: + type: string + PrometheusAuth: + type: object + properties: + userName: + type: string + password: + type: string + tlsClientCert: + type: string + tlsClientKey: + type: string + DefaultClusterComponent: + type: object + properties: + name: + type: string + appId: + type: integer + installedAppId: + type: integer + envId: + type: integer + envname: + type: string + status: + type: string Cluster: type: object required: diff --git a/util/K8sUtil.go b/util/K8sUtil.go new file mode 100644 index 0000000000..bf8d1f863b --- /dev/null +++ b/util/K8sUtil.go @@ -0,0 +1,20 @@ +package util + +import ( + "errors" + "fmt" + "k8s.io/apimachinery/pkg/util/validation" +) + +func CheckIfValidLabel(labelKey string, labelValue string) error { + errs := validation.IsQualifiedName(labelKey) + if len(errs) > 0 { + return errors.New(fmt.Sprintf("Validation error - label key - %s is not satisfying the label key criteria", labelKey)) + } + + errs = validation.IsValidLabelValue(labelValue) + if len(errs) > 0 { + return errors.New(fmt.Sprintf("Validation error - label value - %s is not satisfying the label value criteria for label key - %s", labelValue, labelKey)) + } + return nil +} diff --git a/util/k8s/k8sApplicationRestHandler.go b/util/k8s/k8sApplicationRestHandler.go index 952735fdbe..c3d68e1006 100644 --- a/util/k8s/k8sApplicationRestHandler.go +++ b/util/k8s/k8sApplicationRestHandler.go @@ -1,6 +1,7 @@ package k8s import ( + "context" "encoding/json" "errors" "fmt" @@ -10,7 +11,6 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/k8s/application" util2 "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/terminal" "github.com/devtron-labs/devtron/pkg/user" "github.com/devtron-labs/devtron/pkg/user/casbin" @@ -51,7 +51,6 @@ type K8sApplicationRestHandlerImpl struct { enforcer casbin.Enforcer enforcerUtil rbac.EnforcerUtil enforcerUtilHelm rbac.EnforcerUtilHelm - clusterService cluster.ClusterService helmAppService client.HelmAppService userService user.UserService } @@ -59,7 +58,7 @@ type K8sApplicationRestHandlerImpl struct { func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, k8sApplicationService K8sApplicationService, pump connector.Pump, terminalSessionHandler terminal.TerminalSessionHandler, - enforcer casbin.Enforcer, enforcerUtilHelm rbac.EnforcerUtilHelm, enforcerUtil rbac.EnforcerUtil, clusterService cluster.ClusterService, + enforcer casbin.Enforcer, enforcerUtilHelm rbac.EnforcerUtilHelm, enforcerUtil rbac.EnforcerUtil, helmAppService client.HelmAppService, userService user.UserService) *K8sApplicationRestHandlerImpl { return &K8sApplicationRestHandlerImpl{ logger: logger, @@ -70,7 +69,6 @@ func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, enforcerUtilHelm: enforcerUtilHelm, enforcerUtil: enforcerUtil, helmAppService: helmAppService, - clusterService: clusterService, userService: userService, } } @@ -112,11 +110,7 @@ func (handler *K8sApplicationRestHandlerImpl) GetResource(w http.ResponseWriter, common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) return } - } else if request.ClusterId > 0 { - if ok := handler.validateRbac(w, token, request, casbin.ActionGet); !ok { - return - } - } else { + } else if request.ClusterId <= 0 { common.WriteJsonResp(w, errors.New("can not resource manifest as target cluster is not provided"), nil, http.StatusBadRequest) return } @@ -133,7 +127,14 @@ func (handler *K8sApplicationRestHandlerImpl) GetResource(w http.ResponseWriter, // Obfuscate secret if user does not have edit access canUpdate = handler.enforcer.Enforce(token, casbin.ResourceHelmApp, casbin.ActionUpdate, rbacObject) || handler.enforcer.Enforce(token, casbin.ResourceHelmApp, casbin.ActionUpdate, rbacObject2) } else if request.ClusterId > 0 { - canUpdate = handler.validateRbac(nil, token, request, casbin.ActionUpdate) + canUpdate = handler.k8sApplicationService.ValidateClusterResourceBean(r.Context(), request.ClusterId, resource.Manifest, request.K8sRequest.ResourceIdentifier.GroupVersionKind, handler.getRbacCallbackForResource(token, casbin.ActionUpdate)) + if !canUpdate { + readAllowed := handler.k8sApplicationService.ValidateClusterResourceBean(r.Context(), request.ClusterId, resource.Manifest, request.K8sRequest.ResourceIdentifier.GroupVersionKind, handler.getRbacCallbackForResource(token, casbin.ActionGet)) + if !readAllowed { + common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) + return + } + } } if !canUpdate && resource != nil { modifiedManifest, err := k8sObjectsUtil.HideValuesIfSecret(&resource.Manifest) @@ -148,23 +149,6 @@ func (handler *K8sApplicationRestHandlerImpl) GetResource(w http.ResponseWriter, common.WriteJsonResp(w, nil, resource, http.StatusOK) } -func (handler *K8sApplicationRestHandlerImpl) validateRbac(w http.ResponseWriter, token string, request ResourceRequestBean, casbinAction string) bool { - clusterBean, err := handler.clusterService.FindById(request.ClusterId) - if err != nil { - if w != nil { - common.WriteJsonResp(w, errors.New("clusterId is not valid"), nil, http.StatusBadRequest) - } - return false - } - if ok := handler.verifyRbacForCluster(token, clusterBean.ClusterName, request, casbinAction); !ok { - if w != nil { - common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) - } - return false - } - return true -} - func (handler *K8sApplicationRestHandlerImpl) GetHostUrlsByBatch(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) clusterIdString := vars["appId"] @@ -298,7 +282,7 @@ func (handler *K8sApplicationRestHandlerImpl) UpdateResource(w http.ResponseWrit //RBAC enforcer Ends } else if request.ClusterId > 0 { // assume direct update in cluster - if ok := handler.validateRbac(w, token, request, casbin.ActionUpdate); !ok { + if ok := handler.handleRbac(r, w, request, token, casbin.ActionUpdate); !ok { return } } else { @@ -315,6 +299,18 @@ func (handler *K8sApplicationRestHandlerImpl) UpdateResource(w http.ResponseWrit common.WriteJsonResp(w, nil, resource, http.StatusOK) } +func (handler *K8sApplicationRestHandlerImpl) handleRbac(r *http.Request, w http.ResponseWriter, request ResourceRequestBean, token string, casbinAction string) bool { + allowed, err := handler.k8sApplicationService.ValidateClusterResourceRequest(r.Context(), &request, handler.getRbacCallbackForResource(token, casbinAction)) + if err != nil { + common.WriteJsonResp(w, errors.New("invalid request"), nil, http.StatusBadRequest) + return false + } + if !allowed { + common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) + } + return allowed +} + func (handler *K8sApplicationRestHandlerImpl) DeleteResource(w http.ResponseWriter, r *http.Request) { userId, err := handler.userService.GetLoggedInUser(r) @@ -363,7 +359,7 @@ func (handler *K8sApplicationRestHandlerImpl) DeleteResource(w http.ResponseWrit } //RBAC enforcer Ends } else if request.ClusterId > 0 { - if ok := handler.validateRbac(w, token, request, casbin.ActionDelete); !ok { + if ok := handler.handleRbac(r, w, request, token, casbin.ActionDelete); !ok { return } } else { @@ -419,7 +415,7 @@ func (handler *K8sApplicationRestHandlerImpl) ListEvents(w http.ResponseWriter, } //RBAC enforcer Ends } else if request.ClusterId > 0 { - if ok := handler.validateRbac(w, token, request, casbin.ActionGet); !ok { + if ok := handler.handleRbac(r, w, request, token, casbin.ActionGet); !ok { return } } else { @@ -526,7 +522,7 @@ func (handler *K8sApplicationRestHandlerImpl) GetPodLogs(w http.ResponseWriter, }, }, } - if ok := handler.validateRbac(w, token, *request, casbin.ActionGet); !ok { + if ok := handler.handleRbac(r, w, *request, token, casbin.ActionGet); !ok { return } } else { @@ -548,6 +544,17 @@ func (handler *K8sApplicationRestHandlerImpl) GetPodLogs(w http.ResponseWriter, isReconnect = true } stream, err := handler.k8sApplicationService.GetPodLogs(r.Context(), request) + ctx, cancel := context.WithCancel(r.Context()) + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + defer cancel() defer util.Close(stream, handler.logger) handler.pump.StartK8sStreamWithHeartBeat(w, isReconnect, stream, err) } @@ -612,7 +619,7 @@ func (handler *K8sApplicationRestHandlerImpl) GetTerminalSession(w http.Response }, }, } - if ok := handler.validateRbac(w, token, resourceRequestBean, casbin.ActionUpdate); !ok { + if ok := handler.handleRbac(r, w, resourceRequestBean, token, casbin.ActionUpdate); !ok { return } } else { @@ -723,9 +730,18 @@ func (handler *K8sApplicationRestHandlerImpl) ApplyResources(w http.ResponseWrit common.WriteJsonResp(w, nil, response, http.StatusOK) } +func (handler *K8sApplicationRestHandlerImpl) getRbacCallbackForResource(token string, casbinAction string) func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool { + return func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool { + return handler.verifyRbacForResource(token, clusterName, resourceIdentifier, casbinAction) + } +} + +func (handler *K8sApplicationRestHandlerImpl) verifyRbacForResource(token string, clusterName string, resourceIdentifier application.ResourceIdentifier, casbinAction string) bool { + resourceName, objectName := handler.enforcerUtil.GetRBACNameForClusterEntity(clusterName, resourceIdentifier) + return handler.enforcer.Enforce(token, strings.ToLower(resourceName), casbinAction, strings.ToLower(objectName)) +} + func (handler *K8sApplicationRestHandlerImpl) verifyRbacForCluster(token string, clusterName string, request ResourceRequestBean, casbinAction string) bool { k8sRequest := request.K8sRequest - resourceIdentifier := k8sRequest.ResourceIdentifier - resourceName, objectName := handler.enforcerUtil.GetRBACNameForClusterEntity(clusterName, resourceIdentifier) - return handler.enforcer.Enforce(token, resourceName, casbinAction, strings.ToLower(objectName)) + return handler.verifyRbacForResource(token, clusterName, k8sRequest.ResourceIdentifier, casbinAction) } diff --git a/util/k8s/k8sApplicationService.go b/util/k8s/k8sApplicationService.go index ad938634c1..a65318eace 100644 --- a/util/k8s/k8sApplicationService.go +++ b/util/k8s/k8sApplicationService.go @@ -42,6 +42,9 @@ type K8sApplicationService interface { ListEvents(ctx context.Context, request *ResourceRequestBean) (*application.EventsResponse, error) GetPodLogs(ctx context.Context, request *ResourceRequestBean) (io.ReadCloser, error) ValidateResourceRequest(ctx context.Context, appIdentifier *client.AppIdentifier, request *application.K8sRequestBean) (bool, error) + ValidateClusterResourceRequest(ctx context.Context, clusterResourceRequest *ResourceRequestBean, + rbacCallback func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool) (bool, error) + ValidateClusterResourceBean(ctx context.Context, clusterId int, manifest unstructured.Unstructured, gvk schema.GroupVersionKind, rbacCallback func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool) bool GetResourceInfo(ctx context.Context) (*ResourceInfo, error) GetRestConfigByClusterId(ctx context.Context, clusterId int) (*rest.Config, error) GetRestConfigByCluster(ctx context.Context, cluster *cluster.ClusterBean) (*rest.Config, error) @@ -52,7 +55,6 @@ type K8sApplicationService interface { GetResourceList(ctx context.Context, token string, request *ResourceRequestBean, validateResourceAccess func(token string, clusterName string, request ResourceRequestBean, casbinAction string) bool) (*util.ClusterResourceListMap, error) ApplyResources(ctx context.Context, token string, request *application.ApplyResourcesRequest, resourceRbacHandler func(token string, clusterName string, request ResourceRequestBean, casbinAction string) bool) ([]*application.ApplyResourcesResponse, error) } - type K8sApplicationServiceImpl struct { logger *zap.SugaredLogger clusterService cluster.ClusterService @@ -452,6 +454,53 @@ func (impl *K8sApplicationServiceImpl) GetRestConfigByCluster(ctx context.Contex return restConfig, nil } +func (impl *K8sApplicationServiceImpl) ValidateClusterResourceRequest(ctx context.Context, clusterResourceRequest *ResourceRequestBean, + rbacCallback func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool) (bool, error) { + clusterId := clusterResourceRequest.ClusterId + clusterBean, err := impl.clusterService.FindById(clusterId) + if err != nil { + impl.logger.Errorw("error in getting clusterBean by cluster Id", "clusterId", clusterId, "err", err) + return false, err + } + clusterName := clusterBean.ClusterName + restConfig, err := impl.GetRestConfigByCluster(ctx, clusterBean) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster", "clusterId", clusterId, "err", err) + return false, err + } + k8sRequest := clusterResourceRequest.K8sRequest + respManifest, err := impl.k8sClientService.GetResource(ctx, restConfig, k8sRequest) + if err != nil { + impl.logger.Errorw("error in getting resource", "err", err, "request", clusterResourceRequest) + return false, err + } + return impl.validateResourceManifest(clusterName, respManifest.Manifest, k8sRequest.ResourceIdentifier.GroupVersionKind, rbacCallback), nil +} + +func (impl *K8sApplicationServiceImpl) validateResourceManifest(clusterName string, resourceManifest unstructured.Unstructured, gvk schema.GroupVersionKind, rbacCallback func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool) bool { + validateCallback := func(namespace, group, kind, resourceName string) bool { + resourceIdentifier := application.ResourceIdentifier{ + Name: resourceName, + Namespace: namespace, + GroupVersionKind: schema.GroupVersionKind{ + Group: group, + Kind: kind, + }, + } + return rbacCallback(clusterName, resourceIdentifier) + } + return impl.K8sUtil.ValidateResource(resourceManifest.Object, gvk, validateCallback) +} + +func (impl *K8sApplicationServiceImpl) ValidateClusterResourceBean(ctx context.Context, clusterId int, manifest unstructured.Unstructured, gvk schema.GroupVersionKind, rbacCallback func(clusterName string, resourceIdentifier application.ResourceIdentifier) bool) bool { + clusterBean, err := impl.clusterService.FindById(clusterId) + if err != nil { + impl.logger.Errorw("error in getting clusterBean by cluster Id", "clusterId", clusterId, "err", err) + return false + } + return impl.validateResourceManifest(clusterBean.ClusterName, manifest, gvk, rbacCallback) +} + func (impl *K8sApplicationServiceImpl) ValidateResourceRequest(ctx context.Context, appIdentifier *client.AppIdentifier, request *application.K8sRequestBean) (bool, error) { app, err := impl.helmAppService.GetApplicationDetail(ctx, appIdentifier) if err != nil { @@ -555,7 +604,8 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResources(ctx context.Context, c if clusterBean.ClusterName != role.Cluster { continue } - if role.Group == "" && role.Kind == "" { + kind := role.Kind + if role.Group == "" && kind == "" { allowedAll = true break } @@ -565,7 +615,17 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResources(ctx context.Context, c } else if groupName == casbin.ClusterEmptyGroupPlaceholder { groupName = "" } - allowedGroupKinds[groupName+"||"+role.Kind] = true + allowedGroupKinds[groupName+"||"+kind] = true + // add children for this kind + children, found := util.KindVsChildrenGvk[kind] + if found { + // if rollout kind other than argo, then neglect only + if kind != util.K8sClusterResourceRolloutKind || groupName == util.K8sClusterResourceRolloutGroup { + for _, child := range children { + allowedGroupKinds[child.Group+"||"+child.Kind] = true + } + } + } } if !allowedAll { @@ -615,14 +675,17 @@ func (impl *K8sApplicationServiceImpl) GetResourceList(ctx context.Context, toke impl.logger.Errorw("error in getting resource list", "err", err, "request", request) return resourceList, err } - checkForResourceCallback := func(namespace, resourceName string) bool { + checkForResourceCallback := func(namespace, group, kind, resourceName string) bool { resourceIdentifier := k8sRequest.ResourceIdentifier resourceIdentifier.Name = resourceName resourceIdentifier.Namespace = namespace + if group != "" && kind != "" { + resourceIdentifier.GroupVersionKind = schema.GroupVersionKind{Group: group, Kind: kind} + } k8sRequest.ResourceIdentifier = resourceIdentifier return validateResourceAccess(token, clusterBean.ClusterName, *request, casbin.ActionGet) } - resourceList, err = impl.K8sUtil.BuildK8sObjectListTableData(&resp.Resources, namespaced, request.K8sRequest.ResourceIdentifier.GroupVersionKind.Kind, checkForResourceCallback) + resourceList, err = impl.K8sUtil.BuildK8sObjectListTableData(&resp.Resources, namespaced, request.K8sRequest.ResourceIdentifier.GroupVersionKind, checkForResourceCallback) if err != nil { impl.logger.Errorw("error on parsing for k8s resource", "err", err) return resourceList, err diff --git a/wire_gen.go b/wire_gen.go index f42e2c5488..91dfae2674 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,8 +1,7 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire -//go:build !wireinject -// +build !wireinject +//go:generate wire +//+build !wireinject package main @@ -622,7 +621,7 @@ func InitializeApp() (*App, error) { coreAppRouterImpl := router.NewCoreAppRouterImpl(coreAppRestHandlerImpl) helmAppRestHandlerImpl := client3.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImplExtended, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client3.NewHelmAppRouterImpl(helmAppRestHandlerImpl) - k8sApplicationRestHandlerImpl := k8s.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, clusterServiceImplExtended, helmAppServiceImpl, userServiceImpl) + k8sApplicationRestHandlerImpl := k8s.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl) k8sApplicationRouterImpl := k8s.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) pProfRestHandlerImpl := restHandler.NewPProfRestHandler(userServiceImpl) pProfRouterImpl := router.NewPProfRouter(sugaredLogger, pProfRestHandlerImpl) From 292e7bb082cb55f378e6502a915e30d8e512ab66 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Feb 2023 17:34:24 +0530 Subject: [PATCH 008/118] made changes to the queries --- internal/sql/repository/app/AppRepository.go | 16 ++++++++-------- .../helper/AppListingRepositoryQueryBuilder.go | 2 +- pkg/appClone/AppCloneService.go | 17 ----------------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index c111bd3a3f..6a80dd53d6 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -174,7 +174,7 @@ func (repo AppRepositoryImpl) FindAppsByTeamName(teamName string) ([]App, error) func (repo AppRepositoryImpl) FindAll() ([]*App, error) { var apps []*App - err := repo.dbConnection.Model(&apps).Where("active = ?", true).Where("app_store = ?", false).Select() + err := repo.dbConnection.Model(&apps).Where("active = ?", true).Where("app_store = ?", 0).Select() return apps, err } @@ -189,7 +189,7 @@ func (repo AppRepositoryImpl) FindAppsByEnvironmentId(environmentId int) ([]App, func (repo AppRepositoryImpl) FindAllActiveAppsWithTeam() ([]*App, error) { var apps []*App err := repo.dbConnection.Model(&apps).Column("Team"). - Where("app.active = ?", true).Where("app.app_store = ?", false). + Where("app.active = ?", true).Where("app.app_store = ?", 0). Select() return apps, err } @@ -207,25 +207,25 @@ func (repo AppRepositoryImpl) FetchAppsByFilterV2(appNameIncludes string, appNam err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Join("inner join pipeline p on p.app_id=app.id"). Where("app.app_name like ?", ""+appNameIncludes+"%").Where("app.app_name not like ?", ""+appNameExcludes+"%"). - Where("app.active=?", true).Where("app_store=?", false). + Where("app.active=?", true).Where("app_store=?", 0). Where("p.environment_id = ?", environmentId).Where("p.deleted = ?", false). Select() } else if environmentId > 0 && appNameExcludes == "" { err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Join("inner join pipeline p on p.app_id=app.id"). Where("app.app_name like ?", ""+appNameIncludes+"%"). - Where("app.active=?", true).Where("app_store=?", false). + Where("app.active=?", true).Where("app_store=?", 0). Where("p.environment_id = ?", environmentId).Where("p.deleted = ?", false). Select() } else if environmentId == 0 && len(appNameExcludes) > 0 { err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Where("app.app_name like ?", ""+appNameIncludes+"%").Where("app.app_name not like ?", ""+appNameExcludes+"%"). - Where("app.active=?", true).Where("app_store=?", false). + Where("app.active=?", true).Where("app_store=?", 0). Select() } else if environmentId == 0 && appNameExcludes == "" { err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Where("app.app_name like ?", ""+appNameIncludes+"%"). - Where("app.active=?", true).Where("app_store=?", false). + Where("app.active=?", true).Where("app_store=?", 0). Select() } return apps, err @@ -251,7 +251,7 @@ func (repo AppRepositoryImpl) FindAppAndProjectByAppName(appName string) (*App, func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string) ([]*App, error) { var apps []*App - err := repo.dbConnection.Model(&apps).Where("app_name ILIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", false).Select() + err := repo.dbConnection.Model(&apps).Where("app_name ILIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", 0).Select() return apps, err } @@ -313,7 +313,7 @@ func (repo AppRepositoryImpl) FetchAllActiveDevtronAppsWithAppIdAndName() ([]*Ap err := repo.dbConnection.Model(&apps). Column("id", "app_name"). - Where("app_store = ?", false). + Where("app_store = ?", 0). Where("active", true). Select() if err != nil && err != pg.ErrNoRows { diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 33c8c896c2..7486612b6e 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -94,7 +94,7 @@ func (impl AppListingRepositoryQueryBuilder) buildAppListingSortBy(appListingFil } func (impl AppListingRepositoryQueryBuilder) buildAppListingWhereCondition(appListingFilter AppListingFilter) string { - whereCondition := "WHERE a.active = true and a.app_store is 0 " + whereCondition := "WHERE a.active = true and a.app_store = 0 " if len(appListingFilter.Environments) > 0 { envIds := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(appListingFilter.Environments)), ","), "[]") whereCondition = whereCondition + "and env.id IN (" + envIds + ") " diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 68aa03fa02..b8421f689b 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -286,23 +286,6 @@ func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) return app, nil } - if isSameProject { - _, err = impl.CreateEnvCm(context, cloneReq.RefAppId, newAppId, userId) - if err != nil { - impl.logger.Errorw("error in creating env cm", "err", err) - return nil, err - } - _, err = impl.CreateEnvSecret(context, cloneReq.RefAppId, newAppId, userId) - if err != nil { - impl.logger.Errorw("error in creating env secret", "err", err) - return nil, err - } - _, err = impl.createEnvOverride(cloneReq.RefAppId, newAppId, userId, context) - if err != nil { - impl.logger.Errorw("error in cloning env override", "err", err) - return nil, err - } - } _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) if err != nil { From d170f30260a4d49aec27f2e447a5dea5f4d59f66 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Mon, 13 Feb 2023 19:44:26 +0530 Subject: [PATCH 009/118] Update AppListingRepository.go --- internal/sql/repository/AppListingRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index a6d75c4769..0389f480f1 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -313,7 +313,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig " INNER JOIN ci_pipeline cp on cp.id = p.ci_pipeline_id" + " INNER JOIN app a ON a.id = p.app_id" + " INNER JOIN environment env on env.id = p.environment_id" + - " WHERE p.app_id=? and p.deleted=false and a.app_store is 0 AND env.active = TRUE;" + " WHERE p.app_id=? and p.deleted=false and a.app_store = 0 AND env.active = TRUE;" impl.Logger.Debugw("query", query) _, err := impl.dbConnection.Query(&triggerView, query, appId) From 864702c30ed6ac78bd5a1018f430db61e8b45bde Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Mon, 13 Feb 2023 19:45:06 +0530 Subject: [PATCH 010/118] Update version.go --- util/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/version.go b/util/version.go index 417b3fb010..18a794dcb1 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE_FULL} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE} } func IsBaseStack() bool { From caab518666448d364e601f37e280084594e7d270 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Mon, 13 Feb 2023 19:46:02 +0530 Subject: [PATCH 011/118] Update version.go --- util/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/version.go b/util/version.go index 18a794dcb1..f64cb32956 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: ServerMode} } func IsBaseStack() bool { From 83b3aa2b81ce95d4b071c707ca7a0fe2d5fc1e9c Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Mon, 13 Feb 2023 19:50:11 +0530 Subject: [PATCH 012/118] Update 1_insert.up.sql --- scripts/sql/1_insert.up.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/sql/1_insert.up.sql b/scripts/sql/1_insert.up.sql index 28013c19ec..06dc7c72b4 100644 --- a/scripts/sql/1_insert.up.sql +++ b/scripts/sql/1_insert.up.sql @@ -32,7 +32,7 @@ CREATE TABLE public.app ( updated_on timestamp with time zone NOT NULL, updated_by integer NOT NULL, team_id integer, - app_store integer DEFAULT 0 + app_store integer DEFAULT false ); @@ -5987,4 +5987,4 @@ INSERT INTO "public"."environment" ("id", "environment_name", "cluster_id", "act -- -- PostgreSQL database dump complete --- \ No newline at end of file +-- From e8b2b0a7a5e8170e0874c0f20c525c1ed6f89dfe Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Mon, 13 Feb 2023 19:50:54 +0530 Subject: [PATCH 013/118] Update 1_insert.up.sql --- scripts/sql/1_insert.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sql/1_insert.up.sql b/scripts/sql/1_insert.up.sql index 06dc7c72b4..663fab9a73 100644 --- a/scripts/sql/1_insert.up.sql +++ b/scripts/sql/1_insert.up.sql @@ -32,7 +32,7 @@ CREATE TABLE public.app ( updated_on timestamp with time zone NOT NULL, updated_by integer NOT NULL, team_id integer, - app_store integer DEFAULT false + app_store boolean DEFAULT false ); From c494b180178a550db3c8d1f7880f6240aabaf661 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:45:56 +0530 Subject: [PATCH 014/118] Update AppCloneService.go --- pkg/appClone/AppCloneService.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index b8421f689b..f1fcbe5c59 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -88,19 +88,19 @@ type CloneRequest struct { func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template app - templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { - impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) - err = &util.ApiError{ - Code: constants.AppDoesNotExist.Code, - InternalMessage: "app does not exist", - UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - } - return nil, err - } + //templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + //if err != nil && err != pg.ErrNoRows { + // return nil, err + //} + //if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { + // impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) + // err = &util.ApiError{ + // Code: constants.AppDoesNotExist.Code, + // InternalMessage: "app does not exist", + // UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + // } + // return nil, err + //} //create new app cloneReq := &CloneRequest{ RefAppId: createReq.TemplateId, From c10d7b98e99f8beff8345ba2281f53ae7c0f57c0 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:47:16 +0530 Subject: [PATCH 015/118] Update AppCloneService.go --- pkg/appClone/AppCloneService.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index f1fcbe5c59..5a6db16f38 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -209,19 +209,19 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template job - templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { - impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) - err = &util.ApiError{ - Code: constants.AppDoesNotExist.Code, - InternalMessage: "job does not exist", - UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - } - return nil, err - } + //templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + //if err != nil && err != pg.ErrNoRows { + // return nil, err + //} + //if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { + // impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) + // err = &util.ApiError{ + // Code: constants.AppDoesNotExist.Code, + // InternalMessage: "job does not exist", + // UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + // } + // return nil, err + //} //create new job cloneReq := &CloneRequest{ From a918e70b7158bcfde4731cb28f04d3d280a379f2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 12:51:33 +0530 Subject: [PATCH 016/118] Uodated app listing api --- api/bean/AppView.go | 15 +++++ api/restHandler/AppListingRestHandler.go | 23 ++++++++ api/router/AppListingRouter.go | 4 ++ .../sql/repository/AppListingRepository.go | 17 +++++- internal/sql/repository/app/AppRepository.go | 1 + .../AppListingRepositoryQueryBuilder.go | 36 ++++++++++++ pkg/app/AppListingService.go | 21 +++++++ pkg/appClone/AppCloneService.go | 55 +++++++++---------- pkg/bean/app.go | 15 ++--- pkg/pipeline/CiCdPipelineOrchestrator.go | 15 ++--- 10 files changed, 156 insertions(+), 46 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 681ac4f5d9..6e36cb9166 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -52,6 +52,21 @@ type CiMaterialDTO struct { SourceValue string `json:"value"` } +type JobsContainer struct { + JobId int `json:"id"` + JobName string `json:"name""` + LastRun []LastRunStatus `json:"last-run"'` + Description string `json:"description"` + ProjectId int `json:"projectId"` +} + +type LastRunStatus struct { + PipelineId int `json:"pipeline-id"` + LastRunStatus int `json:"lastRunStatus"` + LastRunAt string `json:"lastRunAt"` + LastSuccessAt string `json:"lastRunAt"` +} + type AppEnvironmentContainer struct { AppId int `json:"appId"` AppName string `json:"appName"` diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 6210a55dc7..a255c57226 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -56,6 +56,7 @@ import ( type AppListingRestHandler interface { FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) + FetchJobs(w http.ResponseWriter, r *http.Request) FetchAppDetails(w http.ResponseWriter, r *http.Request) FetchAllDevtronManagedApps(w http.ResponseWriter, r *http.Request) FetchAppTriggerView(w http.ResponseWriter, r *http.Request) @@ -156,6 +157,28 @@ func (handler AppListingRestHandlerImpl) FetchAllDevtronManagedApps(w http.Respo res, err := handler.appListingService.FetchAllDevtronManagedApps() common.WriteJsonResp(w, err, res, http.StatusOK) } +func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *http.Request) { + newCtx, span := otel.Tracer("userService").Start(r.Context(), "GetLoggedInUser") + userId, err := handler.userService.GetLoggedInUser(r) + span.End() + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + newCtx, span = otel.Tracer("userService").Start(newCtx, "GetById") + + var fetchJobListingRequest app.FetchAppListingRequest + decoder := json.NewDecoder(r.Body) + err = decoder.Decode(&fetchJobListingRequest) + if err != nil { + handler.logger.Errorw("request err, FetchAppsByEnvironment", "err", err, "payload", fetchJobListingRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + var dg *deploymentGroup. + +} func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) { //Allow CORS here By * or specific origin setupResponse(&w, r) diff --git a/api/router/AppListingRouter.go b/api/router/AppListingRouter.go index de115f96eb..0f51031911 100644 --- a/api/router/AppListingRouter.go +++ b/api/router/AppListingRouter.go @@ -48,6 +48,10 @@ func (router AppListingRouterImpl) initAppListingRouter(appListingRouter *mux.Ro HandlerFunc(router.appListingRestHandler.FetchAppsByEnvironment). Methods("POST") + appListingRouter.Path("/jobs/list"). + HandlerFunc(router.appListingRestHandler.FetchJobs). + Methods("POST") + //This API used for fetch app details, not deployment details appListingRouter.Path("/detail").Queries("app-id", "{app-id}").Queries("env-id", "{env-id}"). HandlerFunc(router.appListingRestHandler.FetchAppDetails). diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index a6d75c4769..f90d19ad36 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -37,6 +37,7 @@ import ( type AppListingRepository interface { FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) + FetchJobs(appListingFilter helper.AppListingFilter) ([]*bean.JobsContainer, error) DeploymentDetailsByAppIdAndEnvId(ctx context.Context, appId int, envId int) (bean.DeploymentDetailContainer, error) FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -84,7 +85,17 @@ func NewAppListingRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB, * It will return the list of filtered apps with details related to each env */ - +func (impl AppListingRepositoryImpl) FetchJobs(jobListingFilter helper.AppListingFilter) ([]*bean.JobsContainer, error) { + var jobContainer []*bean.JobsContainer + jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(jobListingFilter) + impl.Logger.Debugw("basic app detail query: ", jobsQuery) + _, appsErr := impl.dbConnection.Query(&jobContainer, jobsQuery) + if appsErr != nil { + impl.Logger.Error(appsErr) + return jobContainer, appsErr + } + return jobContainer, nil +} func (impl AppListingRepositoryImpl) FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") var appEnvArr []*bean.AppEnvironmentContainer @@ -313,7 +324,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig " INNER JOIN ci_pipeline cp on cp.id = p.ci_pipeline_id" + " INNER JOIN app a ON a.id = p.app_id" + " INNER JOIN environment env on env.id = p.environment_id" + - " WHERE p.app_id=? and p.deleted=false and a.app_store is 0 AND env.active = TRUE;" + " WHERE p.app_id=? and p.deleted=false and a.app_store = 0 AND env.active = TRUE;" impl.Logger.Debugw("query", query) _, err := impl.dbConnection.Query(&triggerView, query, appId) @@ -392,7 +403,7 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppS " LEFT JOIN charts ch on ch.app_id=app.id" + " LEFT JOIN pipeline p on p.app_id=app.id" + " LEFT JOIN chart_env_config_override ceco on ceco.chart_id=ch.id" + - " WHERE app.id=? and app.app_store is false;" + " WHERE app.id=? and app.app_store = 0;" impl.Logger.Debugw("last app stages status query:", "query", query) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 6a80dd53d6..27b13d3c86 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -33,6 +33,7 @@ type App struct { TeamId int `sql:"team_id"` AppStore int `sql:"app_store, notnull"` AppOfferingMode string `sql:"app_offering_mode,notnull"` + Description string `sql:"-"` Team team.Team sql.AuditLog } diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 7486612b6e..a627dc31b8 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -60,6 +60,17 @@ const ( AppNameSortBy SortBy = "appNameSort" ) +func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appListingFilter AppListingFilter) string { + whereCondition := impl.buildJobListingWhereCondition(appListingFilter) + orderByClause := impl.buildJobListingSortBy(appListingFilter) + + query := "SELECT app_id, a.app_name, a.team_id, a.description" + "status, started_on, finished_on" + + "" + + query = query + whereCondition + orderByClause + return query +} + func (impl AppListingRepositoryQueryBuilder) BuildAppListingQuery(appListingFilter AppListingFilter) string { whereCondition := impl.buildAppListingWhereCondition(appListingFilter) orderByClause := impl.buildAppListingSortBy(appListingFilter) @@ -93,6 +104,31 @@ func (impl AppListingRepositoryQueryBuilder) buildAppListingSortBy(appListingFil return orderByCondition } +func (impl AppListingRepositoryQueryBuilder) buildJobListingSortBy(appListingFilter AppListingFilter) string { + orderByCondition := " ORDER BY a.name" + return orderByCondition +} + +func (impl AppListingRepositoryQueryBuilder) buildJobListingWhereCondition(jobListingFilter AppListingFilter) string { + whereCondition := "WHERE a.active = true and a.app_store = 2 " + + if len(jobListingFilter.Teams) > 0 { + teamIds := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(jobListingFilter.Teams)), ","), "[]") + whereCondition = whereCondition + "and a.team_id IN (" + teamIds + ") " + } + + if jobListingFilter.AppNameSearch != "" { + likeClause := "'%" + jobListingFilter.AppNameSearch + "%'" + whereCondition = whereCondition + "and a.app_name like " + likeClause + " " + } + // add job stats filter here + if len(jobListingFilter.AppStatuses) > 0 { + appStatuses := util.ProcessAppStatuses(jobListingFilter.AppStatuses) + whereCondition = whereCondition + "and aps.status IN (" + appStatuses + ") " + } + return whereCondition +} + func (impl AppListingRepositoryQueryBuilder) buildAppListingWhereCondition(appListingFilter AppListingFilter) string { whereCondition := "WHERE a.active = true and a.app_store = 0 " if len(appListingFilter.Environments) > 0 { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 00bfbdff76..11db5a4d48 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -53,6 +53,7 @@ import ( type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) + FetchJobs(fetchJobListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request) ([]*bean.JobsContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) FetchAppDetails(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -220,6 +221,26 @@ func (impl AppListingServiceImpl) FetchAllDevtronManagedApps() ([]AppNameTypeIdC } return apps, nil } +func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.JobsContainer, error) { + + jobListingFilter := helper.AppListingFilter{ + Statuses: fetchJobListingRequest.Statuses, + Teams: fetchJobListingRequest.Teams, + AppNameSearch: fetchJobListingRequest.AppNameSearch, + SortOrder: fetchJobListingRequest.SortOrder, + SortBy: fetchJobListingRequest.SortBy, + Offset: fetchJobListingRequest.Offset, + Size: fetchJobListingRequest.Size, + DeploymentGroupId: fetchJobListingRequest.DeploymentGroupId, + AppStatuses: fetchJobListingRequest.AppStatuses, + } + jobContainers, err := impl.appListingRepository.FetchJobs(jobListingFilter) + if err != nil { + impl.Logger.Errorw("error in fetching app list", "error", err) + return []*bean.JobsContainer{}, err + } + return jobContainers, err +} func (impl AppListingServiceImpl) FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") // TODO: check statuses diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index b8421f689b..1871aa2bb0 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -20,11 +20,8 @@ package appClone import ( "context" bean2 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/constants" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/chart" - "github.com/go-pg/pg" "strings" "fmt" @@ -88,19 +85,19 @@ type CloneRequest struct { func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template app - templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { - impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) - err = &util.ApiError{ - Code: constants.AppDoesNotExist.Code, - InternalMessage: "app does not exist", - UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - } - return nil, err - } + //templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + //if err != nil && err != pg.ErrNoRows { + // return nil, err + //} + //if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { + // impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) + // err = &util.ApiError{ + // Code: constants.AppDoesNotExist.Code, + // InternalMessage: "app does not exist", + // UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + // } + // return nil, err + //} //create new app cloneReq := &CloneRequest{ RefAppId: createReq.TemplateId, @@ -209,19 +206,19 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template job - templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { - impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) - err = &util.ApiError{ - Code: constants.AppDoesNotExist.Code, - InternalMessage: "job does not exist", - UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - } - return nil, err - } + //templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + //if err != nil && err != pg.ErrNoRows { + // return nil, err + //} + //if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { + // impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) + // err = &util.ApiError{ + // Code: constants.AppDoesNotExist.Code, + // InternalMessage: "job does not exist", + // UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + // } + // return nil, err + //} //create new job cloneReq := &CloneRequest{ diff --git a/pkg/bean/app.go b/pkg/bean/app.go index f007f52e17..85eb48cfa3 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -38,13 +38,14 @@ type SourceTypeConfig struct { } type CreateAppDTO struct { - Id int `json:"id,omitempty" validate:"number"` - AppName string `json:"appName" validate:"name-component,max=100"` - UserId int32 `json:"-"` //not exposed to UI - Material []*GitMaterial `json:"material" validate:"dive,min=1"` - TeamId int `json:"teamId,omitempty" validate:"number,required"` - TemplateId int `json:"templateId"` - AppLabels []*Label `json:"labels,omitempty" validate:"dive"` + Id int `json:"id,omitempty" validate:"number"` + AppName string `json:"appName" validate:"name-component,max=100"` + UserId int32 `json:"-"` //not exposed to UI + Material []*GitMaterial `json:"material" validate:"dive,min=1"` + TeamId int `json:"teamId,omitempty" validate:"number,required"` + TemplateId int `json:"templateId"` + AppLabels []*Label `json:"labels,omitempty" validate:"dive"` + Description string `json:"description"` } type CreateMaterialDTO struct { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index d8149521ee..4a584bab7b 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -851,7 +851,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateJob(createRequest *bean.CreateApp } // Rollback tx on error. defer tx.Rollback() - app, err := impl.createJobGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, tx) + app, err := impl.createJobGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.Description, tx) if err != nil { return nil, err } @@ -1083,7 +1083,7 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 return pg, nil } -func (impl CiCdPipelineOrchestratorImpl) createJobGroup(name string, userId int32, teamId int, tx *pg.Tx) (*app2.App, error) { +func (impl CiCdPipelineOrchestratorImpl) createJobGroup(name string, userId int32, teamId int, description string, tx *pg.Tx) (*app2.App, error) { app, err := impl.appRepository.FindActiveByName(name) if err != nil && err != pg.ErrNoRows { return nil, err @@ -1098,11 +1098,12 @@ func (impl CiCdPipelineOrchestratorImpl) createJobGroup(name string, userId int3 return nil, err } pg := &app2.App{ - Active: true, - AppName: name, - TeamId: teamId, - AppStore: 2, - AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, + Active: true, + AppName: name, + TeamId: teamId, + AppStore: 2, + Description: description, + AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } err = impl.appRepository.SaveWithTxn(pg, tx) if err != nil { From 9d68eaa42c1b3d2b375f751448715ec3e8cf99e6 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 14:02:06 +0530 Subject: [PATCH 017/118] Uodated app listing api restHandler --- api/restHandler/AppListingRestHandler.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index a255c57226..e97c63ec01 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -175,9 +175,19 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + _, err = handler.appListingService.FetchJobs(fetchJobListingRequest, w, r) + // Apply pagination + appsCount := len(apps) + offset := fetchJobListingRequest.Offset + limit := fetchJobListingRequest.Size - var dg *deploymentGroup. - + if offset+limit <= len(apps) { + apps = apps[offset : offset+limit] + } else { + apps = apps[offset:] + } + jobContainerResponse := bean.JobsContainer{} + common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) } func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) { //Allow CORS here By * or specific origin From 5dbc6c7f0fe942baa46dad3ed21118a6cfac6ee9 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 14:10:36 +0530 Subject: [PATCH 018/118] Updated app listing api restHandler --- api/restHandler/AppListingRestHandler.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index e97c63ec01..674149bc20 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -175,16 +175,16 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - _, err = handler.appListingService.FetchJobs(fetchJobListingRequest, w, r) + fetchJob, err := handler.appListingService.FetchJobs(fetchJobListingRequest, w, r) // Apply pagination - appsCount := len(apps) + offset := fetchJobListingRequest.Offset limit := fetchJobListingRequest.Size - if offset+limit <= len(apps) { - apps = apps[offset : offset+limit] + if offset+limit <= len(fetchJob) { + fetchJob = fetchJob[offset : offset+limit] } else { - apps = apps[offset:] + fetchJob = fetchJob[offset:] } jobContainerResponse := bean.JobsContainer{} common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) From 4ec7114d1429265f0476dd3fced593b72febbb7e Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 14:17:16 +0530 Subject: [PATCH 019/118] Updated app listing api service layer --- pkg/app/AppListingService.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 11db5a4d48..20069bc3b4 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -221,18 +221,16 @@ func (impl AppListingServiceImpl) FetchAllDevtronManagedApps() ([]AppNameTypeIdC } return apps, nil } -func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.JobsContainer, error) { +func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request) ([]*bean.JobsContainer, error) { jobListingFilter := helper.AppListingFilter{ - Statuses: fetchJobListingRequest.Statuses, - Teams: fetchJobListingRequest.Teams, - AppNameSearch: fetchJobListingRequest.AppNameSearch, - SortOrder: fetchJobListingRequest.SortOrder, - SortBy: fetchJobListingRequest.SortBy, - Offset: fetchJobListingRequest.Offset, - Size: fetchJobListingRequest.Size, - DeploymentGroupId: fetchJobListingRequest.DeploymentGroupId, - AppStatuses: fetchJobListingRequest.AppStatuses, + Teams: fetchJobListingRequest.Teams, + AppNameSearch: fetchJobListingRequest.AppNameSearch, + SortOrder: fetchJobListingRequest.SortOrder, + SortBy: fetchJobListingRequest.SortBy, + Offset: fetchJobListingRequest.Offset, + Size: fetchJobListingRequest.Size, + AppStatuses: fetchJobListingRequest.AppStatuses, } jobContainers, err := impl.appListingRepository.FetchJobs(jobListingFilter) if err != nil { From d117efb18b42347a128c0ee6bb3c9cdd07f10025 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 15:31:27 +0530 Subject: [PATCH 020/118] Updated app listing api service layer --- api/bean/AppView.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 6e36cb9166..0a13b71809 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -57,7 +57,6 @@ type JobsContainer struct { JobName string `json:"name""` LastRun []LastRunStatus `json:"last-run"'` Description string `json:"description"` - ProjectId int `json:"projectId"` } type LastRunStatus struct { From 0c7c51a1fb7976accba42a0cb36fd44695a04a85 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 15:33:34 +0530 Subject: [PATCH 021/118] Updated app listing api service layer --- api/bean/AppView.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 0a13b71809..73934789ab 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -20,6 +20,7 @@ package bean import ( "encoding/json" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "time" ) type AppContainer struct { @@ -60,10 +61,10 @@ type JobsContainer struct { } type LastRunStatus struct { - PipelineId int `json:"pipeline-id"` - LastRunStatus int `json:"lastRunStatus"` - LastRunAt string `json:"lastRunAt"` - LastSuccessAt string `json:"lastRunAt"` + PipelineId int `json:"pipeline-id"` + LastRunStatus string `json:"lastRunStatus"` + LastRunAt time.Time `json:"lastRunAt"` + LastSuccessAt time.Time `json:"lastRunAt"` } type AppEnvironmentContainer struct { From 0af2f3512ce7a5b2044e492340f9c8d32b32d227 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Feb 2023 15:43:43 +0530 Subject: [PATCH 022/118] Updated app listing api service layer --- api/restHandler/AppListingRestHandler.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 674149bc20..2cdcc4b89d 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -158,14 +158,11 @@ func (handler AppListingRestHandlerImpl) FetchAllDevtronManagedApps(w http.Respo common.WriteJsonResp(w, err, res, http.StatusOK) } func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *http.Request) { - newCtx, span := otel.Tracer("userService").Start(r.Context(), "GetLoggedInUser") userId, err := handler.userService.GetLoggedInUser(r) - span.End() if userId == 0 || err != nil { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - newCtx, span = otel.Tracer("userService").Start(newCtx, "GetById") var fetchJobListingRequest app.FetchAppListingRequest decoder := json.NewDecoder(r.Body) From 6f77fecb62ed1247fb4f938eb69a41b60ec276f6 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Feb 2023 11:25:06 +0530 Subject: [PATCH 023/118] Updated app listing api service layer --- api/bean/AppView.go | 4 ++++ api/restHandler/AppListingRestHandler.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 73934789ab..3ddabbab66 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -67,6 +67,10 @@ type LastRunStatus struct { LastSuccessAt time.Time `json:"lastRunAt"` } +type JobListingContainer struct { + JobId int `json:"jobId"` + JobName string `json:"jobName"` +} type AppEnvironmentContainer struct { AppId int `json:"appId"` AppName string `json:"appName"` diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 2cdcc4b89d..4e85138a1d 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -183,8 +183,8 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt } else { fetchJob = fetchJob[offset:] } - jobContainerResponse := bean.JobsContainer{} - common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) + //jobContainerResponse := bean.JobsContainer{} + common.WriteJsonResp(w, err, fetchJob, http.StatusOK) } func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) { //Allow CORS here By * or specific origin From dfea05a32d06c945f7006db485a88a50fcf3c8ea Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 18 Feb 2023 14:19:05 +0530 Subject: [PATCH 024/118] Updated app listing api service layer --- api/bean/AppView.go | 34 +++++++---- api/restHandler/AppListingRestHandler.go | 29 ++++++--- .../app/AutoCompleteRestHandler.go | 4 +- .../app/BuildPipelineRestHandler.go | 14 +++++ .../sql/repository/AppListingRepository.go | 14 ++--- internal/sql/repository/app/AppRepository.go | 27 +++++++- .../AppListingRepositoryQueryBuilder.go | 16 ++--- pkg/app/AppListingService.go | 49 +++++++++++++-- pkg/appClone/AppCloneService.go | 61 ++++++++++--------- pkg/bean/app.go | 1 + pkg/pipeline/PipelineBuilder.go | 6 +- wire_gen.go | 5 +- 12 files changed, 182 insertions(+), 78 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 3ddabbab66..0a7bbff644 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -37,6 +37,11 @@ type AppContainerResponse struct { DeploymentGroupDTO DeploymentGroupDTO `json:"deploymentGroup,omitempty"` } +type JobContainerResponse struct { + JobContainers []*JobContainer `json:"jobContainers"` + JobCount int `json:"jobCount"` +} + type DeploymentGroupDTO struct { Id int `json:"id"` Name string `json:"name"` @@ -53,24 +58,31 @@ type CiMaterialDTO struct { SourceValue string `json:"value"` } -type JobsContainer struct { - JobId int `json:"id"` - JobName string `json:"name""` - LastRun []LastRunStatus `json:"last-run"'` - Description string `json:"description"` +type JobContainer struct { + AppId int `json:"app_id"` + AppName string `json:"app_name""` + Description string `json:"description"` + JobCiPipelines []JobCIPipeline `json:"ci_pipelines"'` } -type LastRunStatus struct { - PipelineId int `json:"pipeline-id"` - LastRunStatus string `json:"lastRunStatus"` +type JobCIPipeline struct { + CiPipelineId int `json:"ciPipelineId"` + Status string `json:"status"` LastRunAt time.Time `json:"lastRunAt"` - LastSuccessAt time.Time `json:"lastRunAt"` + LastSuccessAt time.Time `json:"lastSuccessAt"` } type JobListingContainer struct { - JobId int `json:"jobId"` - JobName string `json:"jobName"` + AppId int `json:"app_id"` + AppName string `json:"app_name"` + Description string `json:"-"` + CiPipelineID int `json:"ci_pipeline_id"` + CiPipelineName string `json:"ci_pipeline_name"` + Status string `json:"status"` + StartedOn time.Time `json:"started_on"` + LastSuccessAt time.Time `json:"-"` } + type AppEnvironmentContainer struct { AppId int `json:"appId"` AppName string `json:"appName"` diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 4e85138a1d..e73ebcf832 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -172,19 +172,28 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - fetchJob, err := handler.appListingService.FetchJobs(fetchJobListingRequest, w, r) - // Apply pagination + jobs, err := handler.appListingService.FetchJobs(fetchJobListingRequest) + if err != nil { + handler.logger.Errorw("service err, FetchJobs", "err", err, "payload", fetchJobListingRequest) + common.WriteJsonResp(w, err, "", http.StatusInternalServerError) + } - offset := fetchJobListingRequest.Offset - limit := fetchJobListingRequest.Size + //jobsCount := len(jobs) + //offset := fetchJobListingRequest.Offset + //limit := fetchJobListingRequest.Size + // + //if offset+limit <= len(jobs) { + // fetchJob = fetchJob[offset : offset+limit] + //} else { + // fetchJob = fetchJob[offset:] + //} - if offset+limit <= len(fetchJob) { - fetchJob = fetchJob[offset : offset+limit] - } else { - fetchJob = fetchJob[offset:] + jobContainerResponse := bean.JobContainerResponse{ + JobContainers: jobs, + JobCount: len(jobs), } - //jobContainerResponse := bean.JobsContainer{} - common.WriteJsonResp(w, err, fetchJob, http.StatusOK) + + common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) } func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) { //Allow CORS here By * or specific origin diff --git a/api/restHandler/app/AutoCompleteRestHandler.go b/api/restHandler/app/AutoCompleteRestHandler.go index 0239249a61..65035a5060 100644 --- a/api/restHandler/app/AutoCompleteRestHandler.go +++ b/api/restHandler/app/AutoCompleteRestHandler.go @@ -32,10 +32,12 @@ func (handler PipelineConfigRestHandlerImpl) GetAppListForAutocomplete(w http.Re v := r.URL.Query() teamId := v.Get("teamId") appName := v.Get("appName") + isJobParam := v.Get("isJob") + isJob := isJobParam == "true" handler.Logger.Infow("request payload, GetAppListForAutocomplete", "teamId", teamId) var apps []*pipeline.AppBean if len(teamId) == 0 { - apps, err = handler.pipelineBuilder.FindAllMatchesByAppName(appName) + apps, err = handler.pipelineBuilder.FindAllMatchesByAppName(appName, isJob) if err != nil { handler.Logger.Errorw("service err, GetAppListForAutocomplete", "err", err, "teamId", teamId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 887fd2353f..d87a9981b0 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -214,6 +214,20 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + var ciConfig *bean.CiConfigRequest + ciConfig, err = impl.getCiTemplateVariables(patchRequest.AppId) + if err != nil { + impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", patchRequest.AppId) + return nil, err + } + ciConfig.AppWorkflowId = patchRequest.AppWorkflowId + if patchRequest.IsJob && patchRequest.AppWorkflowId == 0 { + var createRequest = bean.CiConfigRequest{} + createRequest.AppId = patchRequest.AppId + createRequest.CiBuildConfig.CiBuildType = "skip-build" + createRequest.CiBuildConfig.GitMaterialId = patchRequest.CiPipeline.CiMaterial[0].GitMaterialId + handler.pipelineBuilder.CreateCiPipeline(&createRequest) + } handler.Logger.Infow("request payload, PatchCiPipelines", "PatchCiPipelines", patchRequest) err = handler.validator.Struct(patchRequest) if err != nil { diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index f90d19ad36..ad7e0bfa19 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -37,7 +37,7 @@ import ( type AppListingRepository interface { FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) - FetchJobs(appListingFilter helper.AppListingFilter) ([]*bean.JobsContainer, error) + FetchJobs(appIds []int) ([]*bean.JobListingContainer, error) DeploymentDetailsByAppIdAndEnvId(ctx context.Context, appId int, envId int) (bean.DeploymentDetailContainer, error) FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -85,16 +85,16 @@ func NewAppListingRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB, * It will return the list of filtered apps with details related to each env */ -func (impl AppListingRepositoryImpl) FetchJobs(jobListingFilter helper.AppListingFilter) ([]*bean.JobsContainer, error) { - var jobContainer []*bean.JobsContainer - jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(jobListingFilter) +func (impl AppListingRepositoryImpl) FetchJobs(appIds []int) ([]*bean.JobListingContainer, error) { + var jobContainers []*bean.JobListingContainer + jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery() impl.Logger.Debugw("basic app detail query: ", jobsQuery) - _, appsErr := impl.dbConnection.Query(&jobContainer, jobsQuery) + _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, pg.In(appIds)) if appsErr != nil { impl.Logger.Error(appsErr) - return jobContainer, appsErr + return jobContainers, appsErr } - return jobContainer, nil + return jobContainers, nil } func (impl AppListingRepositoryImpl) FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 27b13d3c86..3098c5947e 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -19,6 +19,7 @@ package app import ( "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/pkg/team" "github.com/go-pg/pg" @@ -59,12 +60,13 @@ type AppRepository interface { FindAppAndProjectByAppId(appId int) (*App, error) FindAppAndProjectByAppName(appName string) (*App, error) GetConnection() *pg.DB - FindAllMatchesByAppName(appName string) ([]*App, error) + FindAllMatchesByAppName(appName string, isJob bool) ([]*App, error) FindIdsByTeamIdsAndTeamNames(teamIds []int, teamNames []string) ([]int, error) FindIdsByNames(appNames []string) ([]int, error) FetchAllActiveInstalledAppsWithAppIdAndName() ([]*App, error) FetchAllActiveDevtronAppsWithAppIdAndName() ([]*App, error) FindEnvironmentIdForInstalledApp(appId int) (int, error) + FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) } const DevtronApp = "DevtronApp" @@ -250,9 +252,13 @@ func (repo AppRepositoryImpl) FindAppAndProjectByAppName(appName string) (*App, return app, err } -func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string) ([]*App, error) { +func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string, isJob bool) ([]*App, error) { var apps []*App - err := repo.dbConnection.Model(&apps).Where("app_name ILIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", 0).Select() + appStore := 0 + if isJob { + appStore = 2 + } + err := repo.dbConnection.Model(&apps).Where("app_name ILIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", appStore).Select() return apps, err } @@ -334,3 +340,18 @@ func (repo AppRepositoryImpl) FindEnvironmentIdForInstalledApp(appId int) (int, _, err := repo.dbConnection.Query(&res, query, appId) return res.envId, err } +func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) { + type AppId struct { + Id int `json:"id"` + } + var appIds []AppId + query := fmt.Sprintf("select id "+ + "from app where app_name like "+"%"+jobListingFilter.AppNameSearch+"%"+" and team_id in ? order by %s %s limit %s offset %s", + jobListingFilter.SortBy, jobListingFilter.SortOrder, jobListingFilter.Size, jobListingFilter.Offset) + _, err := repo.dbConnection.Query(&appIds, query, pg.In(jobListingFilter.Teams)) + appIdsResult := make([]int, 0) + for _, id := range appIds { + appIdsResult = append(appIdsResult, id.Id) + } + return appIdsResult, err +} diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index a627dc31b8..384982b6dd 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -60,14 +60,14 @@ const ( AppNameSortBy SortBy = "appNameSort" ) -func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appListingFilter AppListingFilter) string { - whereCondition := impl.buildJobListingWhereCondition(appListingFilter) - orderByClause := impl.buildJobListingSortBy(appListingFilter) - - query := "SELECT app_id, a.app_name, a.team_id, a.description" + "status, started_on, finished_on" + - "" - - query = query + whereCondition + orderByClause +func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery() string { + query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as app_id,app.app_name,cwr.started_on,cwr.status " + + "from ci_pipeline left join " + + "(select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw " + + "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + + "on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr" + + " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + + "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = false and app.id in ?;" return query } diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 20069bc3b4..7be37266ce 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -53,8 +53,9 @@ import ( type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) - FetchJobs(fetchJobListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request) ([]*bean.JobsContainer, error) + FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) + BuildJobListingResponse(jobContainers []*bean.JobListingContainer) []*bean.JobContainer FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) FetchAppDetails(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -221,7 +222,7 @@ func (impl AppListingServiceImpl) FetchAllDevtronManagedApps() ([]AppNameTypeIdC } return apps, nil } -func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request) ([]*bean.JobsContainer, error) { +func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) { jobListingFilter := helper.AppListingFilter{ Teams: fetchJobListingRequest.Teams, @@ -232,12 +233,14 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi Size: fetchJobListingRequest.Size, AppStatuses: fetchJobListingRequest.AppStatuses, } - jobContainers, err := impl.appListingRepository.FetchJobs(jobListingFilter) + appIds, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) + jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) - return []*bean.JobsContainer{}, err + return []*bean.JobContainer{}, err } - return jobContainers, err + jobContainers := impl.BuildJobListingResponse(jobListingContainers) + return jobContainers, nil } func (impl AppListingServiceImpl) FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") @@ -355,6 +358,42 @@ func (impl AppListingServiceImpl) BuildAppListingResponse(fetchAppListingRequest appContainerResponses, err := impl.appListingViewBuilder.BuildView(fetchAppListingRequest, appEnvMapping) return appContainerResponses, err } +func (impl AppListingServiceImpl) BuildJobListingResponse(jobContainers []*bean.JobListingContainer) []*bean.JobContainer { + jobContainersMapping := make(map[int]bean.JobContainer) + var appIds []int + //Storing the sequence in appIds array + for _, jobContainer := range jobContainers { + val, ok := jobContainersMapping[jobContainer.AppId] + if !ok { + appIds = append(appIds, jobContainer.AppId) + val = bean.JobContainer{} + val.AppId = jobContainer.AppId + val.AppName = jobContainer.AppName + val.Description = jobContainer.Description + } + + if len(val.JobCiPipelines) == 0 { + val.JobCiPipelines = make([]bean.JobCIPipeline, 0) + } + ciPipelineObj := bean.JobCIPipeline{ + CiPipelineId: jobContainer.CiPipelineID, + Status: jobContainer.Status, + LastRunAt: jobContainer.StartedOn, + LastSuccessAt: jobContainer.LastSuccessAt, + } + val.JobCiPipelines = append(val.JobCiPipelines, ciPipelineObj) + jobContainersMapping[jobContainer.AppId] = val + + } + + result := make([]*bean.JobContainer, 0) + for _, appId := range appIds { + val := jobContainersMapping[appId] + result = append(result, &val) + } + + return result +} func (impl AppListingServiceImpl) fetchACDAppStatus(fetchAppListingRequest FetchAppListingRequest, existingAppEnvContainers []*bean.AppEnvironmentContainer) (map[string][]*bean.AppEnvironmentContainer, error) { appEnvMapping := make(map[string][]*bean.AppEnvironmentContainer) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 1871aa2bb0..1c16ef16c1 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -20,8 +20,11 @@ package appClone import ( "context" bean2 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/internal/constants" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/chart" + "github.com/go-pg/pg" "strings" "fmt" @@ -39,7 +42,6 @@ type AppCloneService interface { CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) } type AppCloneServiceImpl struct { - appRepository app2.AppRepository logger *zap.SugaredLogger pipelineBuilder pipeline.PipelineBuilder materialRepository pipelineConfig.MaterialRepository @@ -50,6 +52,7 @@ type AppCloneServiceImpl struct { propertiesConfigService pipeline.PropertiesConfigService pipelineStageService pipeline.PipelineStageService ciTemplateService pipeline.CiTemplateService + appRepository app2.AppRepository } func NewAppCloneServiceImpl(logger *zap.SugaredLogger, @@ -61,7 +64,8 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, appListingService app.AppListingService, propertiesConfigService pipeline.PropertiesConfigService, ciTemplateOverrideRepository pipelineConfig.CiTemplateOverrideRepository, - pipelineStageService pipeline.PipelineStageService, ciTemplateService pipeline.CiTemplateService) *AppCloneServiceImpl { + pipelineStageService pipeline.PipelineStageService, ciTemplateService pipeline.CiTemplateService, + appRepository app2.AppRepository) *AppCloneServiceImpl { return &AppCloneServiceImpl{ logger: logger, pipelineBuilder: pipelineBuilder, @@ -73,6 +77,7 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, propertiesConfigService: propertiesConfigService, pipelineStageService: pipelineStageService, ciTemplateService: ciTemplateService, + appRepository: appRepository, } } @@ -85,19 +90,19 @@ type CloneRequest struct { func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template app - //templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - //if err != nil && err != pg.ErrNoRows { - // return nil, err - //} - //if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { - // impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) - // err = &util.ApiError{ - // Code: constants.AppDoesNotExist.Code, - // InternalMessage: "app does not exist", - // UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - // } - // return nil, err - //} + templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { + impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) + err = &util.ApiError{ + Code: constants.AppDoesNotExist.Code, + InternalMessage: "app does not exist", + UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + } + return nil, err + } //create new app cloneReq := &CloneRequest{ RefAppId: createReq.TemplateId, @@ -206,19 +211,19 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { //validate template job - //templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - //if err != nil && err != pg.ErrNoRows { - // return nil, err - //} - //if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { - // impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) - // err = &util.ApiError{ - // Code: constants.AppDoesNotExist.Code, - // InternalMessage: "job does not exist", - // UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - // } - // return nil, err - //} + templateApp, err := impl.appRepository.FindById(createReq.TemplateId) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { + impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) + err = &util.ApiError{ + Code: constants.AppDoesNotExist.Code, + InternalMessage: "job does not exist", + UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), + } + return nil, err + } //create new job cloneReq := &CloneRequest{ diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 85eb48cfa3..dd8a081d19 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -206,6 +206,7 @@ type CiPatchRequest struct { Action PatchAction `json:"action"` AppWorkflowId int `json:"appWorkflowId,omitempty"` UserId int32 `json:"-"` + IsJob bool `json:"isJob""` } type CiRegexPatchRequest struct { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 290ecd38c3..f918453099 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -119,7 +119,7 @@ type PipelineBuilder interface { GetCiPipelineById(pipelineId int) (ciPipeline *bean.CiPipeline, err error) GetMaterialsForAppId(appId int) []*bean.GitMaterial - FindAllMatchesByAppName(appName string) ([]*AppBean, error) + FindAllMatchesByAppName(appName string, isJob bool) ([]*AppBean, error) GetEnvironmentByCdPipelineId(pipelineId int) (int, error) PatchRegexCiPipeline(request *bean.CiRegexPatchRequest) (err error) @@ -2821,14 +2821,14 @@ func (impl PipelineBuilderImpl) GetCiPipelineById(pipelineId int) (ciPipeline *b return ciPipeline, err } -func (impl PipelineBuilderImpl) FindAllMatchesByAppName(appName string) ([]*AppBean, error) { +func (impl PipelineBuilderImpl) FindAllMatchesByAppName(appName string, isJob bool) ([]*AppBean, error) { var appsRes []*AppBean var apps []*app2.App var err error if len(appName) == 0 { apps, err = impl.appRepo.FindAll() } else { - apps, err = impl.appRepo.FindAllMatchesByAppName(appName) + apps, err = impl.appRepo.FindAllMatchesByAppName(appName, isJob) } if err != nil { impl.logger.Errorw("error while fetching app", "err", err) diff --git a/wire_gen.go b/wire_gen.go index 91dfae2674..2f1546c907 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,7 +1,8 @@ // Code generated by Wire. DO NOT EDIT. //go:generate wire -//+build !wireinject +//go:build !wireinject +// +build !wireinject package main @@ -412,7 +413,7 @@ func InitializeApp() (*App, error) { cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, cdConfig, userServiceImpl, cdWorkflowRepositoryImpl, cdWorkflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, ciConfig, helmAppServiceImpl, pipelineOverrideRepositoryImpl, workflowDagExecutorImpl, appListingServiceImpl, appListingRepositoryImpl, pipelineStatusTimelineRepositoryImpl, applicationServiceClientImpl, argoUserServiceImpl, deploymentEventHandlerImpl, eventRESTClientImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl) - appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl) + appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, materialRepositoryImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, ciTemplateOverrideRepositoryImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl) imageScanObjectMetaRepositoryImpl := security.NewImageScanObjectMetaRepositoryImpl(db, sugaredLogger) cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) From 837fa267b95599b0e660eb3e19a80527f816f5cc Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sun, 19 Feb 2023 01:54:50 +0530 Subject: [PATCH 025/118] MAde the patch-ci-pipeline api --- .../app/BuildPipelineRestHandler.go | 73 +++++++++++-------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index d87a9981b0..36990d8f4e 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -214,20 +214,7 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - var ciConfig *bean.CiConfigRequest - ciConfig, err = impl.getCiTemplateVariables(patchRequest.AppId) - if err != nil { - impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", patchRequest.AppId) - return nil, err - } - ciConfig.AppWorkflowId = patchRequest.AppWorkflowId - if patchRequest.IsJob && patchRequest.AppWorkflowId == 0 { - var createRequest = bean.CiConfigRequest{} - createRequest.AppId = patchRequest.AppId - createRequest.CiBuildConfig.CiBuildType = "skip-build" - createRequest.CiBuildConfig.GitMaterialId = patchRequest.CiPipeline.CiMaterial[0].GitMaterialId - handler.pipelineBuilder.CreateCiPipeline(&createRequest) - } + handler.Logger.Infow("request payload, PatchCiPipelines", "PatchCiPipelines", patchRequest) err = handler.validator.Struct(patchRequest) if err != nil { @@ -242,31 +229,57 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { + var ok bool + if patchRequest.IsJob { + ok = handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionCreate, resourceName) + } else { + ok = handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName) + } + if !ok { common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) return } - pipelineData, err := handler.pipelineRepository.FindActiveByAppIdAndPipelineId(patchRequest.AppId, patchRequest.CiPipeline.Id) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - var environmentIds []int - for _, pipeline := range pipelineData { - environmentIds = append(environmentIds, pipeline.EnvironmentId) + if patchRequest.IsJob { + ciConfigRequest := bean.CiConfigRequest{} + ciConfigRequest.AppId = patchRequest.AppId + ciConfigRequest.CiBuildConfig.CiBuildType = "skip-build" + if patchRequest.CiPipeline == nil || patchRequest.CiPipeline.CiMaterial == nil || len(patchRequest.CiPipeline.CiMaterial) != 1 { + handler.Logger.Errorw("Invalid patch ci-pipeline request", "request", patchRequest, "err", "invalid CiPipeline data") + common.WriteJsonResp(w, fmt.Errorf("invalid CiPipeline data"), nil, http.StatusBadRequest) + return + } + ciConfigRequest.CiBuildConfig.GitMaterialId = patchRequest.CiPipeline.CiMaterial[0].GitMaterialId + _, err = handler.pipelineBuilder.CreateCiPipeline(&ciConfigRequest) + if err != nil { + handler.Logger.Errorw("error occurred in creating ci-pipeline for the Job", "payload", ciConfigRequest, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } } - if handler.appWorkflowService.CheckCdPipelineByCiPipelineId(patchRequest.CiPipeline.Id) { - for _, envId := range environmentIds { - envObject := handler.enforcerUtil.GetEnvRBACNameByCiPipelineIdAndEnvId(patchRequest.CiPipeline.Id, envId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, envObject); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return + + if !patchRequest.IsJob { + pipelineData, err := handler.pipelineRepository.FindActiveByAppIdAndPipelineId(patchRequest.AppId, patchRequest.CiPipeline.Id) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + var environmentIds []int + for _, pipeline := range pipelineData { + environmentIds = append(environmentIds, pipeline.EnvironmentId) + } + if handler.appWorkflowService.CheckCdPipelineByCiPipelineId(patchRequest.CiPipeline.Id) { + for _, envId := range environmentIds { + envObject := handler.enforcerUtil.GetEnvRBACNameByCiPipelineIdAndEnvId(patchRequest.CiPipeline.Id, envId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, envObject); !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } } } } - createResp, err := handler.pipelineBuilder.PatchCiPipeline(&patchRequest) if err != nil { handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) From e409097ffc275c833516bc06ba06b2a5468f73c4 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sun, 19 Feb 2023 06:00:11 +0530 Subject: [PATCH 026/118] MAde the patch-ci-pipeline api --- api/restHandler/AppListingRestHandler.go | 1 + .../app/BuildPipelineRestHandler.go | 3 ++ .../app/PipelineConfigRestHandler.go | 32 +++++++++---------- .../sql/repository/AppListingRepository.go | 4 ++- internal/sql/repository/app/AppRepository.go | 16 +++++++--- .../AppListingRepositoryQueryBuilder.go | 12 ++++++- pkg/app/AppListingService.go | 4 +++ pkg/bean/app.go | 2 +- 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index e73ebcf832..6cf506681b 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -176,6 +176,7 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt if err != nil { handler.logger.Errorw("service err, FetchJobs", "err", err, "payload", fetchJobListingRequest) common.WriteJsonResp(w, err, "", http.StatusInternalServerError) + return } //jobsCount := len(jobs) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 36990d8f4e..7ba58be226 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -12,6 +12,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" + bean1 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/user/casbin" "github.com/gorilla/mux" "io" @@ -245,7 +246,9 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri if patchRequest.IsJob { ciConfigRequest := bean.CiConfigRequest{} ciConfigRequest.AppId = patchRequest.AppId + ciConfigRequest.CiBuildConfig = &bean1.CiBuildConfigBean{} ciConfigRequest.CiBuildConfig.CiBuildType = "skip-build" + ciConfigRequest.UserId = patchRequest.UserId if patchRequest.CiPipeline == nil || patchRequest.CiPipeline.CiMaterial == nil || len(patchRequest.CiPipeline.CiMaterial) != 1 { handler.Logger.Errorw("Invalid patch ci-pipeline request", "request", patchRequest, "err", "invalid CiPipeline data") common.WriteJsonResp(w, fmt.Errorf("invalid CiPipeline data"), nil, http.StatusBadRequest) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 11b8bb23ee..469fa58273 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -284,7 +284,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r } func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get("token") + //token := r.Header.Get("token") decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { @@ -292,11 +292,11 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r return } - //isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - //if !isSuperAdmin || err != nil { - // common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - // return - //} + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } var createRequest bean.CreateAppDTO err = decoder.Decode(&createRequest) @@ -315,16 +315,16 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r return } - project, err := handler.teamService.FetchOne(createRequest.TeamId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, fmt.Sprintf("%s/%s", strings.ToLower(project.Name), "*")); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } + //project, err := handler.teamService.FetchOne(createRequest.TeamId) + //if err != nil { + // common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + // return + //} + //// with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) + //if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, fmt.Sprintf("%s/%s", strings.ToLower(project.Name), "*")); !ok { + // common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + // return + //} var createResp *bean.CreateAppDTO err = nil if createRequest.TemplateId == 0 { diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index ad7e0bfa19..ad6ae3d418 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -24,6 +24,7 @@ import ( "context" "encoding/json" "fmt" + "go.opentelemetry.io/otel" "strconv" "strings" @@ -88,8 +89,9 @@ It will return the list of filtered apps with details related to each env func (impl AppListingRepositoryImpl) FetchJobs(appIds []int) ([]*bean.JobListingContainer, error) { var jobContainers []*bean.JobListingContainer jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery() + appIdsString := helper.GetCommaSepratedString(appIds) impl.Logger.Debugw("basic app detail query: ", jobsQuery) - _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, pg.In(appIds)) + _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, appIdsString) if appsErr != nil { impl.Logger.Error(appsErr) return jobContainers, appsErr diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 3098c5947e..613a40ee53 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -345,10 +345,18 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL Id int `json:"id"` } var appIds []AppId - query := fmt.Sprintf("select id "+ - "from app where app_name like "+"%"+jobListingFilter.AppNameSearch+"%"+" and team_id in ? order by %s %s limit %s offset %s", - jobListingFilter.SortBy, jobListingFilter.SortOrder, jobListingFilter.Size, jobListingFilter.Offset) - _, err := repo.dbConnection.Query(&appIds, query, pg.In(jobListingFilter.Teams)) + orderByCondition := " order by app_name " + if jobListingFilter.SortOrder == "DESC" { + orderByCondition += string(jobListingFilter.SortOrder) + } + orderByCondition += " limit ? offset ? " + whereCondition := " where app_name like ? and active = true and app_store = 2 " + if len(jobListingFilter.Teams) > 0 { + whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" + } + query := "select id " + "from app " + whereCondition + orderByCondition + appName := "%" + jobListingFilter.AppNameSearch + "%" + _, err := repo.dbConnection.Query(&appIds, query, appName, jobListingFilter.Size, jobListingFilter.Offset) appIdsResult := make([]int, 0) for _, id := range appIds { appIdsResult = append(appIdsResult, id.Id) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 384982b6dd..23298db13a 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -67,7 +67,7 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery() string { "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + "on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr" + " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + - "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = false and app.id in ?;" + "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 and app.id in (?)" return query } @@ -156,3 +156,13 @@ func (impl AppListingRepositoryQueryBuilder) buildAppListingWhereCondition(appLi } return whereCondition } +func GetCommaSepratedString(appIds []int) string { + appIdsString := "" + for i, appId := range appIds { + appIdsString += fmt.Sprintf("%d", appId) + if i != len(appIds)-1 { + appIdsString += "," + } + } + return appIdsString +} diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 7be37266ce..18b064d56b 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -234,6 +234,10 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi AppStatuses: fetchJobListingRequest.AppStatuses, } appIds, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) + if err != nil { + impl.Logger.Errorw("error in fetching app ids list", "error", err) + return []*bean.JobContainer{}, err + } jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index dd8a081d19..80ae55b647 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -206,7 +206,7 @@ type CiPatchRequest struct { Action PatchAction `json:"action"` AppWorkflowId int `json:"appWorkflowId,omitempty"` UserId int32 `json:"-"` - IsJob bool `json:"isJob""` + IsJob bool `json:"isJob"` } type CiRegexPatchRequest struct { From 86f6d0d3bb166b2dca12602013d77b7af44790a2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sun, 19 Feb 2023 17:44:24 +0530 Subject: [PATCH 027/118] Reduced Code duplicacy --- .../app/PipelineConfigRestHandler.go | 4 +- pkg/appClone/AppCloneService.go | 180 +++++++++--------- pkg/bean/app.go | 1 + pkg/pipeline/CiCdPipelineOrchestrator.go | 102 +--------- pkg/pipeline/PipelineBuilder.go | 10 - 5 files changed, 102 insertions(+), 195 deletions(-) diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 469fa58273..2a2929861a 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -328,7 +328,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r var createResp *bean.CreateAppDTO err = nil if createRequest.TemplateId == 0 { - createResp, err = handler.pipelineBuilder.CreateJob(&createRequest) + createResp, err = handler.pipelineBuilder.CreateApp(&createRequest) } else { ctx, cancel := context.WithCancel(r.Context()) if cn, ok := w.(http.CloseNotifier); ok { @@ -348,7 +348,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r return } ctx = context.WithValue(r.Context(), "token", acdToken) - createResp, err = handler.appCloneService.CloneJob(&createRequest, ctx) + createResp, err = handler.appCloneService.CloneApp(&createRequest, ctx) } if err != nil { handler.Logger.Errorw("service err, CreateApp", "err", err, "CreateApp", createRequest) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 1c16ef16c1..1443aa48fa 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -39,7 +39,7 @@ import ( type AppCloneService interface { CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) - CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) + //CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) } type AppCloneServiceImpl struct { logger *zap.SugaredLogger @@ -94,7 +94,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context if err != nil && err != pg.ErrNoRows { return nil, err } - if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 0) { + if (templateApp == nil && templateApp.Id == 0) || (createReq.IsJob == false && templateApp.AppStore != 0) || (createReq.IsJob == true && templateApp.AppStore != 2) { impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) err = &util.ApiError{ Code: constants.AppDoesNotExist.Code, @@ -209,94 +209,94 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context return app, nil } -func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { - //validate template job - templateApp, err := impl.appRepository.FindById(createReq.TemplateId) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { - impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) - err = &util.ApiError{ - Code: constants.AppDoesNotExist.Code, - InternalMessage: "job does not exist", - UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), - } - return nil, err - } - //create new job - - cloneReq := &CloneRequest{ - RefAppId: createReq.TemplateId, - Name: createReq.AppName, - ProjectId: createReq.TeamId, - AppLabels: createReq.AppLabels, - } - userId := createReq.UserId - appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) - if err != nil { - return nil, err - } - refApp, err := impl.pipelineBuilder.GetApp(cloneReq.RefAppId) - if err != nil { - return nil, err - } - isSameProject := refApp.TeamId == cloneReq.ProjectId - /* appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId)) - appStageStatus = append(appStageStatus, impl.makeAppStageStatus(1, "MATERIAL", materialExists)) - appStageStatus = append(appStageStatus, impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId)) - appStageStatus = append(appStageStatus, impl.makeAppStageStatus(3, "CI_PIPELINE", stages.CiPipelineId)) - appStageStatus = append(appStageStatus, impl.makeAppStageStatus(4, "CHART", stages.ChartId)) - appStageStatus = append(appStageStatus, impl.makeAppStageStatus(5, "CD_PIPELINE", stages.PipelineId)) - */ - refAppStatus := make(map[string]bool) - for _, as := range appStatus { - refAppStatus[as.StageName] = as.Status - } - - //TODO check stage of current app - if !refAppStatus["APP"] { - impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) - return nil, nil - } - app, err := impl.CreateApp(cloneReq, userId) - if err != nil { - impl.logger.Errorw("error in creating app", "req", cloneReq, "err", err) - return nil, err - } - newAppId := app.Id - if !refAppStatus["MATERIAL"] { - impl.logger.Errorw("status not", "MATERIAL", cloneReq.RefAppId) - return app, nil - } - _, gitMaerialMap, err := impl.CloneGitRepo(cloneReq.RefAppId, newAppId, userId) - if err != nil { - impl.logger.Errorw("error in cloning git", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) - return nil, err - } - - _, err = impl.CreateCiTemplate(cloneReq.RefAppId, newAppId, userId) - if err != nil { - impl.logger.Errorw("error in cloning docker template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) - return nil, err - } - if !refAppStatus["TEMPLATE"] { - impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) - return app, nil - } - if !refAppStatus["CHART"] { - impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) - return app, nil - } - - _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) - if err != nil { - impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) - return nil, err - } - - return app, nil -} +//func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { +// //validate template job +// templateApp, err := impl.appRepository.FindById(createReq.TemplateId) +// if err != nil && err != pg.ErrNoRows { +// return nil, err +// } +// if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { +// impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) +// err = &util.ApiError{ +// Code: constants.AppDoesNotExist.Code, +// InternalMessage: "job does not exist", +// UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), +// } +// return nil, err +// } +// //create new job +// +// cloneReq := &CloneRequest{ +// RefAppId: createReq.TemplateId, +// Name: createReq.AppName, +// ProjectId: createReq.TeamId, +// AppLabels: createReq.AppLabels, +// } +// userId := createReq.UserId +// appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) +// if err != nil { +// return nil, err +// } +// refApp, err := impl.pipelineBuilder.GetApp(cloneReq.RefAppId) +// if err != nil { +// return nil, err +// } +// isSameProject := refApp.TeamId == cloneReq.ProjectId +// /* appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId)) +// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(1, "MATERIAL", materialExists)) +// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId)) +// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(3, "CI_PIPELINE", stages.CiPipelineId)) +// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(4, "CHART", stages.ChartId)) +// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(5, "CD_PIPELINE", stages.PipelineId)) +// */ +// refAppStatus := make(map[string]bool) +// for _, as := range appStatus { +// refAppStatus[as.StageName] = as.Status +// } +// +// //TODO check stage of current app +// if !refAppStatus["APP"] { +// impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) +// return nil, nil +// } +// app, err := impl.CreateApp(cloneReq, userId) +// if err != nil { +// impl.logger.Errorw("error in creating app", "req", cloneReq, "err", err) +// return nil, err +// } +// newAppId := app.Id +// if !refAppStatus["MATERIAL"] { +// impl.logger.Errorw("status not", "MATERIAL", cloneReq.RefAppId) +// return app, nil +// } +// _, gitMaerialMap, err := impl.CloneGitRepo(cloneReq.RefAppId, newAppId, userId) +// if err != nil { +// impl.logger.Errorw("error in cloning git", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) +// return nil, err +// } +// +// _, err = impl.CreateCiTemplate(cloneReq.RefAppId, newAppId, userId) +// if err != nil { +// impl.logger.Errorw("error in cloning docker template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) +// return nil, err +// } +// if !refAppStatus["TEMPLATE"] { +// impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) +// return app, nil +// } +// if !refAppStatus["CHART"] { +// impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) +// return app, nil +// } +// +// _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) +// if err != nil { +// impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) +// return nil, err +// } +// +// return app, nil +//} func (impl *AppCloneServiceImpl) CreateApp(cloneReq *CloneRequest, userId int32) (*bean.CreateAppDTO, error) { createAppReq := &bean.CreateAppDTO{ diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 80ae55b647..1ec6b31b1b 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -46,6 +46,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` + IsJob bool `json:"isJob"` } type CreateMaterialDTO struct { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 4a584bab7b..d7b31547c4 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -58,7 +58,6 @@ import ( type CiCdPipelineOrchestrator interface { CreateApp(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) - CreateJob(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) DeleteApp(appId int, userId int32) error CreateMaterials(createMaterialRequest *bean.CreateMaterialDTO) (*bean.CreateMaterialDTO, error) UpdateMaterial(updateMaterialRequest *bean.UpdateMaterialDTO) (*bean.UpdateMaterialDTO, error) @@ -813,7 +812,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp } // Rollback tx on error. defer tx.Rollback() - app, err := impl.createAppGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, tx) + app, err := impl.createAppGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.IsJob, tx) if err != nil { return nil, err } @@ -843,43 +842,6 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp return createRequest, nil } -func (impl CiCdPipelineOrchestratorImpl) CreateJob(createRequest *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { - dbConnection := impl.appRepository.GetConnection() - tx, err := dbConnection.Begin() - if err != nil { - return nil, err - } - // Rollback tx on error. - defer tx.Rollback() - app, err := impl.createJobGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.Description, tx) - if err != nil { - return nil, err - } - // create labels and tags with app - if app.Active && len(createRequest.AppLabels) > 0 { - for _, label := range createRequest.AppLabels { - request := &bean.AppLabelDto{ - AppId: app.Id, - Key: label.Key, - Value: label.Value, - UserId: createRequest.UserId, - } - _, err := impl.appLabelsService.Create(request, tx) - if err != nil { - impl.logger.Errorw("error on creating labels for app id ", "err", err, "appId", app.Id) - return nil, err - } - } - } - err = tx.Commit() - if err != nil { - impl.logger.Errorw("error in commit repo", "error", err) - return nil, err - } - createRequest.Id = app.Id - return createRequest, nil -} - func (impl CiCdPipelineOrchestratorImpl) DeleteApp(appId int, userId int32) error { // Delete git materials,call git sensor and delete app impl.logger.Debug("deleting materials in orchestrator") @@ -1032,7 +994,7 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b } // FIXME: not thread safe -func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int32, teamId int, tx *pg.Tx) (*app2.App, error) { +func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int32, teamId int, isJob bool, tx *pg.Tx) (*app2.App, error) { app, err := impl.appRepository.FindActiveByName(name) if err != nil && err != pg.ErrNoRows { return nil, err @@ -1046,10 +1008,17 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 } return nil, err } + var appStore int + if isJob { + appStore = 2 + } else { + appStore = 0 + } pg := &app2.App{ Active: true, AppName: name, TeamId: teamId, + AppStore: appStore, AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } err = impl.appRepository.SaveWithTxn(pg, tx) @@ -1083,59 +1052,6 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 return pg, nil } -func (impl CiCdPipelineOrchestratorImpl) createJobGroup(name string, userId int32, teamId int, description string, tx *pg.Tx) (*app2.App, error) { - app, err := impl.appRepository.FindActiveByName(name) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if app != nil && app.Id > 0 { - impl.logger.Warnw("job already exists", "name", name) - err = &util.ApiError{ - Code: constants.AppAlreadyExists.Code, - InternalMessage: "job already exists", - UserMessage: constants.AppAlreadyExists.UserMessage(name), - } - return nil, err - } - pg := &app2.App{ - Active: true, - AppName: name, - TeamId: teamId, - AppStore: 2, - Description: description, - AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, - } - err = impl.appRepository.SaveWithTxn(pg, tx) - if err != nil { - impl.logger.Errorw("error in saving entity ", "entity", pg) - return nil, err - } - - apps, err := impl.appRepository.FindActiveListByName(name) - if err != nil { - return nil, err - } - appLen := len(apps) - if appLen > 1 { - firstElement := apps[0] - if firstElement.Id != pg.Id { - pg.Active = false - err = impl.appRepository.UpdateWithTxn(pg, tx) - if err != nil { - impl.logger.Errorw("error in saving entity ", "entity", pg) - return nil, err - } - err = &util.ApiError{ - Code: constants.AppAlreadyExists.Code, - InternalMessage: "job already exists", - UserMessage: constants.AppAlreadyExists.UserMessage(name), - } - return nil, err - } - } - return pg, nil -} - func (impl CiCdPipelineOrchestratorImpl) validateCheckoutPathsForMultiGit(allPaths map[int]string) error { dockerfilePathMap := make(map[string]bool) impl.logger.Debugw("all paths ", "path", allPaths) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index f918453099..2cabb6a9bc 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -86,7 +86,6 @@ func GetDeploymentServiceTypeConfig() (*DeploymentServiceTypeConfig, error) { type PipelineBuilder interface { CreateCiPipeline(createRequest *bean.CiConfigRequest) (*bean.PipelineCreateResponse, error) CreateApp(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) - CreateJob(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) CreateMaterialsForApp(request *bean.CreateMaterialDTO) (*bean.CreateMaterialDTO, error) UpdateMaterialsForApp(request *bean.UpdateMaterialDTO) (*bean.UpdateMaterialDTO, error) DeleteMaterial(request *bean.UpdateMaterialDTO) error @@ -310,15 +309,6 @@ func (impl PipelineBuilderImpl) CreateApp(request *bean.CreateAppDTO) (*bean.Cre return res, err } -func (impl PipelineBuilderImpl) CreateJob(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { - impl.logger.Debugw("job create request received", "req", request) - res, err := impl.ciCdPipelineOrchestrator.CreateJob(request) - if err != nil { - impl.logger.Errorw("error in saving create job req", "req", request, "err", err) - } - return res, err -} - func (impl PipelineBuilderImpl) DeleteApp(appId int, userId int32) error { impl.logger.Debugw("app delete request received", "app", appId) err := impl.ciCdPipelineOrchestrator.DeleteApp(appId, userId) From 1121d6cf5fe7554599d3c6c607a95614262c170b Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sun, 19 Feb 2023 23:01:46 +0530 Subject: [PATCH 028/118] Corrected the code of clone job. --- pkg/appClone/AppCloneService.go | 11 ++++++++--- pkg/pipeline/PipelineBuilder.go | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 1443aa48fa..60076e9546 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -86,6 +86,7 @@ type CloneRequest struct { Name string `json:"name"` ProjectId int `json:"projectId"` AppLabels []*bean.Label `json:"labels,omitempty" validate:"dive"` + isJob bool `json:"isJob"` } func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { @@ -109,6 +110,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context Name: createReq.AppName, ProjectId: createReq.TeamId, AppLabels: createReq.AppLabels, + isJob: createReq.IsJob, } userId := createReq.UserId appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) @@ -133,9 +135,11 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context } //TODO check stage of current app - if !refAppStatus["APP"] { - impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) - return nil, nil + if !createReq.IsJob { + if !refAppStatus["APP"] { + impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) + return nil, nil + } } app, err := impl.CreateApp(cloneReq, userId) if err != nil { @@ -304,6 +308,7 @@ func (impl *AppCloneServiceImpl) CreateApp(cloneReq *CloneRequest, userId int32) UserId: userId, TeamId: cloneReq.ProjectId, AppLabels: cloneReq.AppLabels, + IsJob: cloneReq.isJob, } createRes, err := impl.pipelineBuilder.CreateApp(createAppReq) return createRes, err diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 2cabb6a9bc..cf6868b4e4 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -378,13 +378,14 @@ func (impl PipelineBuilderImpl) GetApp(appId int) (application *bean.CreateAppDT impl.logger.Errorw("error in fetching app", "id", appId, "err", err) return nil, err } - gitMaterials := impl.GetMaterialsForAppId(appId) + gitMaterials := impl.GetMaterialsForAppId(appId) application = &bean.CreateAppDTO{ Id: app.Id, AppName: app.AppName, Material: gitMaterials, TeamId: app.TeamId, + IsJob: app.AppStore == 2, } return application, nil } From 843d7d2006d695907f7943da6e6ce89e2b32516a Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 20 Feb 2023 19:08:45 +0530 Subject: [PATCH 029/118] Corrected the code of clone job. --- api/bean/AppView.go | 7 +- .../sql/repository/AppListingRepository.go | 34 +++-- internal/sql/repository/app/AppRepository.go | 4 +- .../AppListingRepositoryQueryBuilder.go | 25 +++- pkg/app/AppListingService.go | 36 +++-- pkg/pipeline/CiCdPipelineOrchestrator.go | 24 ++-- pkg/pipeline/PipelineBuilder.go | 2 + specs/jobs.yaml | 125 ++++++++++++++++++ util/version.go | 2 +- 9 files changed, 225 insertions(+), 34 deletions(-) create mode 100644 specs/jobs.yaml diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 0a7bbff644..5e3b9ecac7 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -80,7 +80,12 @@ type JobListingContainer struct { CiPipelineName string `json:"ci_pipeline_name"` Status string `json:"status"` StartedOn time.Time `json:"started_on"` - LastSuccessAt time.Time `json:"-"` + //LastSuccessAt time.Time `json:"-"` +} + +type CiPipelineLastSucceededTime struct { + CiPipelineID int `json:"ci_pipeline_id"` + LastSuccessOn time.Time `json:"last_succeeded_on"` } type AppEnvironmentContainer struct { diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index ad6ae3d418..af95333938 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -24,7 +24,6 @@ import ( "context" "encoding/json" "fmt" - "go.opentelemetry.io/otel" "strconv" "strings" @@ -38,7 +37,8 @@ import ( type AppListingRepository interface { FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) - FetchJobs(appIds []int) ([]*bean.JobListingContainer, error) + FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) + FetchJobsLastSucceededOn(ciPipelineIDs []int) ([]*bean.CiPipelineLastSucceededTime, error) DeploymentDetailsByAppIdAndEnvId(ctx context.Context, appId int, envId int) (bean.DeploymentDetailContainer, error) FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -82,22 +82,36 @@ func NewAppListingRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB, return &AppListingRepositoryImpl{dbConnection: dbConnection, Logger: Logger, appListingRepositoryQueryBuilder: appListingRepositoryQueryBuilder} } -/* -* -It will return the list of filtered apps with details related to each env -*/ -func (impl AppListingRepositoryImpl) FetchJobs(appIds []int) ([]*bean.JobListingContainer, error) { +func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) { var jobContainers []*bean.JobListingContainer - jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery() - appIdsString := helper.GetCommaSepratedString(appIds) + //if len(appIds) == 0 { + // return jobContainers, nil + //} + jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(appIds, statuses) + impl.Logger.Debugw("basic app detail query: ", jobsQuery) - _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, appIdsString) + _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, pg.In(appIds)) if appsErr != nil { impl.Logger.Error(appsErr) return jobContainers, appsErr } return jobContainers, nil } +func (impl AppListingRepositoryImpl) FetchJobsLastSucceededOn(CiPipelineIDs []int) ([]*bean.CiPipelineLastSucceededTime, error) { + var lastSucceededTimeArray []*bean.CiPipelineLastSucceededTime + if len(CiPipelineIDs) == 0 { + return lastSucceededTimeArray, nil + } + jobsLastFinishedOnQuery := impl.appListingRepositoryQueryBuilder.JobsLastSucceededOnTimeQuery(CiPipelineIDs) + impl.Logger.Debugw("basic app detail query: ", jobsLastFinishedOnQuery) + _, appsErr := impl.dbConnection.Query(&lastSucceededTimeArray, jobsLastFinishedOnQuery) + impl.Logger.Debugw("basic app detail query: ", jobsLastFinishedOnQuery) + if appsErr != nil { + impl.Logger.Error(appsErr) + return lastSucceededTimeArray, appsErr + } + return lastSucceededTimeArray, nil +} func (impl AppListingRepositoryImpl) FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") var appEnvArr []*bean.AppEnvironmentContainer diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 613a40ee53..71bfd854c4 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -30,11 +30,12 @@ type App struct { tableName struct{} `sql:"app" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` AppName string `sql:"app_name,notnull"` //same as app name + DisplayName string `sql:"display_name"` Active bool `sql:"active, notnull"` TeamId int `sql:"team_id"` AppStore int `sql:"app_store, notnull"` AppOfferingMode string `sql:"app_offering_mode,notnull"` - Description string `sql:"-"` + Description string `sql:"description"` Team team.Team sql.AuditLog } @@ -354,6 +355,7 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL if len(jobListingFilter.Teams) > 0 { whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" } + query := "select id " + "from app " + whereCondition + orderByCondition appName := "%" + jobListingFilter.AppNameSearch + "%" _, err := repo.dbConnection.Query(&appIds, query, appName, jobListingFilter.Size, jobListingFilter.Offset) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 23298db13a..7a09d84e60 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -60,14 +60,33 @@ const ( AppNameSortBy SortBy = "appNameSort" ) -func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery() string { - query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as app_id,app.app_name,cwr.started_on,cwr.status " + +func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, statuses []string) string { + query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as app_id,app.display_name as app_name,cwr.started_on,cwr.status " + "from ci_pipeline left join " + "(select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw " + "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + "on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr" + " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + - "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 and app.id in (?)" + "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 " + if len(appIDs) > 0 { + query += "and app_id IN (" + GetCommaSepratedString(appIDs) + ") " + } + if len(statuses) > 0 { + query += "and cwr.status IN (" + util.ProcessAppStatuses(statuses) + ") " + } + return query +} + +// use this query with atleast 1 cipipeline id +func (impl AppListingRepositoryQueryBuilder) JobsLastSucceededOnTimeQuery(ciPipelineIDs []int) string { + // use this query with atleast 1 cipipeline id + query := "select cw.ci_pipeline_id,cw.finished_on " + + "as last_succeeded_on from ci_workflow cw inner join " + + "(SELECT ci_pipeline_id, MAX(finished_on) finished_on " + + "FROM ci_workflow WHERE ci_workflow.status = 'Succeeded'" + + "GROUP BY ci_pipeline_id) cws on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.finished_on = cws.finished_on " + + "where cw.ci_pipeline_id IN (" + GetCommaSepratedString(ciPipelineIDs) + "); " + return query } diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 18b064d56b..95337c7040 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -55,7 +55,6 @@ type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) - BuildJobListingResponse(jobContainers []*bean.JobListingContainer) []*bean.JobContainer FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) FetchAppDetails(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -238,12 +237,14 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi impl.Logger.Errorw("error in fetching app ids list", "error", err) return []*bean.JobContainer{}, err } - jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds) + jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds, jobListingFilter.AppStatuses) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) return []*bean.JobContainer{}, err } - jobContainers := impl.BuildJobListingResponse(jobListingContainers) + CiPipelineIDs := GetCIPipelineIDs(jobListingContainers) + JobsLastSucceededOnTime, err := impl.appListingRepository.FetchJobsLastSucceededOn(CiPipelineIDs) + jobContainers := BuildJobListingResponse(jobListingContainers, JobsLastSucceededOnTime) return jobContainers, nil } func (impl AppListingServiceImpl) FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) { @@ -362,9 +363,23 @@ func (impl AppListingServiceImpl) BuildAppListingResponse(fetchAppListingRequest appContainerResponses, err := impl.appListingViewBuilder.BuildView(fetchAppListingRequest, appEnvMapping) return appContainerResponses, err } -func (impl AppListingServiceImpl) BuildJobListingResponse(jobContainers []*bean.JobListingContainer) []*bean.JobContainer { +func GetCIPipelineIDs(jobContainers []*bean.JobListingContainer) []int { + + var ciPipelineIDs []int + for _, jobContainer := range jobContainers { + ciPipelineIDs = append(ciPipelineIDs, jobContainer.CiPipelineID) + } + return ciPipelineIDs +} +func BuildJobListingResponse(jobContainers []*bean.JobListingContainer, JobsLastSucceededOnTime []*bean.CiPipelineLastSucceededTime) []*bean.JobContainer { jobContainersMapping := make(map[int]bean.JobContainer) var appIds []int + + lastSucceededTimeMapping := make(map[int]time.Time) + for _, lastSuccessTime := range JobsLastSucceededOnTime { + lastSucceededTimeMapping[lastSuccessTime.CiPipelineID] = lastSuccessTime.LastSuccessOn + } + //Storing the sequence in appIds array for _, jobContainer := range jobContainers { val, ok := jobContainersMapping[jobContainer.AppId] @@ -379,12 +394,17 @@ func (impl AppListingServiceImpl) BuildJobListingResponse(jobContainers []*bean. if len(val.JobCiPipelines) == 0 { val.JobCiPipelines = make([]bean.JobCIPipeline, 0) } + ciPipelineObj := bean.JobCIPipeline{ - CiPipelineId: jobContainer.CiPipelineID, - Status: jobContainer.Status, - LastRunAt: jobContainer.StartedOn, - LastSuccessAt: jobContainer.LastSuccessAt, + CiPipelineId: jobContainer.CiPipelineID, + Status: jobContainer.Status, + LastRunAt: jobContainer.StartedOn, + //LastSuccessAt: jobContainer.LastSuccessAt, } + if lastSuccessAt, ok := lastSucceededTimeMapping[jobContainer.CiPipelineID]; ok { + ciPipelineObj.LastSuccessAt = lastSuccessAt + } + val.JobCiPipelines = append(val.JobCiPipelines, ciPipelineObj) jobContainersMapping[jobContainer.AppId] = val diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index d7b31547c4..9690980624 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -812,7 +812,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp } // Rollback tx on error. defer tx.Rollback() - app, err := impl.createAppGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.IsJob, tx) + app, err := impl.createAppGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.IsJob, createRequest.Description, tx) if err != nil { return nil, err } @@ -839,6 +839,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp return nil, err } createRequest.Id = app.Id + createRequest.AppName = app.DisplayName return createRequest, nil } @@ -994,7 +995,7 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b } // FIXME: not thread safe -func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int32, teamId int, isJob bool, tx *pg.Tx) (*app2.App, error) { +func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int32, teamId int, isJob bool, description string, tx *pg.Tx) (*app2.App, error) { app, err := impl.appRepository.FindActiveByName(name) if err != nil && err != pg.ErrNoRows { return nil, err @@ -1008,18 +1009,21 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 } return nil, err } - var appStore int + appStore := 0 + displayName := name + appName := name if isJob { appStore = 2 - } else { - appStore = 0 + appName = name + "/" + util2.Generate(8) + "J" } pg := &app2.App{ - Active: true, - AppName: name, - TeamId: teamId, - AppStore: appStore, - AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, + Active: true, + AppName: appName, + DisplayName: displayName, + TeamId: teamId, + AppStore: appStore, + Description: description, + AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } err = impl.appRepository.SaveWithTxn(pg, tx) if err != nil { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index cf6868b4e4..f9ba3e78eb 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -302,6 +302,7 @@ func formatDate(t time.Time, layout string) string { func (impl PipelineBuilderImpl) CreateApp(request *bean.CreateAppDTO) (*bean.CreateAppDTO, error) { impl.logger.Debugw("app create request received", "req", request) + res, err := impl.ciCdPipelineOrchestrator.CreateApp(request) if err != nil { impl.logger.Errorw("error in saving create app req", "req", request, "err", err) @@ -430,6 +431,7 @@ func (impl PipelineBuilderImpl) GetMaterialsForAppId(appId int) []*bean.GitMater */ func (impl PipelineBuilderImpl) getDefaultArtifactStore(id string) (store *dockerRegistryRepository.DockerArtifactStore, err error) { + if id == "" { impl.logger.Debugw("docker repo is empty adding default repo") store, err = impl.dockerArtifactStoreRepository.FindActiveDefaultStore() diff --git a/specs/jobs.yaml b/specs/jobs.yaml new file mode 100644 index 0000000000..21d41d67d8 --- /dev/null +++ b/specs/jobs.yaml @@ -0,0 +1,125 @@ +openapi: "3.0.0" +info: + title: Jobs + version: "1.0" +paths: + /orchestrator/app/jobs: + post: + description: Create and clones a job + requestBody: + required: true + content: + application/json: + schema: + items: + $ref: "#/components/schemas/CreateJob" + responses: + "200 OK": + description: Used to give response once a job is created + content: + application/json: + schema: + $ref: "#/components/schemas/ActionResponse" + /orchestrator/app/jobs/list: + get: + description: Get the list of all the jobs by applying filter + requestBody: + required: true + content: + application/json: + schema: + items: + $ref: "#/components/schemas/JobList" + +#components +components: + schemas: + CreateJob: + type: object + properties: + appName: + type: string + description: Used to give the name of the job + example: "my-job-1" + teamId: + type: integer + description: Used to give team id + example: 1 + templateId: + type: integer + description: Used to give the id of the job it wants to clone + example: 18 + labels: + type: []label + label: + key: + type: string + example: "hello" + value: + type: string + example: "world" + propogate: + type: bool + example: false + description: + type: string + description: Used to give the description of the job once it is made. + example: "This is my first Job" + isJob: + type: boolean + description: Used to state whether it is an app or a job + example: true + ActionResponse: + name: result + type: object + properties: + id: + type: integer + description: Used to give the id of job once its created + example: 25 + appName: + type: string + description: Used to give the name of job once its created + example: "my-job-1" + material: + type: []gitMaterial + gitMaterial: + name: + type: string + url: + type: string + id: + type: integer + gitProviderId: + type: integer + checkoutPath: + type: string + fetchSubmodules: + type: bool + isUsedInCiConfig: + type: bool + teamId: + type: integer + description: Used to give the team id + example: 1 + templateId: + type: integer + description: Used to give the templateId + example: 0 + description: + type: string + description: Used to give the description of the job once it is made. + example: "This is my first Job" + isJob: + type: boolean + description: used to tell whether it is a job or an app + example: true + JobList: + type: object + properties: + + + + + + diff --git a/util/version.go b/util/version.go index f64cb32956..417b3fb010 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: ServerMode} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE_FULL} } func IsBaseStack() bool { From 70a90b22dc7feb6682e5a777c808267789c27c12 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 21 Feb 2023 11:28:54 +0530 Subject: [PATCH 030/118] Updated the api specs --- specs/jobs.yaml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 21d41d67d8..2e335897b9 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -117,7 +117,30 @@ components: JobList: type: object properties: - + teams: + type: [] integer + description: used to give the project id + example: [1,2] + appStatuses: + type: [] string + description: used to give the filter of app ci-build status + example: ["Succeeded", "Starting"] + sortBy: + type: string + description: used to give the sort by constraint + example: "appNameSort" + sortOrder: + type: string + description: used to give the sort order + example: "ASC" + offset: + type: integer + description: used to give the number from which we want our job (if the offset is 20 means we want list of jobs from 20) + example: 0 + size: + type: integer + description: used to give the number of jobs we want + example: 20 From ddbe1976f0cb6b744aa350bd65d147a0881964b2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 21 Feb 2023 11:47:14 +0530 Subject: [PATCH 031/118] Updated the api specs --- specs/jobs.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 2e335897b9..52ae664b69 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -30,6 +30,13 @@ paths: schema: items: $ref: "#/components/schemas/JobList" + response: + description: Used to give response of list of jobs + content: + application/json: + schema: + items: + $ref: "#/components/schemas/JobListResponse" #components components: @@ -142,6 +149,34 @@ components: description: used to give the number of jobs we want example: 20 + JobListResponse: + type: object + properties: + jobContainers: + type: [] jobContainer + jobContainer: + properties: + appId: + type: integer + appName: + type: string + description: + type: string + ciPipelines: + type: []jobCiPipeline + jobCiPipeline: + properties: + ciPipelineId: + type: integer + status: + type: string + lastRunAt: + type: time.Time + lastSuccessAt: + type: time.Time + jobCount: + type: integer + From 4ad51e327d075cf244963db1194974887e683a4f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 21 Feb 2023 16:18:24 +0530 Subject: [PATCH 032/118] Updated the api specs --- api/bean/AppView.go | 8 +- .../app/BuildPipelineRestHandler.go | 3 + go.mod | 2 +- go.sum | 16 - .../sql/repository/AppListingRepository.go | 2 +- internal/sql/repository/app/AppRepository.go | 18 +- .../AppListingRepositoryQueryBuilder.go | 4 +- pkg/app/AppCrudOperationService.go | 8 +- pkg/appClone/AppCloneService.go | 33 +- pkg/bean/app.go | 2 + pkg/pipeline/CiCdPipelineOrchestrator.go | 20 +- pkg/pipeline/CiService.go | 48 +- pkg/pipeline/PipelineBuilder.go | 85 +- .../argoproj/argo-cd/assets/badge.svg | 22 - .../argo-cd/assets/builtin-policy.csv | 34 - .../argoproj/argo-cd/assets/model.conf | 14 - .../argoproj/argo-cd/assets/swagger.json | 3887 ----------------- 17 files changed, 153 insertions(+), 4053 deletions(-) delete mode 100644 vendor/github.com/argoproj/argo-cd/assets/badge.svg delete mode 100644 vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv delete mode 100644 vendor/github.com/argoproj/argo-cd/assets/model.conf delete mode 100644 vendor/github.com/argoproj/argo-cd/assets/swagger.json diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 5e3b9ecac7..819f071dae 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -59,10 +59,10 @@ type CiMaterialDTO struct { } type JobContainer struct { - AppId int `json:"app_id"` - AppName string `json:"app_name""` + AppId int `json:"appId"` + AppName string `json:"appName""` Description string `json:"description"` - JobCiPipelines []JobCIPipeline `json:"ci_pipelines"'` + JobCiPipelines []JobCIPipeline `json:"ciPipelines"'` } type JobCIPipeline struct { @@ -75,7 +75,7 @@ type JobCIPipeline struct { type JobListingContainer struct { AppId int `json:"app_id"` AppName string `json:"app_name"` - Description string `json:"-"` + Description string `json:"description"` CiPipelineID int `json:"ci_pipeline_id"` CiPipelineName string `json:"ci_pipeline_name"` Status string `json:"status"` diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 7ba58be226..a0d5e95f5c 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -243,8 +243,10 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri return } + var emptyDockerRegistry string if patchRequest.IsJob { ciConfigRequest := bean.CiConfigRequest{} + ciConfigRequest.DockerRegistry = emptyDockerRegistry ciConfigRequest.AppId = patchRequest.AppId ciConfigRequest.CiBuildConfig = &bean1.CiBuildConfigBean{} ciConfigRequest.CiBuildConfig.CiBuildType = "skip-build" @@ -255,6 +257,7 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri return } ciConfigRequest.CiBuildConfig.GitMaterialId = patchRequest.CiPipeline.CiMaterial[0].GitMaterialId + ciConfigRequest.IsJob = true _, err = handler.pipelineBuilder.CreateCiPipeline(&ciConfigRequest) if err != nil { handler.Logger.Errorw("error occurred in creating ci-pipeline for the Job", "payload", ciConfigRequest, "err", err) diff --git a/go.mod b/go.mod index ebff02a186..5d9557cfc1 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,6 @@ require ( github.com/ktrysmt/go-bitbucket v0.9.40 github.com/lib/pq v1.10.4 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 - github.com/nats-io/nats.go v1.19.0 github.com/otiai10/copy v1.0.2 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 @@ -190,6 +189,7 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a // indirect + github.com/nats-io/nats.go v1.19.0 // indirect github.com/nats-io/nkeys v0.3.0 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect diff --git a/go.sum b/go.sum index 569aa81969..801dbdd485 100644 --- a/go.sum +++ b/go.sum @@ -282,20 +282,6 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53 h1:oHDpsCsuYiL+u5TEhilKNLGhH9lwRNkIqrzhHQA6d3c= github.com/devtron-labs/authenticator v0.4.31-0.20221213131053-6e4668309f53/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.0-20221028123910-ac349a64526a h1:g2S75h+OAXXN5SGsdxxCfmKZXJFGUKTNtPaUW5Gp9ck= -github.com/devtron-labs/common-lib v0.0.0-20221028123910-ac349a64526a/go.mod h1:wDldcLG0CzfnxCBEXeitEkYF9bk62yG+HLGqQg6VZ4o= -github.com/devtron-labs/common-lib v0.0.0-20221111092855-6dae5a34a041 h1:4yQ7v/fC39TzKzcn24CHzB8XL2wl3JFhsJ1VsXiTjXQ= -github.com/devtron-labs/common-lib v0.0.0-20221111092855-6dae5a34a041/go.mod h1:PWHsHmhRVy50rvYlTv2bfoW/4oX8lPTcITiF8l9+ULg= -github.com/devtron-labs/common-lib v0.0.0-20221111102018-8cffc22d80e3 h1:wswsi1r6tDtmycUaUGGUtoUNbKvQyJByKyxiDm+o9Lw= -github.com/devtron-labs/common-lib v0.0.0-20221111102018-8cffc22d80e3/go.mod h1:PWHsHmhRVy50rvYlTv2bfoW/4oX8lPTcITiF8l9+ULg= -github.com/devtron-labs/common-lib v0.0.0-20221117072858-bd2f19e16dd2 h1:dEtjnQqSWGCl4sIWYvsBsJuBLfmVV2kw1OyuBpkAp64= -github.com/devtron-labs/common-lib v0.0.0-20221117072858-bd2f19e16dd2/go.mod h1:PWHsHmhRVy50rvYlTv2bfoW/4oX8lPTcITiF8l9+ULg= -github.com/devtron-labs/common-lib v0.0.0-20221126093411-794588b004ea h1:cJWdGd6bXNGqt7yWjsIhV3mHcnhe2T3s3e7hKOgK2iU= -github.com/devtron-labs/common-lib v0.0.0-20221126093411-794588b004ea/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20221130083108-d35a1c63675b h1:Z9tehff98lhxdX9Z7ggdBuFzrCNacr4hIjdNCRubVt8= -github.com/devtron-labs/common-lib v0.0.0-20221130083108-d35a1c63675b/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= -github.com/devtron-labs/common-lib v0.0.0-20221208044859-269008fed09b h1:yGXerrqwyjPIr62q1RadJh6MYH3vG6T9nChH+yOJnnA= -github.com/devtron-labs/common-lib v0.0.0-20221208044859-269008fed09b/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= github.com/devtron-labs/common-lib v0.0.0-20230109070754-ff4dca200a2c h1:jz7yPUlIJXFg9AvJh2fb0QW7JT6+RKj8LOl1mWM/HQA= github.com/devtron-labs/common-lib v0.0.0-20230109070754-ff4dca200a2c/go.mod h1:R24nOqgk4buk9zv+BXzORfObZsOe3NE9P55KrZXGX9k= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -883,8 +869,6 @@ github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hl github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= github.com/nats-io/nats-server/v2 v2.8.4 h1:0jQzze1T9mECg8YZEl8+WYUXb9JKluJfCBriPUtluB4= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nats.go v1.15.0 h1:3IXNBolWrwIUf2soxh6Rla8gPzYWEZQBUBK6RV21s+o= -github.com/nats-io/nats.go v1.15.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nats.go v1.19.0 h1:H6j8aBnTQFoVrTGB6Xjd903UMdE7jz6DS4YkmAqgZ9Q= github.com/nats-io/nats.go v1.19.0/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index af95333938..ddd2589ed8 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -90,7 +90,7 @@ func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string) jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(appIds, statuses) impl.Logger.Debugw("basic app detail query: ", jobsQuery) - _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, pg.In(appIds)) + _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery) if appsErr != nil { impl.Logger.Error(appsErr) return jobContainers, appsErr diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 71bfd854c4..083376ea01 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -46,6 +46,7 @@ type AppRepository interface { Update(app *App) error UpdateWithTxn(app *App, tx *pg.Tx) error FindActiveByName(appName string) (pipelineGroup *App, err error) + FindJobByName(appName string) (pipelineGroup *App, err error) FindActiveListByName(appName string) ([]*App, error) FindById(id int) (pipelineGroup *App, err error) FindAppsByTeamId(teamId int) ([]*App, error) @@ -121,6 +122,17 @@ func (repo AppRepositoryImpl) FindActiveByName(appName string) (*App, error) { // there is only single active app will be present in db with a same name. return pipelineGroup, err } +func (repo AppRepositoryImpl) FindJobByName(appName string) (*App, error) { + pipelineGroup := &App{} + err := repo.dbConnection. + Model(pipelineGroup). + Where("display_name = ?", appName). + Where("active = ?", true). + Order("id DESC").Limit(1). + Select() + // there is only single active app will be present in db with a same name. + return pipelineGroup, err +} func (repo AppRepositoryImpl) FindActiveListByName(appName string) ([]*App, error) { var apps []*App @@ -259,7 +271,7 @@ func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string, isJob bool if isJob { appStore = 2 } - err := repo.dbConnection.Model(&apps).Where("app_name ILIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", appStore).Select() + err := repo.dbConnection.Model(&apps).Where("display_name LIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", appStore).Select() return apps, err } @@ -346,12 +358,12 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL Id int `json:"id"` } var appIds []AppId - orderByCondition := " order by app_name " + orderByCondition := " order by display_name " if jobListingFilter.SortOrder == "DESC" { orderByCondition += string(jobListingFilter.SortOrder) } orderByCondition += " limit ? offset ? " - whereCondition := " where app_name like ? and active = true and app_store = 2 " + whereCondition := " where display_name like ? and active = true and app_store = 2 " if len(jobListingFilter.Teams) > 0 { whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" } diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 7a09d84e60..94fbfb4b8d 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -61,7 +61,7 @@ const ( ) func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, statuses []string) string { - query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as app_id,app.display_name as app_name,cwr.started_on,cwr.status " + + query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as app_id,app.display_name as app_name, app.description ,cwr.started_on,cwr.status " + "from ci_pipeline left join " + "(select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw " + "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + @@ -69,7 +69,7 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 " if len(appIDs) > 0 { - query += "and app_id IN (" + GetCommaSepratedString(appIDs) + ") " + query += "and app.id IN (" + GetCommaSepratedString(appIDs) + ") " } if len(statuses) > 0 { query += "and cwr.status IN (" + util.ProcessAppStatuses(statuses) + ") " diff --git a/pkg/app/AppCrudOperationService.go b/pkg/app/AppCrudOperationService.go index e07aa4bdee..c7da823f73 100644 --- a/pkg/app/AppCrudOperationService.go +++ b/pkg/app/AppCrudOperationService.go @@ -104,6 +104,7 @@ func (impl AppCrudOperationServiceImpl) UpdateApp(request *bean.CreateAppDTO) (* impl.logger.Errorw("error in fetching app", "error", err) return nil, err } + app.Description = request.Description app.TeamId = request.TeamId app.UpdatedOn = time.Now() app.UpdatedBy = request.UserId @@ -313,15 +314,20 @@ func (impl AppCrudOperationServiceImpl) GetAppMetaInfo(appId int) (*bean.AppMeta userEmailId = fmt.Sprintf("%s (inactive)", user.EmailId) } } + appName := app.AppName + if app.AppStore == 2 { + appName = app.DisplayName + } info := &bean.AppMetaInfoDto{ AppId: app.Id, - AppName: app.AppName, + AppName: appName, ProjectId: app.TeamId, ProjectName: app.Team.Name, CreatedBy: userEmailId, CreatedOn: app.CreatedOn, Labels: labels, Active: app.Active, + Description: app.Description, } return info, nil } diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 60076e9546..5f923f11d7 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -82,11 +82,12 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, } type CloneRequest struct { - RefAppId int `json:"refAppId"` - Name string `json:"name"` - ProjectId int `json:"projectId"` - AppLabels []*bean.Label `json:"labels,omitempty" validate:"dive"` - isJob bool `json:"isJob"` + RefAppId int `json:"refAppId"` + Name string `json:"name"` + ProjectId int `json:"projectId"` + AppLabels []*bean.Label `json:"labels,omitempty" validate:"dive"` + Description string `json:"description"` + isJob bool `json:"isJob"` } func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { @@ -106,11 +107,12 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context } //create new app cloneReq := &CloneRequest{ - RefAppId: createReq.TemplateId, - Name: createReq.AppName, - ProjectId: createReq.TeamId, - AppLabels: createReq.AppLabels, - isJob: createReq.IsJob, + RefAppId: createReq.TemplateId, + Name: createReq.AppName, + ProjectId: createReq.TeamId, + AppLabels: createReq.AppLabels, + isJob: createReq.IsJob, + Description: createReq.Description, } userId := createReq.UserId appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) @@ -304,11 +306,12 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context func (impl *AppCloneServiceImpl) CreateApp(cloneReq *CloneRequest, userId int32) (*bean.CreateAppDTO, error) { createAppReq := &bean.CreateAppDTO{ - AppName: cloneReq.Name, - UserId: userId, - TeamId: cloneReq.ProjectId, - AppLabels: cloneReq.AppLabels, - IsJob: cloneReq.isJob, + AppName: cloneReq.Name, + UserId: userId, + TeamId: cloneReq.ProjectId, + AppLabels: cloneReq.AppLabels, + IsJob: cloneReq.isJob, + Description: cloneReq.Description, } createRes, err := impl.pipelineBuilder.CreateApp(createAppReq) return createRes, err diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 1ec6b31b1b..d68e092557 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -291,6 +291,7 @@ type CiConfigRequest struct { CreatedBy int32 `sql:"created_by,type:integer"` UpdatedOn time.Time `sql:"updated_on,type:timestamptz"` UpdatedBy int32 `sql:"updated_by,type:integer"` + IsJob bool `json:"-"` } type TestExecutorImageProperties struct { @@ -607,6 +608,7 @@ type AppMetaInfoDto struct { CreatedOn time.Time `json:"createdOn"` Active bool `json:"active,notnull"` Labels []*Label `json:"labels"` + Description string `json:"description"` UserId int32 `json:"-"` } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 9690980624..a544924c19 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -839,7 +839,9 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp return nil, err } createRequest.Id = app.Id - createRequest.AppName = app.DisplayName + if createRequest.IsJob { + createRequest.AppName = app.DisplayName + } return createRequest, nil } @@ -1009,6 +1011,22 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 } return nil, err } + if isJob { + job, err := impl.appRepository.FindJobByName(name) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if job != nil && job.Id > 0 { + impl.logger.Warnw("job already exists", "name", name) + err = &util.ApiError{ + Code: constants.AppAlreadyExists.Code, + InternalMessage: "job already exists", + UserMessage: constants.AppAlreadyExists.UserMessage(name), + } + return nil, err + } + } + appStore := 0 displayName := name appName := name diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 8e796ae39e..f54e79eee7 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -443,22 +443,22 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } workflowRequest := &WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - DockerRegistryId: dockerRegistry.Id, - DockerRegistryType: string(dockerRegistry.RegistryType), - DockerImageTag: dockerImageTag, - DockerRegistryURL: dockerRegistry.RegistryURL, - DockerRepository: dockerRepository, - CheckoutPath: checkoutPath, - DockerUsername: dockerRegistry.Username, - DockerPassword: dockerRegistry.Password, - AwsRegion: dockerRegistry.AWSRegion, - AccessKey: dockerRegistry.AWSAccessKeyId, - SecretKey: dockerRegistry.AWSSecretAccessKey, - DockerConnection: dockerRegistry.Connection, - DockerCert: dockerRegistry.Cert, + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + //DockerRegistryId: dockerRegistry.Id, + //DockerRegistryType: string(dockerRegistry.RegistryType), + //DockerImageTag: dockerImageTag, + //DockerRegistryURL: dockerRegistry.RegistryURL, + //DockerRepository: dockerRepository, + //CheckoutPath: checkoutPath, + //DockerUsername: dockerRegistry.Username, + //DockerPassword: dockerRegistry.Password, + //AwsRegion: dockerRegistry.AWSRegion, + //AccessKey: dockerRegistry.AWSAccessKeyId, + //SecretKey: dockerRegistry.AWSSecretAccessKey, + //DockerConnection: dockerRegistry.Connection, + //DockerCert: dockerRegistry.Cert, CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", CiProjectDetails: ciProjectDetails, Namespace: ciWorkflowConfig.Namespace, @@ -483,7 +483,23 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. IgnoreDockerCachePull: impl.ciConfig.IgnoreDockerCacheForCI, CacheInvalidate: trigger.InvalidateCache, } + if dockerRegistry != nil { + workflowRequest.DockerRegistryId = dockerRegistry.Id + workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) + workflowRequest.DockerImageTag = dockerImageTag + workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL + workflowRequest.DockerRepository = dockerRepository + workflowRequest.CheckoutPath = checkoutPath + workflowRequest.DockerUsername = dockerRegistry.Username + workflowRequest.DockerPassword = dockerRegistry.Password + workflowRequest.AwsRegion = dockerRegistry.AWSRegion + workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId + workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey + workflowRequest.DockerConnection = dockerRegistry.Connection + workflowRequest.DockerCert = dockerRegistry.Cert + + } if ciWorkflowConfig.LogsBucket == "" { ciWorkflowConfig.LogsBucket = impl.ciConfig.DefaultBuildLogsBucket } diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index f9ba3e78eb..41bb6a4e06 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -481,17 +481,21 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea // impl.logger.Debugw("error in json unmarshal", "app", appId, "err", err) // return nil, err //} - regHost, err := template.DockerRegistry.GetRegistryLocation() - if err != nil { - impl.logger.Errorw("invalid reg url", "err", err) - return nil, err + var regHost string + dockerRegistry := template.DockerRegistry + if dockerRegistry != nil { + regHost, err = dockerRegistry.GetRegistryLocation() + if err != nil { + impl.logger.Errorw("invalid reg url", "err", err) + return nil, err + } } ciConfig = &bean.CiConfigRequest{ - Id: template.Id, - AppId: template.AppId, - AppName: template.App.AppName, - DockerRepository: template.DockerRepository, - DockerRegistry: template.DockerRegistry.Id, + Id: template.Id, + AppId: template.AppId, + AppName: template.App.AppName, + DockerRepository: template.DockerRepository, + //DockerRegistry: dockerRegistry.Id, DockerRegistryUrl: regHost, CiBuildConfig: ciTemplateBean.CiBuildConfig, //DockerBuildConfig: &bean.DockerBuildConfig{DockerfilePath: template.DockerfilePath, Args: dockerArgs, GitMaterialId: template.GitMaterialId, TargetPlatform: template.TargetPlatform}, @@ -503,6 +507,9 @@ func (impl PipelineBuilderImpl) getCiTemplateVariables(appId int) (ciConfig *bea CreatedBy: template.CreatedBy, CreatedOn: template.CreatedOn, } + if dockerRegistry != nil { + ciConfig.DockerRegistry = dockerRegistry.Id + } return ciConfig, err } @@ -948,35 +955,37 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq } //--ecr config createRequest.AppName = app.AppName - store, err := impl.getDefaultArtifactStore(createRequest.DockerRegistry) - if err != nil { - impl.logger.Errorw("error in fetching docker store ", "id", createRequest.DockerRepository, "err", err) - return nil, err - } - regHost, err := store.GetRegistryLocation() - if err != nil { - impl.logger.Errorw("invalid reg url", "err", err) - return nil, err - } - createRequest.DockerRegistryUrl = regHost - createRequest.DockerRegistry = store.Id - - var repo string - if createRequest.DockerRepository != "" { - repo = createRequest.DockerRepository - } else { - repo = impl.ecrConfig.EcrPrefix + app.AppName - } + if !createRequest.IsJob { + store, err := impl.getDefaultArtifactStore(createRequest.DockerRegistry) + if err != nil { + impl.logger.Errorw("error in fetching docker store ", "id", createRequest.DockerRepository, "err", err) + return nil, err + } - if store.RegistryType == dockerRegistryRepository.REGISTRYTYPE_ECR { - err := impl.ciCdPipelineOrchestrator.CreateEcrRepo(repo, store.AWSRegion, store.AWSAccessKeyId, store.AWSSecretAccessKey) + regHost, err := store.GetRegistryLocation() if err != nil { - impl.logger.Errorw("ecr repo creation failed while creating ci pipeline", "repo", repo, "err", err) + impl.logger.Errorw("invalid reg url", "err", err) return nil, err } - } - createRequest.DockerRepository = repo + createRequest.DockerRegistryUrl = regHost + createRequest.DockerRegistry = store.Id + var repo string + if createRequest.DockerRepository != "" { + repo = createRequest.DockerRepository + } else { + repo = impl.ecrConfig.EcrPrefix + app.AppName + } + + if store.RegistryType == dockerRegistryRepository.REGISTRYTYPE_ECR { + err := impl.ciCdPipelineOrchestrator.CreateEcrRepo(repo, store.AWSRegion, store.AWSAccessKeyId, store.AWSSecretAccessKey) + if err != nil { + impl.logger.Errorw("ecr repo creation failed while creating ci pipeline", "repo", repo, "err", err) + return nil, err + } + } + createRequest.DockerRepository = repo + } //--ecr config end //-- template config start @@ -994,9 +1003,9 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq } buildConfig := createRequest.CiBuildConfig ciTemplate := &pipelineConfig.CiTemplate{ - DockerRegistryId: createRequest.DockerRegistry, - DockerRepository: createRequest.DockerRepository, - GitMaterialId: buildConfig.GitMaterialId, + //DockerRegistryId: createRequest.DockerRegistry, + //DockerRepository: createRequest.DockerRepository, + GitMaterialId: buildConfig.GitMaterialId, //DockerfilePath: createRequest.DockerBuildConfig.DockerfilePath, //Args: string(argByte), //TargetPlatform: createRequest.DockerBuildConfig.TargetPlatform, @@ -1008,6 +1017,10 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq BeforeDockerBuild: string(beforeByte), AuditLog: sql.AuditLog{CreatedOn: time.Now(), UpdatedOn: time.Now(), CreatedBy: createRequest.UserId, UpdatedBy: createRequest.UserId}, } + if !createRequest.IsJob { + ciTemplate.DockerRegistryId = createRequest.DockerRegistry + ciTemplate.DockerRepository = createRequest.DockerRepository + } ciTemplateBean := &bean3.CiTemplateBean{ CiTemplate: ciTemplate, diff --git a/vendor/github.com/argoproj/argo-cd/assets/badge.svg b/vendor/github.com/argoproj/argo-cd/assets/badge.svg deleted file mode 100644 index a3234cfdf5..0000000000 --- a/vendor/github.com/argoproj/argo-cd/assets/badge.svg +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv b/vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv deleted file mode 100644 index f74c5b8002..0000000000 --- a/vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv +++ /dev/null @@ -1,34 +0,0 @@ -# Built-in policy which defines two roles: role:readonly and role:admin, -# and additionally assigns the admin user to the role:admin role. -# There are two policy formats: -# 1. Applications (which belong to a project): -# p, , , , / -# 2. All other resources: -# p, , , , - -p, role:readonly, applications, get, */*, allow -p, role:readonly, certificates, get, *, allow -p, role:readonly, clusters, get, *, allow -p, role:readonly, repositories, get, *, allow -p, role:readonly, projects, get, *, allow - -p, role:admin, applications, create, */*, allow -p, role:admin, applications, update, */*, allow -p, role:admin, applications, delete, */*, allow -p, role:admin, applications, sync, */*, allow -p, role:admin, applications, override, */*, allow -p, role:admin, certificates, create, *, allow -p, role:admin, certificates, update, *, allow -p, role:admin, certificates, delete, *, allow -p, role:admin, clusters, create, *, allow -p, role:admin, clusters, update, *, allow -p, role:admin, clusters, delete, *, allow -p, role:admin, repositories, create, *, allow -p, role:admin, repositories, update, *, allow -p, role:admin, repositories, delete, *, allow -p, role:admin, projects, create, *, allow -p, role:admin, projects, update, *, allow -p, role:admin, projects, delete, *, allow - -g, role:admin, role:readonly -g, admin, role:admin \ No newline at end of file diff --git a/vendor/github.com/argoproj/argo-cd/assets/model.conf b/vendor/github.com/argoproj/argo-cd/assets/model.conf deleted file mode 100644 index 240a9180d3..0000000000 --- a/vendor/github.com/argoproj/argo-cd/assets/model.conf +++ /dev/null @@ -1,14 +0,0 @@ -[request_definition] -r = sub, res, act, obj - -[policy_definition] -p = sub, res, act, obj, eft - -[role_definition] -g = _, _ - -[policy_effect] -e = some(where (p.eft == allow)) && !some(where (p.eft == deny)) - -[matchers] -m = g(r.sub, p.sub) && keyMatch(r.res, p.res) && keyMatch(r.act, p.act) && keyMatch(r.obj, p.obj) \ No newline at end of file diff --git a/vendor/github.com/argoproj/argo-cd/assets/swagger.json b/vendor/github.com/argoproj/argo-cd/assets/swagger.json deleted file mode 100644 index 0ad53c18de..0000000000 --- a/vendor/github.com/argoproj/argo-cd/assets/swagger.json +++ /dev/null @@ -1,3887 +0,0 @@ -{ - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "schemes": [ - "http", - "https" - ], - "swagger": "2.0", - "info": { - "description": "Description of all APIs", - "title": "Consolidate Services", - "version": "version not set" - }, - "paths": { - "/api/v1/account/password": { - "put": { - "tags": [ - "AccountService" - ], - "summary": "UpdatePassword updates an account's password to a new value", - "operationId": "UpdatePassword", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/accountUpdatePasswordRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/accountUpdatePasswordResponse" - } - } - } - } - }, - "/api/v1/applications": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "List returns list of applications", - "operationId": "ListMixin6", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "query" - }, - { - "type": "string", - "name": "refresh", - "in": "query" - }, - { - "type": "array", - "items": { - "type": "string" - }, - "name": "project", - "in": "query" - }, - { - "type": "string", - "name": "resourceVersion", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1ApplicationList" - } - } - } - }, - "post": { - "tags": [ - "ApplicationService" - ], - "summary": "Create creates an application", - "operationId": "CreateMixin6", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - } - } - }, - "/api/v1/applications/{application.metadata.name}": { - "put": { - "tags": [ - "ApplicationService" - ], - "summary": "Update updates an application", - "operationId": "UpdateMixin6", - "parameters": [ - { - "type": "string", - "name": "application.metadata.name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - } - } - }, - "/api/v1/applications/{applicationName}/managed-resources": { - "get": { - "tags": [ - "ApplicationService" - ], - "operationId": "ManagedResources", - "parameters": [ - { - "type": "string", - "name": "applicationName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationManagedResourcesResponse" - } - } - } - } - }, - "/api/v1/applications/{applicationName}/resource-tree": { - "get": { - "tags": [ - "ApplicationService" - ], - "operationId": "ResourceTree", - "parameters": [ - { - "type": "string", - "name": "applicationName", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1ApplicationTree" - } - } - } - } - }, - "/api/v1/applications/{name}": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "Get returns an application by name", - "operationId": "GetMixin6", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "refresh", - "in": "query" - }, - { - "type": "array", - "items": { - "type": "string" - }, - "name": "project", - "in": "query" - }, - { - "type": "string", - "name": "resourceVersion", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - } - }, - "delete": { - "tags": [ - "ApplicationService" - ], - "summary": "Delete deletes an application", - "operationId": "DeleteMixin6", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationApplicationResponse" - } - } - } - }, - "patch": { - "tags": [ - "ApplicationService" - ], - "summary": "Patch patch an application", - "operationId": "Patch", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/applicationApplicationPatchRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - } - } - }, - "/api/v1/applications/{name}/events": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "ListResourceEvents returns a list of event resources", - "operationId": "ListResourceEvents", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "resourceNamespace", - "in": "query" - }, - { - "type": "string", - "name": "resourceName", - "in": "query" - }, - { - "type": "string", - "name": "resourceUID", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1EventList" - } - } - } - } - }, - "/api/v1/applications/{name}/manifests": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "GetManifests returns application manifests", - "operationId": "GetManifests", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "revision", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/repositoryManifestResponse" - } - } - } - } - }, - "/api/v1/applications/{name}/operation": { - "delete": { - "tags": [ - "ApplicationService" - ], - "summary": "TerminateOperation terminates the currently running operation", - "operationId": "TerminateOperation", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationOperationTerminateResponse" - } - } - } - } - }, - "/api/v1/applications/{name}/pods/{podName}/logs": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "PodLogs returns stream of log entries for the specified pod. Pod", - "operationId": "PodLogs", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "podName", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "namespace", - "in": "query" - }, - { - "type": "string", - "name": "container", - "in": "query" - }, - { - "type": "string", - "format": "int64", - "name": "sinceSeconds", - "in": "query" - }, - { - "type": "string", - "format": "int64", - "description": "Represents seconds of UTC time since Unix epoch\n1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to\n9999-12-31T23:59:59Z inclusive.", - "name": "sinceTime.seconds", - "in": "query" - }, - { - "type": "integer", - "format": "int32", - "description": "Non-negative fractions of a second at nanosecond resolution. Negative\nsecond values with fractions must still have non-negative nanos values\nthat count forward in time. Must be from 0 to 999,999,999\ninclusive. This field may be limited in precision depending on context.", - "name": "sinceTime.nanos", - "in": "query" - }, - { - "type": "string", - "format": "int64", - "name": "tailLines", - "in": "query" - }, - { - "type": "boolean", - "format": "boolean", - "name": "follow", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(streaming responses)", - "schema": { - "$ref": "#/definitions/applicationLogEntry" - } - } - } - } - }, - "/api/v1/applications/{name}/resource": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "GetResource returns single application resource", - "operationId": "GetResource", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "namespace", - "in": "query" - }, - { - "type": "string", - "name": "resourceName", - "in": "query" - }, - { - "type": "string", - "name": "version", - "in": "query" - }, - { - "type": "string", - "name": "group", - "in": "query" - }, - { - "type": "string", - "name": "kind", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationApplicationResourceResponse" - } - } - } - }, - "post": { - "tags": [ - "ApplicationService" - ], - "summary": "PatchResource patch single application resource", - "operationId": "PatchResource", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationApplicationResourceResponse" - } - } - } - }, - "delete": { - "tags": [ - "ApplicationService" - ], - "summary": "DeleteResource deletes a single application resource", - "operationId": "DeleteResource", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationApplicationResponse" - } - } - } - } - }, - "/api/v1/applications/{name}/resource/actions": { - "get": { - "tags": [ - "ApplicationService" - ], - "operationId": "ListResourceActions", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "namespace", - "in": "query" - }, - { - "type": "string", - "name": "resourceName", - "in": "query" - }, - { - "type": "string", - "name": "version", - "in": "query" - }, - { - "type": "string", - "name": "group", - "in": "query" - }, - { - "type": "string", - "name": "kind", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationResourceActionsListResponse" - } - } - } - }, - "post": { - "tags": [ - "ApplicationService" - ], - "operationId": "RunResourceAction", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/applicationApplicationResponse" - } - } - } - } - }, - "/api/v1/applications/{name}/revisions/{revision}/metadata": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "Get the meta-data (author, date, tags, message) for a specific revision of the application", - "operationId": "RevisionMetadata", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "revision", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1RevisionMetadata" - } - } - } - } - }, - "/api/v1/applications/{name}/rollback": { - "post": { - "tags": [ - "ApplicationService" - ], - "summary": "Rollback syncs an application to its target state", - "operationId": "Rollback", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/applicationApplicationRollbackRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - } - } - }, - "/api/v1/applications/{name}/spec": { - "put": { - "tags": [ - "ApplicationService" - ], - "summary": "UpdateSpec updates an application spec", - "operationId": "UpdateSpec", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1ApplicationSpec" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1ApplicationSpec" - } - } - } - } - }, - "/api/v1/applications/{name}/sync": { - "post": { - "tags": [ - "ApplicationService" - ], - "summary": "Sync syncs an application to its target state", - "operationId": "Sync", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/applicationApplicationSyncRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Application" - } - } - } - } - }, - "/api/v1/certificates": { - "get": { - "tags": [ - "CertificateService" - ], - "summary": "List all available repository certificates", - "operationId": "ListCertificates", - "parameters": [ - { - "type": "string", - "description": "A file-glob pattern (not regular expression) the host name has to match.", - "name": "hostNamePattern", - "in": "query" - }, - { - "type": "string", - "description": "The type of the certificate to match (ssh or https).", - "name": "certType", - "in": "query" - }, - { - "type": "string", - "description": "The sub type of the certificate to match (protocol dependent, usually only used for ssh certs).", - "name": "certSubType", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1RepositoryCertificateList" - } - } - } - }, - "post": { - "tags": [ - "CertificateService" - ], - "summary": "Creates repository certificates on the server", - "operationId": "CreateCertificate", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1RepositoryCertificateList" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1RepositoryCertificateList" - } - } - } - }, - "delete": { - "tags": [ - "CertificateService" - ], - "summary": "Delete the certificates that match the RepositoryCertificateQuery", - "operationId": "DeleteCertificate", - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1RepositoryCertificateList" - } - } - } - } - }, - "/api/v1/clusters": { - "get": { - "tags": [ - "ClusterService" - ], - "summary": "List returns list of clusters", - "operationId": "List", - "parameters": [ - { - "type": "string", - "name": "server", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1ClusterList" - } - } - } - }, - "post": { - "tags": [ - "ClusterService" - ], - "summary": "Create creates a cluster", - "operationId": "Create", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1Cluster" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Cluster" - } - } - } - } - }, - "/api/v1/clusters/{cluster.server}": { - "put": { - "tags": [ - "ClusterService" - ], - "summary": "Update updates a cluster", - "operationId": "Update", - "parameters": [ - { - "type": "string", - "name": "cluster.server", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1Cluster" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Cluster" - } - } - } - } - }, - "/api/v1/clusters/{server}": { - "get": { - "tags": [ - "ClusterService" - ], - "summary": "Get returns a cluster by server address", - "operationId": "GetMixin1", - "parameters": [ - { - "type": "string", - "name": "server", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Cluster" - } - } - } - }, - "delete": { - "tags": [ - "ClusterService" - ], - "summary": "Delete deletes a cluster", - "operationId": "Delete", - "parameters": [ - { - "type": "string", - "name": "server", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/clusterClusterResponse" - } - } - } - } - }, - "/api/v1/clusters/{server}/rotate-auth": { - "post": { - "tags": [ - "ClusterService" - ], - "summary": "RotateAuth returns a cluster by server address", - "operationId": "RotateAuth", - "parameters": [ - { - "type": "string", - "name": "server", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/clusterClusterResponse" - } - } - } - } - }, - "/api/v1/projects": { - "get": { - "tags": [ - "ProjectService" - ], - "summary": "List returns list of projects", - "operationId": "ListMixin4", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1AppProjectList" - } - } - } - }, - "post": { - "tags": [ - "ProjectService" - ], - "summary": "Create a new project.", - "operationId": "CreateMixin4", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/projectProjectCreateRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1AppProject" - } - } - } - } - }, - "/api/v1/projects/{name}": { - "get": { - "tags": [ - "ProjectService" - ], - "summary": "Get returns a project by name", - "operationId": "GetMixin4", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1AppProject" - } - } - } - }, - "delete": { - "tags": [ - "ProjectService" - ], - "summary": "Delete deletes a project", - "operationId": "DeleteMixin4", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/projectEmptyResponse" - } - } - } - } - }, - "/api/v1/projects/{name}/events": { - "get": { - "tags": [ - "ProjectService" - ], - "summary": "ListEvents returns a list of project events", - "operationId": "ListEvents", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1EventList" - } - } - } - } - }, - "/api/v1/projects/{project.metadata.name}": { - "put": { - "tags": [ - "ProjectService" - ], - "summary": "Update updates a project", - "operationId": "UpdateMixin4", - "parameters": [ - { - "type": "string", - "name": "project.metadata.name", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/projectProjectUpdateRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1AppProject" - } - } - } - } - }, - "/api/v1/projects/{project}/roles/{role}/token": { - "post": { - "tags": [ - "ProjectService" - ], - "summary": "Create a new project token.", - "operationId": "CreateToken", - "parameters": [ - { - "type": "string", - "name": "project", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "role", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/projectProjectTokenCreateRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/projectProjectTokenResponse" - } - } - } - } - }, - "/api/v1/projects/{project}/roles/{role}/token/{iat}": { - "delete": { - "tags": [ - "ProjectService" - ], - "summary": "Delete a new project token.", - "operationId": "DeleteToken", - "parameters": [ - { - "type": "string", - "name": "project", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "role", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "int64", - "name": "iat", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/projectEmptyResponse" - } - } - } - } - }, - "/api/v1/repositories": { - "get": { - "tags": [ - "RepositoryService" - ], - "summary": "List returns list of repos", - "operationId": "ListMixin2", - "parameters": [ - { - "type": "string", - "name": "repo", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1RepositoryList" - } - } - } - }, - "post": { - "tags": [ - "RepositoryService" - ], - "summary": "Create creates a repo", - "operationId": "CreateMixin2", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1Repository" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Repository" - } - } - } - } - }, - "/api/v1/repositories/{repo.repo}": { - "put": { - "tags": [ - "RepositoryService" - ], - "summary": "Update updates a repo", - "operationId": "UpdateMixin2", - "parameters": [ - { - "type": "string", - "name": "repo.repo", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/v1alpha1Repository" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/v1alpha1Repository" - } - } - } - } - }, - "/api/v1/repositories/{repo}": { - "delete": { - "tags": [ - "RepositoryService" - ], - "summary": "Delete deletes a repo", - "operationId": "DeleteMixin2", - "parameters": [ - { - "type": "string", - "name": "repo", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/repositoryRepoResponse" - } - } - } - } - }, - "/api/v1/repositories/{repo}/apps": { - "get": { - "tags": [ - "RepositoryService" - ], - "summary": "ListApps returns list of apps in the repo", - "operationId": "ListApps", - "parameters": [ - { - "type": "string", - "name": "repo", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "revision", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/repositoryRepoAppsResponse" - } - } - } - } - }, - "/api/v1/repositories/{repo}/apps/{path}": { - "get": { - "tags": [ - "RepositoryService" - ], - "summary": "GetAppDetails returns application details by given path", - "operationId": "GetAppDetails", - "parameters": [ - { - "type": "string", - "name": "repo", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "path", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "revision", - "in": "query" - }, - { - "type": "array", - "items": { - "type": "string" - }, - "name": "helm.valueFiles", - "in": "query" - }, - { - "type": "string", - "name": "ksonnet.environment", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/repositoryRepoAppDetailsResponse" - } - } - } - } - }, - "/api/v1/repositories/{repo}/validate": { - "post": { - "tags": [ - "RepositoryService" - ], - "summary": "ValidateAccess validates access to a repository with given parameters", - "operationId": "ValidateAccess", - "parameters": [ - { - "type": "string", - "name": "repo", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/repositoryRepoResponse" - } - } - } - } - }, - "/api/v1/session": { - "post": { - "tags": [ - "SessionService" - ], - "summary": "Create a new JWT for authentication and set a cookie if using HTTP.", - "operationId": "CreateMixin8", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/sessionSessionCreateRequest" - } - } - ], - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/sessionSessionResponse" - } - } - } - }, - "delete": { - "tags": [ - "SessionService" - ], - "summary": "Delete an existing JWT cookie if using HTTP.", - "operationId": "DeleteMixin8", - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/sessionSessionResponse" - } - } - } - } - }, - "/api/v1/settings": { - "get": { - "tags": [ - "SettingsService" - ], - "summary": "Get returns Argo CD settings", - "operationId": "Get", - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/clusterSettings" - } - } - } - } - }, - "/api/v1/stream/applications": { - "get": { - "tags": [ - "ApplicationService" - ], - "summary": "Watch returns stream of application change events.", - "operationId": "Watch", - "parameters": [ - { - "type": "string", - "name": "name", - "in": "query" - }, - { - "type": "string", - "name": "refresh", - "in": "query" - }, - { - "type": "array", - "items": { - "type": "string" - }, - "name": "project", - "in": "query" - }, - { - "type": "string", - "name": "resourceVersion", - "in": "query" - } - ], - "responses": { - "200": { - "description": "(streaming responses)", - "schema": { - "$ref": "#/definitions/v1alpha1ApplicationWatchEvent" - } - } - } - } - }, - "/api/version": { - "get": { - "tags": [ - "VersionService" - ], - "summary": "Version returns version information of the API server", - "operationId": "Version", - "responses": { - "200": { - "description": "(empty)", - "schema": { - "$ref": "#/definitions/versionVersionMessage" - } - } - } - } - } - }, - "definitions": { - "accountUpdatePasswordRequest": { - "type": "object", - "properties": { - "currentPassword": { - "type": "string" - }, - "newPassword": { - "type": "string" - } - } - }, - "accountUpdatePasswordResponse": { - "type": "object" - }, - "applicationApplicationPatchRequest": { - "type": "object", - "title": "ApplicationPatchRequest is a request to patch an application", - "properties": { - "name": { - "type": "string" - }, - "patch": { - "type": "string" - }, - "patchType": { - "type": "string" - } - } - }, - "applicationApplicationResourceResponse": { - "type": "object", - "properties": { - "manifest": { - "type": "string" - } - } - }, - "applicationApplicationResponse": { - "type": "object" - }, - "applicationApplicationRollbackRequest": { - "type": "object", - "properties": { - "dryRun": { - "type": "boolean", - "format": "boolean" - }, - "id": { - "type": "string", - "format": "int64" - }, - "name": { - "type": "string" - }, - "prune": { - "type": "boolean", - "format": "boolean" - } - } - }, - "applicationApplicationSyncRequest": { - "type": "object", - "title": "ApplicationSyncRequest is a request to apply the config state to live state", - "properties": { - "dryRun": { - "type": "boolean", - "format": "boolean" - }, - "manifests": { - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "type": "string" - }, - "prune": { - "type": "boolean", - "format": "boolean" - }, - "resources": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1SyncOperationResource" - } - }, - "revision": { - "type": "string" - }, - "strategy": { - "$ref": "#/definitions/v1alpha1SyncStrategy" - } - } - }, - "applicationLogEntry": { - "type": "object", - "properties": { - "content": { - "type": "string" - }, - "timeStamp": { - "$ref": "#/definitions/v1Time" - } - } - }, - "applicationManagedResourcesResponse": { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceDiff" - } - } - } - }, - "applicationOperationTerminateResponse": { - "type": "object" - }, - "applicationResourceActionsListResponse": { - "type": "object", - "properties": { - "actions": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceAction" - } - } - } - }, - "clusterClusterResponse": { - "type": "object" - }, - "clusterConnector": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, - "clusterDexConfig": { - "type": "object", - "properties": { - "connectors": { - "type": "array", - "items": { - "$ref": "#/definitions/clusterConnector" - } - } - } - }, - "clusterGoogleAnalyticsConfig": { - "type": "object", - "properties": { - "anonymizeUsers": { - "type": "boolean", - "format": "boolean" - }, - "trackingID": { - "type": "string" - } - } - }, - "clusterHelp": { - "type": "object", - "title": "Help settings", - "properties": { - "chatText": { - "type": "string", - "title": "the text for getting chat help, defaults to \"Chat now!\"" - }, - "chatUrl": { - "type": "string", - "title": "the URL for getting chat help, this will typically be your Slack channel for support" - } - } - }, - "clusterOIDCConfig": { - "type": "object", - "properties": { - "cliClientID": { - "type": "string" - }, - "clientID": { - "type": "string" - }, - "issuer": { - "type": "string" - }, - "name": { - "type": "string" - }, - "scopes": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "clusterSettings": { - "type": "object", - "properties": { - "appLabelKey": { - "type": "string" - }, - "dexConfig": { - "$ref": "#/definitions/clusterDexConfig" - }, - "googleAnalytics": { - "$ref": "#/definitions/clusterGoogleAnalyticsConfig" - }, - "help": { - "$ref": "#/definitions/clusterHelp" - }, - "kustomizeOptions": { - "$ref": "#/definitions/v1alpha1KustomizeOptions" - }, - "oidcConfig": { - "$ref": "#/definitions/clusterOIDCConfig" - }, - "resourceOverrides": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/v1alpha1ResourceOverride" - } - }, - "statusBadgeEnabled": { - "type": "boolean", - "format": "boolean" - }, - "url": { - "type": "string" - } - } - }, - "projectEmptyResponse": { - "type": "object" - }, - "projectProjectCreateRequest": { - "description": "ProjectCreateRequest defines project creation parameters.", - "type": "object", - "properties": { - "project": { - "$ref": "#/definitions/v1alpha1AppProject" - } - } - }, - "projectProjectTokenCreateRequest": { - "description": "ProjectTokenCreateRequest defines project token creation parameters.", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "expiresIn": { - "type": "string", - "format": "int64", - "title": "expiresIn represents a duration in seconds" - }, - "project": { - "type": "string" - }, - "role": { - "type": "string" - } - } - }, - "projectProjectTokenResponse": { - "description": "ProjectTokenResponse wraps the created token or returns an empty string if deleted.", - "type": "object", - "properties": { - "token": { - "type": "string" - } - } - }, - "projectProjectUpdateRequest": { - "type": "object", - "properties": { - "project": { - "$ref": "#/definitions/v1alpha1AppProject" - } - } - }, - "repositoryAppInfo": { - "type": "object", - "title": "AppInfo contains application type and app file path", - "properties": { - "path": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, - "repositoryDirectoryAppSpec": { - "type": "object", - "title": "DirectoryAppSpec contains directory" - }, - "repositoryHelmAppDetailsQuery": { - "type": "object", - "properties": { - "valueFiles": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "repositoryHelmAppSpec": { - "type": "object", - "title": "HelmAppSpec contains helm app name and path in source repo", - "properties": { - "name": { - "type": "string" - }, - "parameters": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1HelmParameter" - } - }, - "path": { - "type": "string" - }, - "valueFiles": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "repositoryKsonnetAppDetailsQuery": { - "type": "object", - "properties": { - "environment": { - "type": "string" - } - } - }, - "repositoryKsonnetAppSpec": { - "type": "object", - "title": "KsonnetAppSpec contains Ksonnet app response\nThis roughly reflects: ksonnet/ksonnet/metadata/app/schema.go", - "properties": { - "environments": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/repositoryKsonnetEnvironment" - } - }, - "name": { - "type": "string" - }, - "parameters": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1KsonnetParameter" - } - }, - "path": { - "type": "string" - } - } - }, - "repositoryKsonnetEnvironment": { - "type": "object", - "properties": { - "destination": { - "$ref": "#/definitions/repositoryKsonnetEnvironmentDestination" - }, - "k8sVersion": { - "description": "KubernetesVersion is the kubernetes version the targetted cluster is running on.", - "type": "string" - }, - "name": { - "type": "string", - "title": "Name is the user defined name of an environment" - }, - "path": { - "description": "Path is the relative project path containing metadata for this environment.", - "type": "string" - } - } - }, - "repositoryKsonnetEnvironmentDestination": { - "type": "object", - "properties": { - "namespace": { - "type": "string", - "title": "Namespace is the namespace of the Kubernetes server that targets should be deployed to" - }, - "server": { - "description": "Server is the Kubernetes server that the cluster is running on.", - "type": "string" - } - } - }, - "repositoryKustomizeAppSpec": { - "type": "object", - "title": "KustomizeAppSpec contains kustomize app name and path in source repo", - "properties": { - "images": { - "description": "images is a list of available images.", - "type": "array", - "items": { - "type": "string" - } - }, - "path": { - "type": "string" - } - } - }, - "repositoryManifestResponse": { - "type": "object", - "properties": { - "manifests": { - "type": "array", - "items": { - "type": "string" - } - }, - "namespace": { - "type": "string" - }, - "revision": { - "type": "string" - }, - "server": { - "type": "string" - }, - "sourceType": { - "type": "string" - } - } - }, - "repositoryRepoAppDetailsResponse": { - "type": "object", - "title": "RepoAppDetailsResponse application details", - "properties": { - "directory": { - "$ref": "#/definitions/repositoryDirectoryAppSpec" - }, - "helm": { - "$ref": "#/definitions/repositoryHelmAppSpec" - }, - "ksonnet": { - "$ref": "#/definitions/repositoryKsonnetAppSpec" - }, - "kustomize": { - "$ref": "#/definitions/repositoryKustomizeAppSpec" - }, - "type": { - "type": "string" - } - } - }, - "repositoryRepoAppsResponse": { - "type": "object", - "title": "RepoAppsResponse contains applications of specified repository", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/repositoryAppInfo" - } - } - } - }, - "repositoryRepoResponse": { - "type": "object" - }, - "sessionSessionCreateRequest": { - "description": "SessionCreateRequest is for logging in.", - "type": "object", - "properties": { - "password": { - "type": "string" - }, - "token": { - "type": "string" - }, - "username": { - "type": "string" - } - } - }, - "sessionSessionResponse": { - "description": "SessionResponse wraps the created token or returns an empty string if deleted.", - "type": "object", - "properties": { - "token": { - "type": "string" - } - } - }, - "v1Event": { - "description": "Event is a report of an event somewhere in the cluster.", - "type": "object", - "properties": { - "action": { - "type": "string", - "title": "What action was taken/failed regarding to the Regarding object.\n+optional" - }, - "count": { - "type": "integer", - "format": "int32", - "title": "The number of times this event has occurred.\n+optional" - }, - "eventTime": { - "$ref": "#/definitions/v1MicroTime" - }, - "firstTimestamp": { - "$ref": "#/definitions/v1Time" - }, - "involvedObject": { - "$ref": "#/definitions/v1ObjectReference" - }, - "lastTimestamp": { - "$ref": "#/definitions/v1Time" - }, - "message": { - "type": "string", - "title": "A human-readable description of the status of this operation.\nTODO: decide on maximum length.\n+optional" - }, - "metadata": { - "$ref": "#/definitions/v1ObjectMeta" - }, - "reason": { - "type": "string", - "title": "This should be a short, machine understandable string that gives the reason\nfor the transition into the object's current status.\nTODO: provide exact specification for format.\n+optional" - }, - "related": { - "$ref": "#/definitions/v1ObjectReference" - }, - "reportingComponent": { - "type": "string", - "title": "Name of the controller that emitted this Event, e.g. `kubernetes.io/kubelet`.\n+optional" - }, - "reportingInstance": { - "type": "string", - "title": "ID of the controller instance, e.g. `kubelet-xyzf`.\n+optional" - }, - "series": { - "$ref": "#/definitions/v1EventSeries" - }, - "source": { - "$ref": "#/definitions/v1EventSource" - }, - "type": { - "type": "string", - "title": "Type of this event (Normal, Warning), new types could be added in the future\n+optional" - } - } - }, - "v1EventList": { - "description": "EventList is a list of events.", - "type": "object", - "properties": { - "items": { - "type": "array", - "title": "List of events", - "items": { - "$ref": "#/definitions/v1Event" - } - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - } - } - }, - "v1EventSeries": { - "description": "EventSeries contain information on series of events, i.e. thing that was/is happening\ncontinuously for some time.", - "type": "object", - "properties": { - "count": { - "type": "integer", - "format": "int32", - "title": "Number of occurrences in this series up to the last heartbeat time" - }, - "lastObservedTime": { - "$ref": "#/definitions/v1MicroTime" - }, - "state": { - "type": "string", - "title": "State of this Series: Ongoing or Finished" - } - } - }, - "v1EventSource": { - "description": "EventSource contains information for an event.", - "type": "object", - "properties": { - "component": { - "type": "string", - "title": "Component from which the event is generated.\n+optional" - }, - "host": { - "type": "string", - "title": "Node name on which the event is generated.\n+optional" - } - } - }, - "v1Fields": { - "type": "object", - "title": "Fields stores a set of fields in a data structure like a Trie.\nTo understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff", - "properties": { - "map": { - "description": "Map stores a set of fields in a data structure like a Trie.\n\nEach key is either a '.' representing the field itself, and will always map to an empty set,\nor a string representing a sub-field or item. The string will follow one of these four formats:\n'f:', where is the name of a field in a struct, or key in a map\n'v:', where is the exact json formatted value of a list item\n'i:', where is position of a item in a list\n'k:', where is a map of a list item's key fields to their unique values\nIf a key maps to an empty Fields value, the field that key represents is part of the set.\n\nThe exact format is defined in k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/v1Fields" - } - } - } - }, - "v1GroupKind": { - "description": "+protobuf.options.(gogoproto.goproto_stringer)=false", - "type": "object", - "title": "GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying\nconcepts during lookup stages without having partially valid types", - "properties": { - "group": { - "type": "string" - }, - "kind": { - "type": "string" - } - } - }, - "v1Initializer": { - "description": "Initializer is information about an initializer that has not yet completed.", - "type": "object", - "properties": { - "name": { - "description": "name of the process that is responsible for initializing this object.", - "type": "string" - } - } - }, - "v1Initializers": { - "description": "Initializers tracks the progress of initialization.", - "type": "object", - "properties": { - "pending": { - "type": "array", - "title": "Pending is a list of initializers that must execute in order before this object is visible.\nWhen the last pending initializer is removed, and no failing result is set, the initializers\nstruct will be set to nil and the object is considered as initialized and visible to all\nclients.\n+patchMergeKey=name\n+patchStrategy=merge", - "items": { - "$ref": "#/definitions/v1Initializer" - } - }, - "result": { - "$ref": "#/definitions/v1Status" - } - } - }, - "v1ListMeta": { - "description": "ListMeta describes metadata that synthetic resources must have, including lists and\nvarious status objects. A resource may have only one of {ObjectMeta, ListMeta}.", - "type": "object", - "properties": { - "continue": { - "description": "continue may be set if the user set a limit on the number of items returned, and indicates that\nthe server has more data available. The value is opaque and may be used to issue another request\nto the endpoint that served this list to retrieve the next set of available objects. Continuing a\nconsistent list may not be possible if the server configuration has changed or more than a few\nminutes have passed. The resourceVersion field returned when using this continue value will be\nidentical to the value in the first response, unless you have received this token from an error\nmessage.", - "type": "string" - }, - "resourceVersion": { - "type": "string", - "title": "String that identifies the server's internal version of this object that\ncan be used by clients to determine when objects have changed.\nValue must be treated as opaque by clients and passed unmodified back to the server.\nPopulated by the system.\nRead-only.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\n+optional" - }, - "selfLink": { - "type": "string", - "title": "selfLink is a URL representing this object.\nPopulated by the system.\nRead-only.\n+optional" - } - } - }, - "v1LoadBalancerIngress": { - "description": "LoadBalancerIngress represents the status of a load-balancer ingress point:\ntraffic intended for the service should be sent to an ingress point.", - "type": "object", - "properties": { - "hostname": { - "type": "string", - "title": "Hostname is set for load-balancer ingress points that are DNS based\n(typically AWS load-balancers)\n+optional" - }, - "ip": { - "type": "string", - "title": "IP is set for load-balancer ingress points that are IP based\n(typically GCE or OpenStack load-balancers)\n+optional" - } - } - }, - "v1ManagedFieldsEntry": { - "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource\nthat the fieldset applies to.", - "type": "object", - "properties": { - "apiVersion": { - "description": "APIVersion defines the version of this resource that this field set\napplies to. The format is \"group/version\" just like the top-level\nAPIVersion field. It is necessary to track the version of a field\nset because it cannot be automatically converted.", - "type": "string" - }, - "fields": { - "$ref": "#/definitions/v1Fields" - }, - "manager": { - "description": "Manager is an identifier of the workflow managing these fields.", - "type": "string" - }, - "operation": { - "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created.\nThe only valid values for this field are 'Apply' and 'Update'.", - "type": "string" - }, - "time": { - "$ref": "#/definitions/v1Time" - } - } - }, - "v1MicroTime": { - "description": "MicroTime is version of Time with microsecond level precision.\n\n+protobuf.options.marshal=false\n+protobuf.as=Timestamp\n+protobuf.options.(gogoproto.goproto_stringer)=false", - "type": "object", - "properties": { - "nanos": { - "description": "Non-negative fractions of a second at nanosecond resolution. Negative\nsecond values with fractions must still have non-negative nanos values\nthat count forward in time. Must be from 0 to 999,999,999\ninclusive. This field may be limited in precision depending on context.", - "type": "integer", - "format": "int32" - }, - "seconds": { - "description": "Represents seconds of UTC time since Unix epoch\n1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to\n9999-12-31T23:59:59Z inclusive.", - "type": "string", - "format": "int64" - } - } - }, - "v1ObjectMeta": { - "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects\nusers must create.", - "type": "object", - "properties": { - "annotations": { - "type": "object", - "title": "Annotations is an unstructured key value map stored with a resource that may be\nset by external tools to store and retrieve arbitrary metadata. They are not\nqueryable and should be preserved when modifying objects.\nMore info: http://kubernetes.io/docs/user-guide/annotations\n+optional", - "additionalProperties": { - "type": "string" - } - }, - "clusterName": { - "type": "string", - "title": "The name of the cluster which the object belongs to.\nThis is used to distinguish resources with same name and namespace in different clusters.\nThis field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\n+optional" - }, - "creationTimestamp": { - "$ref": "#/definitions/v1Time" - }, - "deletionGracePeriodSeconds": { - "type": "string", - "format": "int64", - "title": "Number of seconds allowed for this object to gracefully terminate before\nit will be removed from the system. Only set when deletionTimestamp is also set.\nMay only be shortened.\nRead-only.\n+optional" - }, - "deletionTimestamp": { - "$ref": "#/definitions/v1Time" - }, - "finalizers": { - "type": "array", - "title": "Must be empty before the object is deleted from the registry. Each entry\nis an identifier for the responsible component that will remove the entry\nfrom the list. If the deletionTimestamp of the object is non-nil, entries\nin this list can only be removed.\n+optional\n+patchStrategy=merge", - "items": { - "type": "string" - } - }, - "generateName": { - "description": "GenerateName is an optional prefix, used by the server, to generate a unique\nname ONLY IF the Name field has not been provided.\nIf this field is used, the name returned to the client will be different\nthan the name passed. This value will also be combined with a unique suffix.\nThe provided value has the same validation rules as the Name field,\nand may be truncated by the length of the suffix required to make the value\nunique on the server.\n\nIf this field is specified and the generated name exists, the server will\nNOT return a 409 - instead, it will either return 201 Created or 500 with Reason\nServerTimeout indicating a unique name could not be found in the time allotted, and the client\nshould retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\n+optional", - "type": "string" - }, - "generation": { - "type": "string", - "format": "int64", - "title": "A sequence number representing a specific generation of the desired state.\nPopulated by the system. Read-only.\n+optional" - }, - "initializers": { - "$ref": "#/definitions/v1Initializers" - }, - "labels": { - "type": "object", - "title": "Map of string keys and values that can be used to organize and categorize\n(scope and select) objects. May match selectors of replication controllers\nand services.\nMore info: http://kubernetes.io/docs/user-guide/labels\n+optional", - "additionalProperties": { - "type": "string" - } - }, - "managedFields": { - "description": "ManagedFields maps workflow-id and version to the set of fields\nthat are managed by that workflow. This is mostly for internal\nhousekeeping, and users typically shouldn't need to set or\nunderstand this field. A workflow can be the user's name, a\ncontroller's name, or the name of a specific apply path like\n\"ci-cd\". The set of fields is always in the version that the\nworkflow used when modifying the object.\n\nThis field is alpha and can be changed or removed without notice.\n\n+optional", - "type": "array", - "items": { - "$ref": "#/definitions/v1ManagedFieldsEntry" - } - }, - "name": { - "type": "string", - "title": "Name must be unique within a namespace. Is required when creating resources, although\nsome resources may allow a client to request the generation of an appropriate name\nautomatically. Name is primarily intended for creation idempotence and configuration\ndefinition.\nCannot be updated.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#names\n+optional" - }, - "namespace": { - "description": "Namespace defines the space within each name must be unique. An empty namespace is\nequivalent to the \"default\" namespace, but \"default\" is the canonical representation.\nNot all objects are required to be scoped to a namespace - the value of this field for\nthose objects will be empty.\n\nMust be a DNS_LABEL.\nCannot be updated.\nMore info: http://kubernetes.io/docs/user-guide/namespaces\n+optional", - "type": "string" - }, - "ownerReferences": { - "type": "array", - "title": "List of objects depended by this object. If ALL objects in the list have\nbeen deleted, this object will be garbage collected. If this object is managed by a controller,\nthen an entry in this list will point to this controller, with the controller field set to true.\nThere cannot be more than one managing controller.\n+optional\n+patchMergeKey=uid\n+patchStrategy=merge", - "items": { - "$ref": "#/definitions/v1OwnerReference" - } - }, - "resourceVersion": { - "description": "An opaque value that represents the internal version of this object that can\nbe used by clients to determine when objects have changed. May be used for optimistic\nconcurrency, change detection, and the watch operation on a resource or set of resources.\nClients must treat these values as opaque and passed unmodified back to the server.\nThey may only be valid for a particular resource or set of resources.\n\nPopulated by the system.\nRead-only.\nValue must be treated as opaque by clients and .\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\n+optional", - "type": "string" - }, - "selfLink": { - "type": "string", - "title": "SelfLink is a URL representing this object.\nPopulated by the system.\nRead-only.\n+optional" - }, - "uid": { - "description": "UID is the unique in time and space value for this object. It is typically generated by\nthe server on successful creation of a resource and is not allowed to change on PUT\noperations.\n\nPopulated by the system.\nRead-only.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#uids\n+optional", - "type": "string" - } - } - }, - "v1ObjectReference": { - "type": "object", - "title": "ObjectReference contains enough information to let you inspect or modify the referred object.\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", - "properties": { - "apiVersion": { - "type": "string", - "title": "API version of the referent.\n+optional" - }, - "fieldPath": { - "type": "string", - "title": "If referring to a piece of an object instead of an entire object, this string\nshould contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].\nFor example, if the object reference is to a container within a pod, this would take on a value like:\n\"spec.containers{name}\" (where \"name\" refers to the name of the container that triggered\nthe event) or if no container name is specified \"spec.containers[2]\" (container with\nindex 2 in this pod). This syntax is chosen only to have some well-defined way of\nreferencing a part of an object.\nTODO: this design is not final and this field is subject to change in the future.\n+optional" - }, - "kind": { - "type": "string", - "title": "Kind of the referent.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\n+optional" - }, - "name": { - "type": "string", - "title": "Name of the referent.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n+optional" - }, - "namespace": { - "type": "string", - "title": "Namespace of the referent.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/\n+optional" - }, - "resourceVersion": { - "type": "string", - "title": "Specific resourceVersion to which this reference is made, if any.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\n+optional" - }, - "uid": { - "type": "string", - "title": "UID of the referent.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids\n+optional" - } - } - }, - "v1OwnerReference": { - "description": "OwnerReference contains enough information to let you identify an owning\nobject. An owning object must be in the same namespace as the dependent, or\nbe cluster-scoped, so there is no namespace field.", - "type": "object", - "properties": { - "apiVersion": { - "description": "API version of the referent.", - "type": "string" - }, - "blockOwnerDeletion": { - "type": "boolean", - "format": "boolean", - "title": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then\nthe owner cannot be deleted from the key-value store until this\nreference is removed.\nDefaults to false.\nTo set this field, a user needs \"delete\" permission of the owner,\notherwise 422 (Unprocessable Entity) will be returned.\n+optional" - }, - "controller": { - "type": "boolean", - "format": "boolean", - "title": "If true, this reference points to the managing controller.\n+optional" - }, - "kind": { - "type": "string", - "title": "Kind of the referent.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" - }, - "name": { - "type": "string", - "title": "Name of the referent.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#names" - }, - "uid": { - "type": "string", - "title": "UID of the referent.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#uids" - } - } - }, - "v1Status": { - "description": "Status is a return value for calls that don't return other objects.", - "type": "object", - "properties": { - "code": { - "type": "integer", - "format": "int32", - "title": "Suggested HTTP return code for this status, 0 if not set.\n+optional" - }, - "details": { - "$ref": "#/definitions/v1StatusDetails" - }, - "message": { - "type": "string", - "title": "A human-readable description of the status of this operation.\n+optional" - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - }, - "reason": { - "type": "string", - "title": "A machine-readable description of why this operation is in the\n\"Failure\" status. If this value is empty there\nis no information available. A Reason clarifies an HTTP status\ncode but does not override it.\n+optional" - }, - "status": { - "type": "string", - "title": "Status of the operation.\nOne of: \"Success\" or \"Failure\".\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\n+optional" - } - } - }, - "v1StatusCause": { - "description": "StatusCause provides more information about an api.Status failure, including\ncases when multiple errors are encountered.", - "type": "object", - "properties": { - "field": { - "description": "The field of the resource that has caused this error, as named by its JSON\nserialization. May include dot and postfix notation for nested attributes.\nArrays are zero-indexed. Fields may appear more than once in an array of\ncauses due to fields having multiple errors.\nOptional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"\n+optional", - "type": "string" - }, - "message": { - "type": "string", - "title": "A human-readable description of the cause of the error. This field may be\npresented as-is to a reader.\n+optional" - }, - "reason": { - "type": "string", - "title": "A machine-readable description of the cause of the error. If this value is\nempty there is no information available.\n+optional" - } - } - }, - "v1StatusDetails": { - "description": "StatusDetails is a set of additional properties that MAY be set by the\nserver to provide additional information about a response. The Reason\nfield of a Status object defines what attributes will be set. Clients\nmust ignore fields that do not match the defined type of each attribute,\nand should assume that any attribute may be empty, invalid, or under\ndefined.", - "type": "object", - "properties": { - "causes": { - "type": "array", - "title": "The Causes array includes more details associated with the StatusReason\nfailure. Not all StatusReasons may provide detailed causes.\n+optional", - "items": { - "$ref": "#/definitions/v1StatusCause" - } - }, - "group": { - "type": "string", - "title": "The group attribute of the resource associated with the status StatusReason.\n+optional" - }, - "kind": { - "type": "string", - "title": "The kind attribute of the resource associated with the status StatusReason.\nOn some operations may differ from the requested resource Kind.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\n+optional" - }, - "name": { - "type": "string", - "title": "The name attribute of the resource associated with the status StatusReason\n(when there is a single name which can be described).\n+optional" - }, - "retryAfterSeconds": { - "type": "integer", - "format": "int32", - "title": "If specified, the time in seconds before the operation should be retried. Some errors may indicate\nthe client must take an alternate action - for those errors this field may indicate how long to wait\nbefore taking the alternate action.\n+optional" - }, - "uid": { - "type": "string", - "title": "UID of the resource.\n(when there is a single resource which can be described).\nMore info: http://kubernetes.io/docs/user-guide/identifiers#uids\n+optional" - } - } - }, - "v1Time": { - "description": "Time is a wrapper around time.Time which supports correct\nmarshaling to YAML and JSON. Wrappers are provided for many\nof the factory methods that the time package offers.\n\n+protobuf.options.marshal=false\n+protobuf.as=Timestamp\n+protobuf.options.(gogoproto.goproto_stringer)=false", - "type": "object", - "properties": { - "nanos": { - "description": "Non-negative fractions of a second at nanosecond resolution. Negative\nsecond values with fractions must still have non-negative nanos values\nthat count forward in time. Must be from 0 to 999,999,999\ninclusive. This field may be limited in precision depending on context.", - "type": "integer", - "format": "int32" - }, - "seconds": { - "description": "Represents seconds of UTC time since Unix epoch\n1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to\n9999-12-31T23:59:59Z inclusive.", - "type": "string", - "format": "int64" - } - } - }, - "v1alpha1AWSAuthConfig": { - "type": "object", - "title": "AWSAuthConfig is an AWS IAM authentication configuration", - "properties": { - "clusterName": { - "type": "string", - "title": "ClusterName contains AWS cluster name" - }, - "roleARN": { - "description": "RoleARN contains optional role ARN. If set then AWS IAM Authenticator assume a role to perform cluster operations instead of the default AWS credential provider chain.", - "type": "string" - } - } - }, - "v1alpha1AppProject": { - "type": "object", - "title": "AppProject provides a logical grouping of applications, providing controls for:\n* where the apps may deploy to (cluster whitelist)\n* what may be deployed (repository whitelist, resource whitelist/blacklist)\n* who can access these applications (roles, OIDC group claims bindings)\n* and what they can do (RBAC policies)\n* automation access to these roles (JWT tokens)\n+genclient\n+genclient:noStatus\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n+kubebuilder:resource:path=appprojects,shortName=appproj;appprojs", - "properties": { - "metadata": { - "$ref": "#/definitions/v1ObjectMeta" - }, - "spec": { - "$ref": "#/definitions/v1alpha1AppProjectSpec" - } - } - }, - "v1alpha1AppProjectList": { - "type": "object", - "title": "AppProjectList is list of AppProject resources\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1AppProject" - } - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - } - } - }, - "v1alpha1AppProjectSpec": { - "type": "object", - "title": "AppProjectSpec is the specification of an AppProject", - "properties": { - "clusterResourceWhitelist": { - "type": "array", - "title": "ClusterResourceWhitelist contains list of whitelisted cluster level resources", - "items": { - "$ref": "#/definitions/v1GroupKind" - } - }, - "description": { - "type": "string", - "title": "Description contains optional project description" - }, - "destinations": { - "type": "array", - "title": "Destinations contains list of destinations available for deployment", - "items": { - "$ref": "#/definitions/v1alpha1ApplicationDestination" - } - }, - "namespaceResourceBlacklist": { - "type": "array", - "title": "NamespaceResourceBlacklist contains list of blacklisted namespace level resources", - "items": { - "$ref": "#/definitions/v1GroupKind" - } - }, - "roles": { - "type": "array", - "title": "Roles are user defined RBAC roles associated with this project", - "items": { - "$ref": "#/definitions/v1alpha1ProjectRole" - } - }, - "sourceRepos": { - "type": "array", - "title": "SourceRepos contains list of git repository URLs which can be used for deployment", - "items": { - "type": "string" - } - } - } - }, - "v1alpha1Application": { - "type": "object", - "title": "Application is a definition of Application resource.\n+genclient\n+genclient:noStatus\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n+kubebuilder:resource:path=applications,shortName=app;apps", - "properties": { - "metadata": { - "$ref": "#/definitions/v1ObjectMeta" - }, - "operation": { - "$ref": "#/definitions/v1alpha1Operation" - }, - "spec": { - "$ref": "#/definitions/v1alpha1ApplicationSpec" - }, - "status": { - "$ref": "#/definitions/v1alpha1ApplicationStatus" - } - } - }, - "v1alpha1ApplicationCondition": { - "type": "object", - "title": "ApplicationCondition contains details about current application condition", - "properties": { - "message": { - "type": "string", - "title": "Message contains human-readable message indicating details about condition" - }, - "type": { - "type": "string", - "title": "Type is an application condition type" - } - } - }, - "v1alpha1ApplicationDestination": { - "type": "object", - "title": "ApplicationDestination contains deployment destination information", - "properties": { - "namespace": { - "type": "string", - "title": "Namespace overrides the environment namespace value in the ksonnet app.yaml" - }, - "server": { - "type": "string", - "title": "Server overrides the environment server value in the ksonnet app.yaml" - } - } - }, - "v1alpha1ApplicationList": { - "type": "object", - "title": "ApplicationList is list of Application resources\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1Application" - } - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - } - } - }, - "v1alpha1ApplicationSource": { - "description": "ApplicationSource contains information about github repository, path within repository and target application environment.", - "type": "object", - "properties": { - "directory": { - "$ref": "#/definitions/v1alpha1ApplicationSourceDirectory" - }, - "helm": { - "$ref": "#/definitions/v1alpha1ApplicationSourceHelm" - }, - "ksonnet": { - "$ref": "#/definitions/v1alpha1ApplicationSourceKsonnet" - }, - "kustomize": { - "$ref": "#/definitions/v1alpha1ApplicationSourceKustomize" - }, - "path": { - "type": "string", - "title": "Path is a directory path within the repository containing a" - }, - "plugin": { - "$ref": "#/definitions/v1alpha1ApplicationSourcePlugin" - }, - "repoURL": { - "type": "string", - "title": "RepoURL is the git repository URL of the application manifests" - }, - "targetRevision": { - "type": "string", - "title": "TargetRevision defines the commit, tag, or branch in which to sync the application to.\nIf omitted, will sync to HEAD" - } - } - }, - "v1alpha1ApplicationSourceDirectory": { - "type": "object", - "properties": { - "jsonnet": { - "$ref": "#/definitions/v1alpha1ApplicationSourceJsonnet" - }, - "recurse": { - "type": "boolean", - "format": "boolean" - } - } - }, - "v1alpha1ApplicationSourceHelm": { - "type": "object", - "title": "ApplicationSourceHelm holds helm specific options", - "properties": { - "parameters": { - "type": "array", - "title": "Parameters are parameters to the helm template", - "items": { - "$ref": "#/definitions/v1alpha1HelmParameter" - } - }, - "releaseName": { - "type": "string", - "title": "The Helm release name. If omitted it will use the application name" - }, - "valueFiles": { - "type": "array", - "title": "ValuesFiles is a list of Helm value files to use when generating a template", - "items": { - "type": "string" - } - } - } - }, - "v1alpha1ApplicationSourceJsonnet": { - "type": "object", - "title": "ApplicationSourceJsonnet holds jsonnet specific options", - "properties": { - "extVars": { - "type": "array", - "title": "ExtVars is a list of Jsonnet External Variables", - "items": { - "$ref": "#/definitions/v1alpha1JsonnetVar" - } - }, - "tlas": { - "type": "array", - "title": "TLAS is a list of Jsonnet Top-level Arguments", - "items": { - "$ref": "#/definitions/v1alpha1JsonnetVar" - } - } - } - }, - "v1alpha1ApplicationSourceKsonnet": { - "type": "object", - "title": "ApplicationSourceKsonnet holds ksonnet specific options", - "properties": { - "environment": { - "type": "string", - "title": "Environment is a ksonnet application environment name" - }, - "parameters": { - "type": "array", - "title": "Parameters are a list of ksonnet component parameter override values", - "items": { - "$ref": "#/definitions/v1alpha1KsonnetParameter" - } - } - } - }, - "v1alpha1ApplicationSourceKustomize": { - "type": "object", - "title": "ApplicationSourceKustomize holds kustomize specific options", - "properties": { - "commonLabels": { - "type": "object", - "title": "CommonLabels adds additional kustomize commonLabels", - "additionalProperties": { - "type": "string" - } - }, - "images": { - "type": "array", - "title": "Images are kustomize image overrides", - "items": { - "type": "string" - } - }, - "namePrefix": { - "type": "string", - "title": "NamePrefix is a prefix appended to resources for kustomize apps" - } - } - }, - "v1alpha1ApplicationSourcePlugin": { - "type": "object", - "title": "ApplicationSourcePlugin holds config management plugin specific options", - "properties": { - "env": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1EnvEntry" - } - }, - "name": { - "type": "string" - } - } - }, - "v1alpha1ApplicationSpec": { - "description": "ApplicationSpec represents desired application state. Contains link to repository with application definition and additional parameters link definition revision.", - "type": "object", - "properties": { - "destination": { - "$ref": "#/definitions/v1alpha1ApplicationDestination" - }, - "ignoreDifferences": { - "type": "array", - "title": "IgnoreDifferences controls resources fields which should be ignored during comparison", - "items": { - "$ref": "#/definitions/v1alpha1ResourceIgnoreDifferences" - } - }, - "info": { - "type": "array", - "title": "Infos contains a list of useful information (URLs, email addresses, and plain text) that relates to the application", - "items": { - "$ref": "#/definitions/v1alpha1Info" - } - }, - "project": { - "description": "Project is a application project name. Empty name means that application belongs to 'default' project.", - "type": "string" - }, - "source": { - "$ref": "#/definitions/v1alpha1ApplicationSource" - }, - "syncPolicy": { - "$ref": "#/definitions/v1alpha1SyncPolicy" - } - } - }, - "v1alpha1ApplicationStatus": { - "type": "object", - "title": "ApplicationStatus contains information about application sync, health status", - "properties": { - "conditions": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ApplicationCondition" - } - }, - "health": { - "$ref": "#/definitions/v1alpha1HealthStatus" - }, - "history": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1RevisionHistory" - } - }, - "observedAt": { - "$ref": "#/definitions/v1Time" - }, - "operationState": { - "$ref": "#/definitions/v1alpha1OperationState" - }, - "reconciledAt": { - "$ref": "#/definitions/v1Time" - }, - "resources": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceStatus" - } - }, - "sourceType": { - "type": "string" - }, - "summary": { - "$ref": "#/definitions/v1alpha1ApplicationSummary" - }, - "sync": { - "$ref": "#/definitions/v1alpha1SyncStatus" - } - } - }, - "v1alpha1ApplicationSummary": { - "type": "object", - "properties": { - "externalURLs": { - "description": "ExternalURLs holds all external URLs of application child resources.", - "type": "array", - "items": { - "type": "string" - } - }, - "images": { - "description": "Images holds all images of application child resources.", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "v1alpha1ApplicationTree": { - "type": "object", - "title": "ApplicationTree holds nodes which belongs to the application", - "properties": { - "nodes": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceNode" - } - } - } - }, - "v1alpha1ApplicationWatchEvent": { - "description": "ApplicationWatchEvent contains information about application change.", - "type": "object", - "properties": { - "application": { - "$ref": "#/definitions/v1alpha1Application" - }, - "type": { - "type": "string" - } - } - }, - "v1alpha1Cluster": { - "type": "object", - "title": "Cluster is the definition of a cluster resource", - "properties": { - "config": { - "$ref": "#/definitions/v1alpha1ClusterConfig" - }, - "connectionState": { - "$ref": "#/definitions/v1alpha1ConnectionState" - }, - "name": { - "type": "string", - "title": "Name of the cluster. If omitted, will use the server address" - }, - "server": { - "type": "string", - "title": "Server is the API server URL of the Kubernetes cluster" - } - } - }, - "v1alpha1ClusterConfig": { - "description": "ClusterConfig is the configuration attributes. This structure is subset of the go-client\nrest.Config with annotations added for marshalling.", - "type": "object", - "properties": { - "awsAuthConfig": { - "$ref": "#/definitions/v1alpha1AWSAuthConfig" - }, - "bearerToken": { - "description": "Server requires Bearer authentication. This client will not attempt to use\nrefresh tokens for an OAuth2 flow.\nTODO: demonstrate an OAuth2 compatible client.", - "type": "string" - }, - "password": { - "type": "string" - }, - "tlsClientConfig": { - "$ref": "#/definitions/v1alpha1TLSClientConfig" - }, - "username": { - "type": "string", - "title": "Server requires Basic authentication" - } - } - }, - "v1alpha1ClusterList": { - "description": "ClusterList is a collection of Clusters.", - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1Cluster" - } - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - } - } - }, - "v1alpha1ComparedTo": { - "type": "object", - "title": "ComparedTo contains application source and target which was used for resources comparison", - "properties": { - "destination": { - "$ref": "#/definitions/v1alpha1ApplicationDestination" - }, - "source": { - "$ref": "#/definitions/v1alpha1ApplicationSource" - } - } - }, - "v1alpha1ConnectionState": { - "type": "object", - "title": "ConnectionState contains information about remote resource connection state", - "properties": { - "attemptedAt": { - "$ref": "#/definitions/v1Time" - }, - "message": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "v1alpha1EnvEntry": { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "the name, usually uppercase" - }, - "value": { - "type": "string", - "title": "the value" - } - } - }, - "v1alpha1HealthStatus": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "v1alpha1HelmParameter": { - "type": "object", - "title": "HelmParameter is a parameter to a helm template", - "properties": { - "forceString": { - "type": "boolean", - "format": "boolean", - "title": "ForceString determines whether to tell Helm to interpret booleans and numbers as strings" - }, - "name": { - "type": "string", - "title": "Name is the name of the helm parameter" - }, - "value": { - "type": "string", - "title": "Value is the value for the helm parameter" - } - } - }, - "v1alpha1Info": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "v1alpha1InfoItem": { - "type": "object", - "title": "InfoItem contains human readable information about object", - "properties": { - "name": { - "description": "Name is a human readable title for this piece of information.", - "type": "string" - }, - "value": { - "description": "Value is human readable content.", - "type": "string" - } - } - }, - "v1alpha1JWTToken": { - "type": "object", - "title": "JWTToken holds the issuedAt and expiresAt values of a token", - "properties": { - "exp": { - "type": "string", - "format": "int64" - }, - "iat": { - "type": "string", - "format": "int64" - } - } - }, - "v1alpha1JsonnetVar": { - "type": "object", - "title": "JsonnetVar is a jsonnet variable", - "properties": { - "code": { - "type": "boolean", - "format": "boolean" - }, - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "v1alpha1KsonnetParameter": { - "type": "object", - "title": "KsonnetParameter is a ksonnet component parameter", - "properties": { - "component": { - "type": "string" - }, - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "v1alpha1KustomizeOptions": { - "type": "object", - "title": "KustomizeOptions are options for kustomize to use when building manifests", - "properties": { - "buildOptions": { - "type": "string", - "title": "BuildOptions is a string of build parameters to use when calling `kustomize build`" - } - } - }, - "v1alpha1Operation": { - "description": "Operation contains requested operation parameters.", - "type": "object", - "properties": { - "sync": { - "$ref": "#/definitions/v1alpha1SyncOperation" - } - } - }, - "v1alpha1OperationState": { - "description": "OperationState contains information about state of currently performing operation on application.", - "type": "object", - "properties": { - "finishedAt": { - "$ref": "#/definitions/v1Time" - }, - "message": { - "description": "Message hold any pertinent messages when attempting to perform operation (typically errors).", - "type": "string" - }, - "operation": { - "$ref": "#/definitions/v1alpha1Operation" - }, - "phase": { - "type": "string", - "title": "Phase is the current phase of the operation" - }, - "startedAt": { - "$ref": "#/definitions/v1Time" - }, - "syncResult": { - "$ref": "#/definitions/v1alpha1SyncOperationResult" - } - } - }, - "v1alpha1ProjectRole": { - "type": "object", - "title": "ProjectRole represents a role that has access to a project", - "properties": { - "description": { - "type": "string", - "title": "Description is a description of the role" - }, - "groups": { - "type": "array", - "title": "Groups are a list of OIDC group claims bound to this role", - "items": { - "type": "string" - } - }, - "jwtTokens": { - "type": "array", - "title": "JWTTokens are a list of generated JWT tokens bound to this role", - "items": { - "$ref": "#/definitions/v1alpha1JWTToken" - } - }, - "name": { - "type": "string", - "title": "Name is a name for this role" - }, - "policies": { - "type": "array", - "title": "Policies Stores a list of casbin formated strings that define access policies for the role in the project", - "items": { - "type": "string" - } - } - } - }, - "v1alpha1Repository": { - "type": "object", - "title": "Repository is a Git repository holding application configurations", - "properties": { - "connectionState": { - "$ref": "#/definitions/v1alpha1ConnectionState" - }, - "enableLfs": { - "type": "boolean", - "format": "boolean", - "title": "Whether git-lfs support should be enabled for this repo" - }, - "insecure": { - "type": "boolean", - "format": "boolean", - "title": "Whether the repo is insecure" - }, - "insecureIgnoreHostKey": { - "type": "boolean", - "format": "boolean", - "title": "InsecureIgnoreHostKey should not be used anymore, Insecure is favoured" - }, - "password": { - "type": "string", - "title": "Password for authenticating at the repo server" - }, - "repo": { - "type": "string", - "title": "URL of the repo" - }, - "sshPrivateKey": { - "type": "string", - "title": "SSH private key data for authenticating at the repo server" - }, - "tlsClientCertData": { - "type": "string", - "title": "TLS client cert data for authenticating at the repo server" - }, - "tlsClientCertKey": { - "type": "string", - "title": "TLS client cert key for authenticating at the repo server" - }, - "username": { - "type": "string", - "title": "Username for authenticating at the repo server" - } - } - }, - "v1alpha1RepositoryCertificate": { - "type": "object", - "title": "A RepositoryCertificate is either SSH known hosts entry or TLS certificate", - "properties": { - "certData": { - "type": "string", - "format": "byte", - "title": "Actual certificate data, protocol dependent" - }, - "certInfo": { - "type": "string", - "title": "Additional certificate info (e.g. SSH fingerprint, X509 CommonName)" - }, - "certSubType": { - "type": "string", - "title": "The sub type of the cert, i.e. \"ssh-rsa\"" - }, - "certType": { - "type": "string", - "title": "Type of certificate - currently \"https\" or \"ssh\"" - }, - "serverName": { - "type": "string", - "title": "Name of the server the certificate is intended for" - } - } - }, - "v1alpha1RepositoryCertificateList": { - "type": "object", - "title": "RepositoryCertificateList is a collection of RepositoryCertificates", - "properties": { - "items": { - "type": "array", - "title": "List of certificates to be processed", - "items": { - "$ref": "#/definitions/v1alpha1RepositoryCertificate" - } - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - } - } - }, - "v1alpha1RepositoryList": { - "description": "RepositoryList is a collection of Repositories.", - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1Repository" - } - }, - "metadata": { - "$ref": "#/definitions/v1ListMeta" - } - } - }, - "v1alpha1ResourceAction": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "params": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceActionParam" - } - } - } - }, - "v1alpha1ResourceActionParam": { - "type": "object", - "properties": { - "default": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "v1alpha1ResourceDiff": { - "type": "object", - "title": "ResourceDiff holds the diff of a live and target resource object", - "properties": { - "diff": { - "type": "string" - }, - "group": { - "type": "string" - }, - "hook": { - "type": "boolean", - "format": "boolean" - }, - "kind": { - "type": "string" - }, - "liveState": { - "type": "string" - }, - "name": { - "type": "string" - }, - "namespace": { - "type": "string" - }, - "targetState": { - "type": "string" - } - } - }, - "v1alpha1ResourceIgnoreDifferences": { - "description": "ResourceIgnoreDifferences contains resource filter and list of json paths which should be ignored during comparison with live state.", - "type": "object", - "properties": { - "group": { - "type": "string" - }, - "jsonPointers": { - "type": "array", - "items": { - "type": "string" - } - }, - "kind": { - "type": "string" - }, - "name": { - "type": "string" - }, - "namespace": { - "type": "string" - } - } - }, - "v1alpha1ResourceNetworkingInfo": { - "type": "object", - "title": "ResourceNetworkingInfo holds networking resource related information", - "properties": { - "externalURLs": { - "description": "ExternalURLs holds list of URLs which should be available externally. List is populated for ingress resources using rules hostnames.", - "type": "array", - "items": { - "type": "string" - } - }, - "ingress": { - "type": "array", - "items": { - "$ref": "#/definitions/v1LoadBalancerIngress" - } - }, - "labels": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "targetLabels": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "targetRefs": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceRef" - } - } - } - }, - "v1alpha1ResourceNode": { - "type": "object", - "title": "ResourceNode contains information about live resource and its children", - "properties": { - "health": { - "$ref": "#/definitions/v1alpha1HealthStatus" - }, - "images": { - "type": "array", - "items": { - "type": "string" - } - }, - "info": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1InfoItem" - } - }, - "networkingInfo": { - "$ref": "#/definitions/v1alpha1ResourceNetworkingInfo" - }, - "parentRefs": { - "type": "array", - "items": { - "$ref": "#/definitions/v1alpha1ResourceRef" - } - }, - "resourceRef": { - "$ref": "#/definitions/v1alpha1ResourceRef" - }, - "resourceVersion": { - "type": "string" - } - } - }, - "v1alpha1ResourceOverride": { - "type": "object", - "title": "ResourceOverride holds configuration to customize resource diffing and health assessment", - "properties": { - "actions": { - "type": "string" - }, - "healthLua": { - "type": "string" - }, - "ignoreDifferences": { - "type": "string" - } - } - }, - "v1alpha1ResourceRef": { - "type": "object", - "title": "ResourceRef includes fields which unique identify resource", - "properties": { - "group": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "name": { - "type": "string" - }, - "namespace": { - "type": "string" - }, - "uid": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, - "v1alpha1ResourceResult": { - "type": "object", - "title": "ResourceResult holds the operation result details of a specific resource", - "properties": { - "group": { - "type": "string" - }, - "hookPhase": { - "type": "string", - "title": "the state of any operation associated with this resource OR hook\nnote: can contain values for non-hook resources" - }, - "hookType": { - "type": "string", - "title": "the type of the hook, empty for non-hook resources" - }, - "kind": { - "type": "string" - }, - "message": { - "type": "string", - "title": "message for the last sync OR operation" - }, - "name": { - "type": "string" - }, - "namespace": { - "type": "string" - }, - "status": { - "type": "string", - "title": "the final result of the sync, this is be empty if the resources is yet to be applied/pruned and is always zero-value for hooks" - }, - "syncPhase": { - "type": "string", - "title": "indicates the particular phase of the sync that this is for" - }, - "version": { - "type": "string" - } - } - }, - "v1alpha1ResourceStatus": { - "type": "object", - "title": "ResourceStatus holds the current sync and health status of a resource", - "properties": { - "group": { - "type": "string" - }, - "health": { - "$ref": "#/definitions/v1alpha1HealthStatus" - }, - "hook": { - "type": "boolean", - "format": "boolean" - }, - "kind": { - "type": "string" - }, - "name": { - "type": "string" - }, - "namespace": { - "type": "string" - }, - "requiresPruning": { - "type": "boolean", - "format": "boolean" - }, - "status": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, - "v1alpha1RevisionHistory": { - "type": "object", - "title": "RevisionHistory contains information relevant to an application deployment", - "properties": { - "deployedAt": { - "$ref": "#/definitions/v1Time" - }, - "id": { - "type": "string", - "format": "int64" - }, - "revision": { - "type": "string" - }, - "source": { - "$ref": "#/definitions/v1alpha1ApplicationSource" - } - } - }, - "v1alpha1RevisionMetadata": { - "type": "object", - "title": "data about a specific revision within a repo", - "properties": { - "author": { - "type": "string", - "title": "who authored this revision,\ntypically their name and email, e.g. \"John Doe \",\nbut might not match this example" - }, - "date": { - "$ref": "#/definitions/v1Time" - }, - "message": { - "type": "string", - "title": "the message associated with the revision,\nprobably the commit message,\nthis is truncated to the first newline or 64 characters (which ever comes first)" - }, - "tags": { - "type": "array", - "title": "tags on the revision,\nnote - tags can move from one revision to another", - "items": { - "type": "string" - } - } - } - }, - "v1alpha1SyncOperation": { - "description": "SyncOperation contains sync operation details.", - "type": "object", - "properties": { - "dryRun": { - "type": "boolean", - "format": "boolean", - "title": "DryRun will perform a `kubectl apply --dry-run` without actually performing the sync" - }, - "manifests": { - "type": "array", - "title": "Manifests is an optional field that overrides sync source with a local directory for development", - "items": { - "type": "string" - } - }, - "prune": { - "type": "boolean", - "format": "boolean", - "title": "Prune deletes resources that are no longer tracked in git" - }, - "resources": { - "type": "array", - "title": "Resources describes which resources to sync", - "items": { - "$ref": "#/definitions/v1alpha1SyncOperationResource" - } - }, - "revision": { - "description": "Revision is the git revision in which to sync the application to.\nIf omitted, will use the revision specified in app spec.", - "type": "string" - }, - "source": { - "$ref": "#/definitions/v1alpha1ApplicationSource" - }, - "syncStrategy": { - "$ref": "#/definitions/v1alpha1SyncStrategy" - } - } - }, - "v1alpha1SyncOperationResource": { - "description": "SyncOperationResource contains resources to sync.", - "type": "object", - "properties": { - "group": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "name": { - "type": "string" - } - } - }, - "v1alpha1SyncOperationResult": { - "type": "object", - "title": "SyncOperationResult represent result of sync operation", - "properties": { - "resources": { - "type": "array", - "title": "Resources holds the sync result of each individual resource", - "items": { - "$ref": "#/definitions/v1alpha1ResourceResult" - } - }, - "revision": { - "type": "string", - "title": "Revision holds the git commit SHA of the sync" - }, - "source": { - "$ref": "#/definitions/v1alpha1ApplicationSource" - } - } - }, - "v1alpha1SyncPolicy": { - "type": "object", - "title": "SyncPolicy controls when a sync will be performed in response to updates in git", - "properties": { - "automated": { - "$ref": "#/definitions/v1alpha1SyncPolicyAutomated" - } - } - }, - "v1alpha1SyncPolicyAutomated": { - "type": "object", - "title": "SyncPolicyAutomated controls the behavior of an automated sync", - "properties": { - "prune": { - "type": "boolean", - "format": "boolean", - "title": "Prune will prune resources automatically as part of automated sync (default: false)" - }, - "selfHeal": { - "type": "boolean", - "format": "boolean", - "title": "SelfHeal enables auto-syncing if (default: false)" - } - } - }, - "v1alpha1SyncStatus": { - "description": "SyncStatus is a comparison result of application spec and deployed application.", - "type": "object", - "properties": { - "comparedTo": { - "$ref": "#/definitions/v1alpha1ComparedTo" - }, - "revision": { - "type": "string" - }, - "status": { - "type": "string" - } - } - }, - "v1alpha1SyncStrategy": { - "type": "object", - "title": "SyncStrategy controls the manner in which a sync is performed", - "properties": { - "apply": { - "$ref": "#/definitions/v1alpha1SyncStrategyApply" - }, - "hook": { - "$ref": "#/definitions/v1alpha1SyncStrategyHook" - } - } - }, - "v1alpha1SyncStrategyApply": { - "type": "object", - "title": "SyncStrategyApply uses `kubectl apply` to perform the apply", - "properties": { - "force": { - "description": "Force indicates whether or not to supply the --force flag to `kubectl apply`.\nThe --force flag deletes and re-create the resource, when PATCH encounters conflict and has\nretried for 5 times.", - "type": "boolean", - "format": "boolean" - } - } - }, - "v1alpha1SyncStrategyHook": { - "description": "SyncStrategyHook will perform a sync using hooks annotations.\nIf no hook annotation is specified falls back to `kubectl apply`.", - "type": "object", - "properties": { - "syncStrategyApply": { - "$ref": "#/definitions/v1alpha1SyncStrategyApply" - } - } - }, - "v1alpha1TLSClientConfig": { - "type": "object", - "title": "TLSClientConfig contains settings to enable transport layer security", - "properties": { - "caData": { - "type": "string", - "format": "byte", - "title": "CAData holds PEM-encoded bytes (typically read from a root certificates bundle).\nCAData takes precedence over CAFile" - }, - "certData": { - "type": "string", - "format": "byte", - "title": "CertData holds PEM-encoded bytes (typically read from a client certificate file).\nCertData takes precedence over CertFile" - }, - "insecure": { - "description": "Server should be accessed without verifying the TLS certificate. For testing only.", - "type": "boolean", - "format": "boolean" - }, - "keyData": { - "type": "string", - "format": "byte", - "title": "KeyData holds PEM-encoded bytes (typically read from a client certificate key file).\nKeyData takes precedence over KeyFile" - }, - "serverName": { - "description": "ServerName is passed to the server for SNI and is used in the client to check server\ncertificates against. If ServerName is empty, the hostname used to contact the\nserver is used.", - "type": "string" - } - } - }, - "versionVersionMessage": { - "type": "object", - "title": "VersionMessage represents version of the Argo CD API server", - "properties": { - "BuildDate": { - "type": "string" - }, - "Compiler": { - "type": "string" - }, - "GitCommit": { - "type": "string" - }, - "GitTag": { - "type": "string" - }, - "GitTreeState": { - "type": "string" - }, - "GoVersion": { - "type": "string" - }, - "KsonnetVersion": { - "type": "string" - }, - "Platform": { - "type": "string" - }, - "Version": { - "type": "string" - } - } - } - } -} \ No newline at end of file From b09e80f31f0d663d85ccc49bb56e0e629e93fc76 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 21 Feb 2023 19:39:26 +0530 Subject: [PATCH 033/118] Updated the api specs --- api/bean/AppView.go | 9 +++-- api/restHandler/AppListingRestHandler.go | 20 ++++++++++ api/router/AppListingRouter.go | 4 ++ .../sql/repository/AppListingRepository.go | 13 ++++++ internal/sql/repository/app/AppRepository.go | 9 +++-- .../AppListingRepositoryQueryBuilder.go | 9 +++++ .../pipelineConfig/CiTemplateRepository.go | 2 +- pkg/app/AppListingService.go | 40 +++++++++++++------ .../DockerRegistryIpsConfigService.go | 6 +-- pkg/pipeline/PipelineBuilder.go | 4 +- .../repository/ciTemplateHistoryRepository.go | 2 +- 11 files changed, 90 insertions(+), 28 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 819f071dae..e4da0e82d2 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -66,10 +66,11 @@ type JobContainer struct { } type JobCIPipeline struct { - CiPipelineId int `json:"ciPipelineId"` - Status string `json:"status"` - LastRunAt time.Time `json:"lastRunAt"` - LastSuccessAt time.Time `json:"lastSuccessAt"` + CiPipelineId int `json:"ciPipelineId"` + CiPipelineName string `json:"ciPipelineName"` + Status string `json:"status"` + LastRunAt time.Time `json:"lastRunAt"` + LastSuccessAt time.Time `json:"lastSuccessAt"` } type JobListingContainer struct { diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 6cf506681b..a0d336ee33 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -57,6 +57,7 @@ import ( type AppListingRestHandler interface { FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) FetchJobs(w http.ResponseWriter, r *http.Request) + FetchOverviewCiPipeline(w http.ResponseWriter, r *http.Request) FetchAppDetails(w http.ResponseWriter, r *http.Request) FetchAllDevtronManagedApps(w http.ResponseWriter, r *http.Request) FetchAppTriggerView(w http.ResponseWriter, r *http.Request) @@ -196,6 +197,25 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) } +func (handler AppListingRestHandlerImpl) FetchOverviewCiPipeline(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + + vars := mux.Vars(r) + appId, err := strconv.Atoi(vars["appId"]) + if err != nil { + handler.logger.Errorw("request err, GetAppMetaInfo", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + + jobCi, err := handler.appListingService.FetchOverviewCiPipeline(appId) + + common.WriteJsonResp(w, err, jobCi, http.StatusOK) +} func (handler AppListingRestHandlerImpl) FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) { //Allow CORS here By * or specific origin setupResponse(&w, r) diff --git a/api/router/AppListingRouter.go b/api/router/AppListingRouter.go index 0f51031911..b97ff50612 100644 --- a/api/router/AppListingRouter.go +++ b/api/router/AppListingRouter.go @@ -52,6 +52,10 @@ func (router AppListingRouterImpl) initAppListingRouter(appListingRouter *mux.Ro HandlerFunc(router.appListingRestHandler.FetchJobs). Methods("POST") + appListingRouter.Path("/jobs/list/{appId}"). + HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipeline). + Methods("GET") + //This API used for fetch app details, not deployment details appListingRouter.Path("/detail").Queries("app-id", "{app-id}").Queries("env-id", "{env-id}"). HandlerFunc(router.appListingRestHandler.FetchAppDetails). diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index ddd2589ed8..43188c7420 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -38,6 +38,7 @@ import ( type AppListingRepository interface { FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) + FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) FetchJobsLastSucceededOn(ciPipelineIDs []int) ([]*bean.CiPipelineLastSucceededTime, error) DeploymentDetailsByAppIdAndEnvId(ctx context.Context, appId int, envId int) (bean.DeploymentDetailContainer, error) FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -97,6 +98,18 @@ func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string) } return jobContainers, nil } +func (impl AppListingRepositoryImpl) FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) { + var jobContainers []*bean.JobListingContainer + jobsQuery := impl.appListingRepositoryQueryBuilder.OverviewCiPipelineQuery() + impl.Logger.Debugw("basic app detail query: ", jobsQuery) + _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, appId) + if appsErr != nil { + impl.Logger.Error(appsErr) + return jobContainers, appsErr + } + return jobContainers, nil +} + func (impl AppListingRepositoryImpl) FetchJobsLastSucceededOn(CiPipelineIDs []int) ([]*bean.CiPipelineLastSucceededTime, error) { var lastSucceededTimeArray []*bean.CiPipelineLastSucceededTime if len(CiPipelineIDs) == 0 { diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 083376ea01..aeeda5af12 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -358,15 +358,16 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL Id int `json:"id"` } var appIds []AppId + whereCondition := " where display_name like ? and active = true and app_store = 2 " + if len(jobListingFilter.Teams) > 0 { + whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" + } + orderByCondition := " order by display_name " if jobListingFilter.SortOrder == "DESC" { orderByCondition += string(jobListingFilter.SortOrder) } orderByCondition += " limit ? offset ? " - whereCondition := " where display_name like ? and active = true and app_store = 2 " - if len(jobListingFilter.Teams) > 0 { - whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" - } query := "select id " + "from app " + whereCondition + orderByCondition appName := "%" + jobListingFilter.AppNameSearch + "%" diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 94fbfb4b8d..1edd621176 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -76,6 +76,15 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, } return query } +func (impl AppListingRepositoryQueryBuilder) OverviewCiPipelineQuery() string { + query := "select ci_pipeline.id as ci_pipeline_id,ci_pipeline.name " + + "as ci_pipeline_name,cwr.status,cwr.started_on from ci_pipeline" + + " left join (select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw" + + " inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id)" + + " cws on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id)" + + " cwr on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true where ci_pipeline.app_id = ? ;" + return query +} // use this query with atleast 1 cipipeline id func (impl AppListingRepositoryQueryBuilder) JobsLastSucceededOnTimeQuery(ciPipelineIDs []int) string { diff --git a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go index 012512965f..cc36ba47f1 100644 --- a/internal/sql/repository/pipelineConfig/CiTemplateRepository.go +++ b/internal/sql/repository/pipelineConfig/CiTemplateRepository.go @@ -30,7 +30,7 @@ type CiTemplate struct { tableName struct{} `sql:"ci_template" pg:",discard_unknown_columns"` Id int `sql:"id"` AppId int `sql:"app_id"` //foreign key of app - DockerRegistryId string `sql:"docker_registry_id"` //foreign key of registry + DockerRegistryId *string `sql:"docker_registry_id"` //foreign key of registry DockerRepository string `sql:"docker_repository"` DockerfilePath string `sql:"dockerfile_path"` Args string `sql:"args"` //json string format of map[string]string diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 95337c7040..7e7aa795b3 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -54,6 +54,7 @@ import ( type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) + FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) FetchAppDetails(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -247,6 +248,16 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi jobContainers := BuildJobListingResponse(jobListingContainers, JobsLastSucceededOnTime) return jobContainers, nil } + +func (impl AppListingServiceImpl) FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) { + jobCiContainers, err := impl.appListingRepository.FetchOverviewCiPipeline(appId) + if err != nil { + impl.Logger.Errorw("error in fetching app list", "error", err) + return []*bean.JobListingContainer{}, err + } + return jobCiContainers, nil +} + func (impl AppListingServiceImpl) FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) { impl.Logger.Debug("reached at FetchAppsByEnvironment:") // TODO: check statuses @@ -395,17 +406,20 @@ func BuildJobListingResponse(jobContainers []*bean.JobListingContainer, JobsLast val.JobCiPipelines = make([]bean.JobCIPipeline, 0) } - ciPipelineObj := bean.JobCIPipeline{ - CiPipelineId: jobContainer.CiPipelineID, - Status: jobContainer.Status, - LastRunAt: jobContainer.StartedOn, - //LastSuccessAt: jobContainer.LastSuccessAt, - } - if lastSuccessAt, ok := lastSucceededTimeMapping[jobContainer.CiPipelineID]; ok { - ciPipelineObj.LastSuccessAt = lastSuccessAt - } + if jobContainer.CiPipelineID != 0 { + ciPipelineObj := bean.JobCIPipeline{ + CiPipelineId: jobContainer.CiPipelineID, + CiPipelineName: jobContainer.CiPipelineName, + Status: jobContainer.Status, + LastRunAt: jobContainer.StartedOn, + //LastSuccessAt: jobContainer.LastSuccessAt, + } + if lastSuccessAt, ok := lastSucceededTimeMapping[jobContainer.CiPipelineID]; ok { + ciPipelineObj.LastSuccessAt = lastSuccessAt + } - val.JobCiPipelines = append(val.JobCiPipelines, ciPipelineObj) + val.JobCiPipelines = append(val.JobCiPipelines, ciPipelineObj) + } jobContainersMapping[jobContainer.AppId] = val } @@ -754,16 +768,16 @@ func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, a return bean.AppDetailContainer{}, err } - if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(ciPipeline.CiTemplate.DockerRegistryId) > 0 { + if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(*ciPipeline.CiTemplate.DockerRegistryId) > 0 { dockerRegistryId := ciPipeline.CiTemplate.DockerRegistryId - appDetailContainer.DockerRegistryId = dockerRegistryId + appDetailContainer.DockerRegistryId = *dockerRegistryId if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 { appDetailContainer.IsExternalCi = false } _, span = otel.Tracer("orchestrator").Start(ctx, "dockerRegistryIpsConfigService.IsImagePullSecretAccessProvided") // check ips access provided to this docker registry for that cluster - ipsAccessProvided, err := impl.dockerRegistryIpsConfigService.IsImagePullSecretAccessProvided(dockerRegistryId, clusterId) + ipsAccessProvided, err := impl.dockerRegistryIpsConfigService.IsImagePullSecretAccessProvided(*dockerRegistryId, clusterId) span.End() if err != nil { impl.Logger.Errorw("error in checking if docker registry ips access provided", "dockerRegistryId", dockerRegistryId, "clusterId", clusterId, "error", err) diff --git a/pkg/dockerRegistry/DockerRegistryIpsConfigService.go b/pkg/dockerRegistry/DockerRegistryIpsConfigService.go index a7bd18b1e3..3a028b6900 100644 --- a/pkg/dockerRegistry/DockerRegistryIpsConfigService.go +++ b/pkg/dockerRegistry/DockerRegistryIpsConfigService.go @@ -105,12 +105,12 @@ func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicatio } dockerRegistryId := ciPipeline.CiTemplate.DockerRegistryId - if len(dockerRegistryId) == 0 { + if len(*dockerRegistryId) == 0 { impl.logger.Warn("returning as dockerRegistryId is found empty") return valuesFileContent, nil } - dockerRegistryBean, err := impl.dockerArtifactStoreRepository.FindOne(dockerRegistryId) + dockerRegistryBean, err := impl.dockerArtifactStoreRepository.FindOne(*dockerRegistryId) if err != nil { impl.logger.Errorw("error in getting docker registry", "dockerRegistryId", dockerRegistryId, "error", err) if err == pg.ErrNoRows { @@ -134,7 +134,7 @@ func (impl DockerRegistryIpsConfigServiceImpl) HandleImagePullSecretOnApplicatio } ipsCredentialType := string(ipsConfig.CredentialType) - ipsName := BuildIpsName(dockerRegistryId, ipsCredentialType, ipsConfig.CredentialValue) + ipsName := BuildIpsName(*dockerRegistryId, ipsCredentialType, ipsConfig.CredentialValue) // Create or update secret of credential type is not of NAME type if ipsCredentialType != IPS_CREDENTIAL_TYPE_NAME { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 41bb6a4e06..a43aadc163 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -912,7 +912,7 @@ func (impl PipelineBuilderImpl) UpdateCiTemplate(updateRequest *bean.CiConfigReq Version: originalCiConf.Version, Id: originalCiConf.Id, DockerRepository: originalCiConf.DockerRepository, - DockerRegistryId: originalCiConf.DockerRegistry, + DockerRegistryId: &originalCiConf.DockerRegistry, Active: true, AuditLog: sql.AuditLog{ CreatedOn: originalCiConf.CreatedOn, @@ -1018,7 +1018,7 @@ func (impl PipelineBuilderImpl) CreateCiPipeline(createRequest *bean.CiConfigReq AuditLog: sql.AuditLog{CreatedOn: time.Now(), UpdatedOn: time.Now(), CreatedBy: createRequest.UserId, UpdatedBy: createRequest.UserId}, } if !createRequest.IsJob { - ciTemplate.DockerRegistryId = createRequest.DockerRegistry + ciTemplate.DockerRegistryId = &createRequest.DockerRegistry ciTemplate.DockerRepository = createRequest.DockerRepository } diff --git a/pkg/pipeline/history/repository/ciTemplateHistoryRepository.go b/pkg/pipeline/history/repository/ciTemplateHistoryRepository.go index 07f53b3676..aafc0aed9a 100644 --- a/pkg/pipeline/history/repository/ciTemplateHistoryRepository.go +++ b/pkg/pipeline/history/repository/ciTemplateHistoryRepository.go @@ -13,7 +13,7 @@ type CiTemplateHistory struct { Id int `sql:"id,pk"` CiTemplateId int `sql:"ci_template_id"` AppId int `sql:"app_id"` //foreign key of app - DockerRegistryId string `sql:"docker_registry_id"` //foreign key of registry + DockerRegistryId *string `sql:"docker_registry_id"` //foreign key of registry DockerRepository string `sql:"docker_repository"` DockerfilePath string `sql:"dockerfile_path"` Args string `sql:"args"` //json string format of map[string]string From db636816e8ed04d04194053426359bb3d5f0bc47 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 21 Feb 2023 20:46:38 +0530 Subject: [PATCH 034/118] Updated the api specs --- scripts/sql/115_jobs.down.sql | 3 +++ scripts/sql/115_jobs.up.sql | 3 +++ scripts/sql/116_job_description.down.sql | 2 ++ scripts/sql/116_job_description.up.sql | 14 ++++++++++++++ 4 files changed, 22 insertions(+) create mode 100644 scripts/sql/115_jobs.down.sql create mode 100644 scripts/sql/115_jobs.up.sql create mode 100644 scripts/sql/116_job_description.down.sql create mode 100644 scripts/sql/116_job_description.up.sql diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/115_jobs.down.sql new file mode 100644 index 0000000000..02ab78bd81 --- /dev/null +++ b/scripts/sql/115_jobs.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; +ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=1 THEN true ELSE false end; +ALTER TABLE app ALTER COLUMN app_store SET DEFAULT FALSE; \ No newline at end of file diff --git a/scripts/sql/115_jobs.up.sql b/scripts/sql/115_jobs.up.sql new file mode 100644 index 0000000000..f97f801545 --- /dev/null +++ b/scripts/sql/115_jobs.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; +ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=true THEN 1 ELSE 0 end; +ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; \ No newline at end of file diff --git a/scripts/sql/116_job_description.down.sql b/scripts/sql/116_job_description.down.sql new file mode 100644 index 0000000000..3b8b4cf26d --- /dev/null +++ b/scripts/sql/116_job_description.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE app DROP COLUMN display_name; +ALTER TABLE app DROP COLUMN description; \ No newline at end of file diff --git a/scripts/sql/116_job_description.up.sql b/scripts/sql/116_job_description.up.sql new file mode 100644 index 0000000000..90a5d4bfc0 --- /dev/null +++ b/scripts/sql/116_job_description.up.sql @@ -0,0 +1,14 @@ +ALTER TABLE app ADD COLUMN display_name varchar(250); +ALTER TABLE app ADD COLUMN description text; + + + + + + + + + + + + From 2e9b3d10bd246f29bfb49682733b276add94aa0b Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 13:37:07 +0530 Subject: [PATCH 035/118] Copied the earlier deleted files --- .../argoproj/argo-cd/assets/badge.svg | 22 + .../argo-cd/assets/builtin-policy.csv | 34 + .../argoproj/argo-cd/assets/model.conf | 14 + .../argoproj/argo-cd/assets/swagger.json | 3887 +++++++++++++++++ 4 files changed, 3957 insertions(+) create mode 100644 vendor/github.com/argoproj/argo-cd/assets/badge.svg create mode 100644 vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv create mode 100644 vendor/github.com/argoproj/argo-cd/assets/model.conf create mode 100644 vendor/github.com/argoproj/argo-cd/assets/swagger.json diff --git a/vendor/github.com/argoproj/argo-cd/assets/badge.svg b/vendor/github.com/argoproj/argo-cd/assets/badge.svg new file mode 100644 index 0000000000..a3234cfdf5 --- /dev/null +++ b/vendor/github.com/argoproj/argo-cd/assets/badge.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv b/vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv new file mode 100644 index 0000000000..f74c5b8002 --- /dev/null +++ b/vendor/github.com/argoproj/argo-cd/assets/builtin-policy.csv @@ -0,0 +1,34 @@ +# Built-in policy which defines two roles: role:readonly and role:admin, +# and additionally assigns the admin user to the role:admin role. +# There are two policy formats: +# 1. Applications (which belong to a project): +# p, , , , / +# 2. All other resources: +# p, , , , + +p, role:readonly, applications, get, */*, allow +p, role:readonly, certificates, get, *, allow +p, role:readonly, clusters, get, *, allow +p, role:readonly, repositories, get, *, allow +p, role:readonly, projects, get, *, allow + +p, role:admin, applications, create, */*, allow +p, role:admin, applications, update, */*, allow +p, role:admin, applications, delete, */*, allow +p, role:admin, applications, sync, */*, allow +p, role:admin, applications, override, */*, allow +p, role:admin, certificates, create, *, allow +p, role:admin, certificates, update, *, allow +p, role:admin, certificates, delete, *, allow +p, role:admin, clusters, create, *, allow +p, role:admin, clusters, update, *, allow +p, role:admin, clusters, delete, *, allow +p, role:admin, repositories, create, *, allow +p, role:admin, repositories, update, *, allow +p, role:admin, repositories, delete, *, allow +p, role:admin, projects, create, *, allow +p, role:admin, projects, update, *, allow +p, role:admin, projects, delete, *, allow + +g, role:admin, role:readonly +g, admin, role:admin \ No newline at end of file diff --git a/vendor/github.com/argoproj/argo-cd/assets/model.conf b/vendor/github.com/argoproj/argo-cd/assets/model.conf new file mode 100644 index 0000000000..240a9180d3 --- /dev/null +++ b/vendor/github.com/argoproj/argo-cd/assets/model.conf @@ -0,0 +1,14 @@ +[request_definition] +r = sub, res, act, obj + +[policy_definition] +p = sub, res, act, obj, eft + +[role_definition] +g = _, _ + +[policy_effect] +e = some(where (p.eft == allow)) && !some(where (p.eft == deny)) + +[matchers] +m = g(r.sub, p.sub) && keyMatch(r.res, p.res) && keyMatch(r.act, p.act) && keyMatch(r.obj, p.obj) \ No newline at end of file diff --git a/vendor/github.com/argoproj/argo-cd/assets/swagger.json b/vendor/github.com/argoproj/argo-cd/assets/swagger.json new file mode 100644 index 0000000000..0ad53c18de --- /dev/null +++ b/vendor/github.com/argoproj/argo-cd/assets/swagger.json @@ -0,0 +1,3887 @@ +{ + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "schemes": [ + "http", + "https" + ], + "swagger": "2.0", + "info": { + "description": "Description of all APIs", + "title": "Consolidate Services", + "version": "version not set" + }, + "paths": { + "/api/v1/account/password": { + "put": { + "tags": [ + "AccountService" + ], + "summary": "UpdatePassword updates an account's password to a new value", + "operationId": "UpdatePassword", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/accountUpdatePasswordRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/accountUpdatePasswordResponse" + } + } + } + } + }, + "/api/v1/applications": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "List returns list of applications", + "operationId": "ListMixin6", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "query" + }, + { + "type": "string", + "name": "refresh", + "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "name": "project", + "in": "query" + }, + { + "type": "string", + "name": "resourceVersion", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1ApplicationList" + } + } + } + }, + "post": { + "tags": [ + "ApplicationService" + ], + "summary": "Create creates an application", + "operationId": "CreateMixin6", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + } + } + }, + "/api/v1/applications/{application.metadata.name}": { + "put": { + "tags": [ + "ApplicationService" + ], + "summary": "Update updates an application", + "operationId": "UpdateMixin6", + "parameters": [ + { + "type": "string", + "name": "application.metadata.name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + } + } + }, + "/api/v1/applications/{applicationName}/managed-resources": { + "get": { + "tags": [ + "ApplicationService" + ], + "operationId": "ManagedResources", + "parameters": [ + { + "type": "string", + "name": "applicationName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationManagedResourcesResponse" + } + } + } + } + }, + "/api/v1/applications/{applicationName}/resource-tree": { + "get": { + "tags": [ + "ApplicationService" + ], + "operationId": "ResourceTree", + "parameters": [ + { + "type": "string", + "name": "applicationName", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1ApplicationTree" + } + } + } + } + }, + "/api/v1/applications/{name}": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "Get returns an application by name", + "operationId": "GetMixin6", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "refresh", + "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "name": "project", + "in": "query" + }, + { + "type": "string", + "name": "resourceVersion", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + } + }, + "delete": { + "tags": [ + "ApplicationService" + ], + "summary": "Delete deletes an application", + "operationId": "DeleteMixin6", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationApplicationResponse" + } + } + } + }, + "patch": { + "tags": [ + "ApplicationService" + ], + "summary": "Patch patch an application", + "operationId": "Patch", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/applicationApplicationPatchRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + } + } + }, + "/api/v1/applications/{name}/events": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "ListResourceEvents returns a list of event resources", + "operationId": "ListResourceEvents", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "resourceNamespace", + "in": "query" + }, + { + "type": "string", + "name": "resourceName", + "in": "query" + }, + { + "type": "string", + "name": "resourceUID", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1EventList" + } + } + } + } + }, + "/api/v1/applications/{name}/manifests": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "GetManifests returns application manifests", + "operationId": "GetManifests", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "revision", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/repositoryManifestResponse" + } + } + } + } + }, + "/api/v1/applications/{name}/operation": { + "delete": { + "tags": [ + "ApplicationService" + ], + "summary": "TerminateOperation terminates the currently running operation", + "operationId": "TerminateOperation", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationOperationTerminateResponse" + } + } + } + } + }, + "/api/v1/applications/{name}/pods/{podName}/logs": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "PodLogs returns stream of log entries for the specified pod. Pod", + "operationId": "PodLogs", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "podName", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "namespace", + "in": "query" + }, + { + "type": "string", + "name": "container", + "in": "query" + }, + { + "type": "string", + "format": "int64", + "name": "sinceSeconds", + "in": "query" + }, + { + "type": "string", + "format": "int64", + "description": "Represents seconds of UTC time since Unix epoch\n1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to\n9999-12-31T23:59:59Z inclusive.", + "name": "sinceTime.seconds", + "in": "query" + }, + { + "type": "integer", + "format": "int32", + "description": "Non-negative fractions of a second at nanosecond resolution. Negative\nsecond values with fractions must still have non-negative nanos values\nthat count forward in time. Must be from 0 to 999,999,999\ninclusive. This field may be limited in precision depending on context.", + "name": "sinceTime.nanos", + "in": "query" + }, + { + "type": "string", + "format": "int64", + "name": "tailLines", + "in": "query" + }, + { + "type": "boolean", + "format": "boolean", + "name": "follow", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(streaming responses)", + "schema": { + "$ref": "#/definitions/applicationLogEntry" + } + } + } + } + }, + "/api/v1/applications/{name}/resource": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "GetResource returns single application resource", + "operationId": "GetResource", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "namespace", + "in": "query" + }, + { + "type": "string", + "name": "resourceName", + "in": "query" + }, + { + "type": "string", + "name": "version", + "in": "query" + }, + { + "type": "string", + "name": "group", + "in": "query" + }, + { + "type": "string", + "name": "kind", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationApplicationResourceResponse" + } + } + } + }, + "post": { + "tags": [ + "ApplicationService" + ], + "summary": "PatchResource patch single application resource", + "operationId": "PatchResource", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationApplicationResourceResponse" + } + } + } + }, + "delete": { + "tags": [ + "ApplicationService" + ], + "summary": "DeleteResource deletes a single application resource", + "operationId": "DeleteResource", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationApplicationResponse" + } + } + } + } + }, + "/api/v1/applications/{name}/resource/actions": { + "get": { + "tags": [ + "ApplicationService" + ], + "operationId": "ListResourceActions", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "namespace", + "in": "query" + }, + { + "type": "string", + "name": "resourceName", + "in": "query" + }, + { + "type": "string", + "name": "version", + "in": "query" + }, + { + "type": "string", + "name": "group", + "in": "query" + }, + { + "type": "string", + "name": "kind", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationResourceActionsListResponse" + } + } + } + }, + "post": { + "tags": [ + "ApplicationService" + ], + "operationId": "RunResourceAction", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/applicationApplicationResponse" + } + } + } + } + }, + "/api/v1/applications/{name}/revisions/{revision}/metadata": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "Get the meta-data (author, date, tags, message) for a specific revision of the application", + "operationId": "RevisionMetadata", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "revision", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1RevisionMetadata" + } + } + } + } + }, + "/api/v1/applications/{name}/rollback": { + "post": { + "tags": [ + "ApplicationService" + ], + "summary": "Rollback syncs an application to its target state", + "operationId": "Rollback", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/applicationApplicationRollbackRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + } + } + }, + "/api/v1/applications/{name}/spec": { + "put": { + "tags": [ + "ApplicationService" + ], + "summary": "UpdateSpec updates an application spec", + "operationId": "UpdateSpec", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1ApplicationSpec" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1ApplicationSpec" + } + } + } + } + }, + "/api/v1/applications/{name}/sync": { + "post": { + "tags": [ + "ApplicationService" + ], + "summary": "Sync syncs an application to its target state", + "operationId": "Sync", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/applicationApplicationSyncRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Application" + } + } + } + } + }, + "/api/v1/certificates": { + "get": { + "tags": [ + "CertificateService" + ], + "summary": "List all available repository certificates", + "operationId": "ListCertificates", + "parameters": [ + { + "type": "string", + "description": "A file-glob pattern (not regular expression) the host name has to match.", + "name": "hostNamePattern", + "in": "query" + }, + { + "type": "string", + "description": "The type of the certificate to match (ssh or https).", + "name": "certType", + "in": "query" + }, + { + "type": "string", + "description": "The sub type of the certificate to match (protocol dependent, usually only used for ssh certs).", + "name": "certSubType", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1RepositoryCertificateList" + } + } + } + }, + "post": { + "tags": [ + "CertificateService" + ], + "summary": "Creates repository certificates on the server", + "operationId": "CreateCertificate", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1RepositoryCertificateList" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1RepositoryCertificateList" + } + } + } + }, + "delete": { + "tags": [ + "CertificateService" + ], + "summary": "Delete the certificates that match the RepositoryCertificateQuery", + "operationId": "DeleteCertificate", + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1RepositoryCertificateList" + } + } + } + } + }, + "/api/v1/clusters": { + "get": { + "tags": [ + "ClusterService" + ], + "summary": "List returns list of clusters", + "operationId": "List", + "parameters": [ + { + "type": "string", + "name": "server", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1ClusterList" + } + } + } + }, + "post": { + "tags": [ + "ClusterService" + ], + "summary": "Create creates a cluster", + "operationId": "Create", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1Cluster" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Cluster" + } + } + } + } + }, + "/api/v1/clusters/{cluster.server}": { + "put": { + "tags": [ + "ClusterService" + ], + "summary": "Update updates a cluster", + "operationId": "Update", + "parameters": [ + { + "type": "string", + "name": "cluster.server", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1Cluster" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Cluster" + } + } + } + } + }, + "/api/v1/clusters/{server}": { + "get": { + "tags": [ + "ClusterService" + ], + "summary": "Get returns a cluster by server address", + "operationId": "GetMixin1", + "parameters": [ + { + "type": "string", + "name": "server", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Cluster" + } + } + } + }, + "delete": { + "tags": [ + "ClusterService" + ], + "summary": "Delete deletes a cluster", + "operationId": "Delete", + "parameters": [ + { + "type": "string", + "name": "server", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/clusterClusterResponse" + } + } + } + } + }, + "/api/v1/clusters/{server}/rotate-auth": { + "post": { + "tags": [ + "ClusterService" + ], + "summary": "RotateAuth returns a cluster by server address", + "operationId": "RotateAuth", + "parameters": [ + { + "type": "string", + "name": "server", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/clusterClusterResponse" + } + } + } + } + }, + "/api/v1/projects": { + "get": { + "tags": [ + "ProjectService" + ], + "summary": "List returns list of projects", + "operationId": "ListMixin4", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1AppProjectList" + } + } + } + }, + "post": { + "tags": [ + "ProjectService" + ], + "summary": "Create a new project.", + "operationId": "CreateMixin4", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/projectProjectCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1AppProject" + } + } + } + } + }, + "/api/v1/projects/{name}": { + "get": { + "tags": [ + "ProjectService" + ], + "summary": "Get returns a project by name", + "operationId": "GetMixin4", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1AppProject" + } + } + } + }, + "delete": { + "tags": [ + "ProjectService" + ], + "summary": "Delete deletes a project", + "operationId": "DeleteMixin4", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/projectEmptyResponse" + } + } + } + } + }, + "/api/v1/projects/{name}/events": { + "get": { + "tags": [ + "ProjectService" + ], + "summary": "ListEvents returns a list of project events", + "operationId": "ListEvents", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1EventList" + } + } + } + } + }, + "/api/v1/projects/{project.metadata.name}": { + "put": { + "tags": [ + "ProjectService" + ], + "summary": "Update updates a project", + "operationId": "UpdateMixin4", + "parameters": [ + { + "type": "string", + "name": "project.metadata.name", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/projectProjectUpdateRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1AppProject" + } + } + } + } + }, + "/api/v1/projects/{project}/roles/{role}/token": { + "post": { + "tags": [ + "ProjectService" + ], + "summary": "Create a new project token.", + "operationId": "CreateToken", + "parameters": [ + { + "type": "string", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "role", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/projectProjectTokenCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/projectProjectTokenResponse" + } + } + } + } + }, + "/api/v1/projects/{project}/roles/{role}/token/{iat}": { + "delete": { + "tags": [ + "ProjectService" + ], + "summary": "Delete a new project token.", + "operationId": "DeleteToken", + "parameters": [ + { + "type": "string", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "role", + "in": "path", + "required": true + }, + { + "type": "string", + "format": "int64", + "name": "iat", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/projectEmptyResponse" + } + } + } + } + }, + "/api/v1/repositories": { + "get": { + "tags": [ + "RepositoryService" + ], + "summary": "List returns list of repos", + "operationId": "ListMixin2", + "parameters": [ + { + "type": "string", + "name": "repo", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1RepositoryList" + } + } + } + }, + "post": { + "tags": [ + "RepositoryService" + ], + "summary": "Create creates a repo", + "operationId": "CreateMixin2", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1Repository" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Repository" + } + } + } + } + }, + "/api/v1/repositories/{repo.repo}": { + "put": { + "tags": [ + "RepositoryService" + ], + "summary": "Update updates a repo", + "operationId": "UpdateMixin2", + "parameters": [ + { + "type": "string", + "name": "repo.repo", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1alpha1Repository" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/v1alpha1Repository" + } + } + } + } + }, + "/api/v1/repositories/{repo}": { + "delete": { + "tags": [ + "RepositoryService" + ], + "summary": "Delete deletes a repo", + "operationId": "DeleteMixin2", + "parameters": [ + { + "type": "string", + "name": "repo", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/repositoryRepoResponse" + } + } + } + } + }, + "/api/v1/repositories/{repo}/apps": { + "get": { + "tags": [ + "RepositoryService" + ], + "summary": "ListApps returns list of apps in the repo", + "operationId": "ListApps", + "parameters": [ + { + "type": "string", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "revision", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/repositoryRepoAppsResponse" + } + } + } + } + }, + "/api/v1/repositories/{repo}/apps/{path}": { + "get": { + "tags": [ + "RepositoryService" + ], + "summary": "GetAppDetails returns application details by given path", + "operationId": "GetAppDetails", + "parameters": [ + { + "type": "string", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "path", + "in": "path", + "required": true + }, + { + "type": "string", + "name": "revision", + "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "name": "helm.valueFiles", + "in": "query" + }, + { + "type": "string", + "name": "ksonnet.environment", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/repositoryRepoAppDetailsResponse" + } + } + } + } + }, + "/api/v1/repositories/{repo}/validate": { + "post": { + "tags": [ + "RepositoryService" + ], + "summary": "ValidateAccess validates access to a repository with given parameters", + "operationId": "ValidateAccess", + "parameters": [ + { + "type": "string", + "name": "repo", + "in": "path", + "required": true + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/repositoryRepoResponse" + } + } + } + } + }, + "/api/v1/session": { + "post": { + "tags": [ + "SessionService" + ], + "summary": "Create a new JWT for authentication and set a cookie if using HTTP.", + "operationId": "CreateMixin8", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/sessionSessionCreateRequest" + } + } + ], + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/sessionSessionResponse" + } + } + } + }, + "delete": { + "tags": [ + "SessionService" + ], + "summary": "Delete an existing JWT cookie if using HTTP.", + "operationId": "DeleteMixin8", + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/sessionSessionResponse" + } + } + } + } + }, + "/api/v1/settings": { + "get": { + "tags": [ + "SettingsService" + ], + "summary": "Get returns Argo CD settings", + "operationId": "Get", + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/clusterSettings" + } + } + } + } + }, + "/api/v1/stream/applications": { + "get": { + "tags": [ + "ApplicationService" + ], + "summary": "Watch returns stream of application change events.", + "operationId": "Watch", + "parameters": [ + { + "type": "string", + "name": "name", + "in": "query" + }, + { + "type": "string", + "name": "refresh", + "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "name": "project", + "in": "query" + }, + { + "type": "string", + "name": "resourceVersion", + "in": "query" + } + ], + "responses": { + "200": { + "description": "(streaming responses)", + "schema": { + "$ref": "#/definitions/v1alpha1ApplicationWatchEvent" + } + } + } + } + }, + "/api/version": { + "get": { + "tags": [ + "VersionService" + ], + "summary": "Version returns version information of the API server", + "operationId": "Version", + "responses": { + "200": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/versionVersionMessage" + } + } + } + } + } + }, + "definitions": { + "accountUpdatePasswordRequest": { + "type": "object", + "properties": { + "currentPassword": { + "type": "string" + }, + "newPassword": { + "type": "string" + } + } + }, + "accountUpdatePasswordResponse": { + "type": "object" + }, + "applicationApplicationPatchRequest": { + "type": "object", + "title": "ApplicationPatchRequest is a request to patch an application", + "properties": { + "name": { + "type": "string" + }, + "patch": { + "type": "string" + }, + "patchType": { + "type": "string" + } + } + }, + "applicationApplicationResourceResponse": { + "type": "object", + "properties": { + "manifest": { + "type": "string" + } + } + }, + "applicationApplicationResponse": { + "type": "object" + }, + "applicationApplicationRollbackRequest": { + "type": "object", + "properties": { + "dryRun": { + "type": "boolean", + "format": "boolean" + }, + "id": { + "type": "string", + "format": "int64" + }, + "name": { + "type": "string" + }, + "prune": { + "type": "boolean", + "format": "boolean" + } + } + }, + "applicationApplicationSyncRequest": { + "type": "object", + "title": "ApplicationSyncRequest is a request to apply the config state to live state", + "properties": { + "dryRun": { + "type": "boolean", + "format": "boolean" + }, + "manifests": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "prune": { + "type": "boolean", + "format": "boolean" + }, + "resources": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1SyncOperationResource" + } + }, + "revision": { + "type": "string" + }, + "strategy": { + "$ref": "#/definitions/v1alpha1SyncStrategy" + } + } + }, + "applicationLogEntry": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "timeStamp": { + "$ref": "#/definitions/v1Time" + } + } + }, + "applicationManagedResourcesResponse": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceDiff" + } + } + } + }, + "applicationOperationTerminateResponse": { + "type": "object" + }, + "applicationResourceActionsListResponse": { + "type": "object", + "properties": { + "actions": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceAction" + } + } + } + }, + "clusterClusterResponse": { + "type": "object" + }, + "clusterConnector": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "clusterDexConfig": { + "type": "object", + "properties": { + "connectors": { + "type": "array", + "items": { + "$ref": "#/definitions/clusterConnector" + } + } + } + }, + "clusterGoogleAnalyticsConfig": { + "type": "object", + "properties": { + "anonymizeUsers": { + "type": "boolean", + "format": "boolean" + }, + "trackingID": { + "type": "string" + } + } + }, + "clusterHelp": { + "type": "object", + "title": "Help settings", + "properties": { + "chatText": { + "type": "string", + "title": "the text for getting chat help, defaults to \"Chat now!\"" + }, + "chatUrl": { + "type": "string", + "title": "the URL for getting chat help, this will typically be your Slack channel for support" + } + } + }, + "clusterOIDCConfig": { + "type": "object", + "properties": { + "cliClientID": { + "type": "string" + }, + "clientID": { + "type": "string" + }, + "issuer": { + "type": "string" + }, + "name": { + "type": "string" + }, + "scopes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "clusterSettings": { + "type": "object", + "properties": { + "appLabelKey": { + "type": "string" + }, + "dexConfig": { + "$ref": "#/definitions/clusterDexConfig" + }, + "googleAnalytics": { + "$ref": "#/definitions/clusterGoogleAnalyticsConfig" + }, + "help": { + "$ref": "#/definitions/clusterHelp" + }, + "kustomizeOptions": { + "$ref": "#/definitions/v1alpha1KustomizeOptions" + }, + "oidcConfig": { + "$ref": "#/definitions/clusterOIDCConfig" + }, + "resourceOverrides": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/v1alpha1ResourceOverride" + } + }, + "statusBadgeEnabled": { + "type": "boolean", + "format": "boolean" + }, + "url": { + "type": "string" + } + } + }, + "projectEmptyResponse": { + "type": "object" + }, + "projectProjectCreateRequest": { + "description": "ProjectCreateRequest defines project creation parameters.", + "type": "object", + "properties": { + "project": { + "$ref": "#/definitions/v1alpha1AppProject" + } + } + }, + "projectProjectTokenCreateRequest": { + "description": "ProjectTokenCreateRequest defines project token creation parameters.", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "expiresIn": { + "type": "string", + "format": "int64", + "title": "expiresIn represents a duration in seconds" + }, + "project": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "projectProjectTokenResponse": { + "description": "ProjectTokenResponse wraps the created token or returns an empty string if deleted.", + "type": "object", + "properties": { + "token": { + "type": "string" + } + } + }, + "projectProjectUpdateRequest": { + "type": "object", + "properties": { + "project": { + "$ref": "#/definitions/v1alpha1AppProject" + } + } + }, + "repositoryAppInfo": { + "type": "object", + "title": "AppInfo contains application type and app file path", + "properties": { + "path": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "repositoryDirectoryAppSpec": { + "type": "object", + "title": "DirectoryAppSpec contains directory" + }, + "repositoryHelmAppDetailsQuery": { + "type": "object", + "properties": { + "valueFiles": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "repositoryHelmAppSpec": { + "type": "object", + "title": "HelmAppSpec contains helm app name and path in source repo", + "properties": { + "name": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1HelmParameter" + } + }, + "path": { + "type": "string" + }, + "valueFiles": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "repositoryKsonnetAppDetailsQuery": { + "type": "object", + "properties": { + "environment": { + "type": "string" + } + } + }, + "repositoryKsonnetAppSpec": { + "type": "object", + "title": "KsonnetAppSpec contains Ksonnet app response\nThis roughly reflects: ksonnet/ksonnet/metadata/app/schema.go", + "properties": { + "environments": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/repositoryKsonnetEnvironment" + } + }, + "name": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1KsonnetParameter" + } + }, + "path": { + "type": "string" + } + } + }, + "repositoryKsonnetEnvironment": { + "type": "object", + "properties": { + "destination": { + "$ref": "#/definitions/repositoryKsonnetEnvironmentDestination" + }, + "k8sVersion": { + "description": "KubernetesVersion is the kubernetes version the targetted cluster is running on.", + "type": "string" + }, + "name": { + "type": "string", + "title": "Name is the user defined name of an environment" + }, + "path": { + "description": "Path is the relative project path containing metadata for this environment.", + "type": "string" + } + } + }, + "repositoryKsonnetEnvironmentDestination": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "title": "Namespace is the namespace of the Kubernetes server that targets should be deployed to" + }, + "server": { + "description": "Server is the Kubernetes server that the cluster is running on.", + "type": "string" + } + } + }, + "repositoryKustomizeAppSpec": { + "type": "object", + "title": "KustomizeAppSpec contains kustomize app name and path in source repo", + "properties": { + "images": { + "description": "images is a list of available images.", + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "type": "string" + } + } + }, + "repositoryManifestResponse": { + "type": "object", + "properties": { + "manifests": { + "type": "array", + "items": { + "type": "string" + } + }, + "namespace": { + "type": "string" + }, + "revision": { + "type": "string" + }, + "server": { + "type": "string" + }, + "sourceType": { + "type": "string" + } + } + }, + "repositoryRepoAppDetailsResponse": { + "type": "object", + "title": "RepoAppDetailsResponse application details", + "properties": { + "directory": { + "$ref": "#/definitions/repositoryDirectoryAppSpec" + }, + "helm": { + "$ref": "#/definitions/repositoryHelmAppSpec" + }, + "ksonnet": { + "$ref": "#/definitions/repositoryKsonnetAppSpec" + }, + "kustomize": { + "$ref": "#/definitions/repositoryKustomizeAppSpec" + }, + "type": { + "type": "string" + } + } + }, + "repositoryRepoAppsResponse": { + "type": "object", + "title": "RepoAppsResponse contains applications of specified repository", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/repositoryAppInfo" + } + } + } + }, + "repositoryRepoResponse": { + "type": "object" + }, + "sessionSessionCreateRequest": { + "description": "SessionCreateRequest is for logging in.", + "type": "object", + "properties": { + "password": { + "type": "string" + }, + "token": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "sessionSessionResponse": { + "description": "SessionResponse wraps the created token or returns an empty string if deleted.", + "type": "object", + "properties": { + "token": { + "type": "string" + } + } + }, + "v1Event": { + "description": "Event is a report of an event somewhere in the cluster.", + "type": "object", + "properties": { + "action": { + "type": "string", + "title": "What action was taken/failed regarding to the Regarding object.\n+optional" + }, + "count": { + "type": "integer", + "format": "int32", + "title": "The number of times this event has occurred.\n+optional" + }, + "eventTime": { + "$ref": "#/definitions/v1MicroTime" + }, + "firstTimestamp": { + "$ref": "#/definitions/v1Time" + }, + "involvedObject": { + "$ref": "#/definitions/v1ObjectReference" + }, + "lastTimestamp": { + "$ref": "#/definitions/v1Time" + }, + "message": { + "type": "string", + "title": "A human-readable description of the status of this operation.\nTODO: decide on maximum length.\n+optional" + }, + "metadata": { + "$ref": "#/definitions/v1ObjectMeta" + }, + "reason": { + "type": "string", + "title": "This should be a short, machine understandable string that gives the reason\nfor the transition into the object's current status.\nTODO: provide exact specification for format.\n+optional" + }, + "related": { + "$ref": "#/definitions/v1ObjectReference" + }, + "reportingComponent": { + "type": "string", + "title": "Name of the controller that emitted this Event, e.g. `kubernetes.io/kubelet`.\n+optional" + }, + "reportingInstance": { + "type": "string", + "title": "ID of the controller instance, e.g. `kubelet-xyzf`.\n+optional" + }, + "series": { + "$ref": "#/definitions/v1EventSeries" + }, + "source": { + "$ref": "#/definitions/v1EventSource" + }, + "type": { + "type": "string", + "title": "Type of this event (Normal, Warning), new types could be added in the future\n+optional" + } + } + }, + "v1EventList": { + "description": "EventList is a list of events.", + "type": "object", + "properties": { + "items": { + "type": "array", + "title": "List of events", + "items": { + "$ref": "#/definitions/v1Event" + } + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + } + } + }, + "v1EventSeries": { + "description": "EventSeries contain information on series of events, i.e. thing that was/is happening\ncontinuously for some time.", + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "int32", + "title": "Number of occurrences in this series up to the last heartbeat time" + }, + "lastObservedTime": { + "$ref": "#/definitions/v1MicroTime" + }, + "state": { + "type": "string", + "title": "State of this Series: Ongoing or Finished" + } + } + }, + "v1EventSource": { + "description": "EventSource contains information for an event.", + "type": "object", + "properties": { + "component": { + "type": "string", + "title": "Component from which the event is generated.\n+optional" + }, + "host": { + "type": "string", + "title": "Node name on which the event is generated.\n+optional" + } + } + }, + "v1Fields": { + "type": "object", + "title": "Fields stores a set of fields in a data structure like a Trie.\nTo understand how this is used, see: https://github.com/kubernetes-sigs/structured-merge-diff", + "properties": { + "map": { + "description": "Map stores a set of fields in a data structure like a Trie.\n\nEach key is either a '.' representing the field itself, and will always map to an empty set,\nor a string representing a sub-field or item. The string will follow one of these four formats:\n'f:', where is the name of a field in a struct, or key in a map\n'v:', where is the exact json formatted value of a list item\n'i:', where is position of a item in a list\n'k:', where is a map of a list item's key fields to their unique values\nIf a key maps to an empty Fields value, the field that key represents is part of the set.\n\nThe exact format is defined in k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/v1Fields" + } + } + } + }, + "v1GroupKind": { + "description": "+protobuf.options.(gogoproto.goproto_stringer)=false", + "type": "object", + "title": "GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying\nconcepts during lookup stages without having partially valid types", + "properties": { + "group": { + "type": "string" + }, + "kind": { + "type": "string" + } + } + }, + "v1Initializer": { + "description": "Initializer is information about an initializer that has not yet completed.", + "type": "object", + "properties": { + "name": { + "description": "name of the process that is responsible for initializing this object.", + "type": "string" + } + } + }, + "v1Initializers": { + "description": "Initializers tracks the progress of initialization.", + "type": "object", + "properties": { + "pending": { + "type": "array", + "title": "Pending is a list of initializers that must execute in order before this object is visible.\nWhen the last pending initializer is removed, and no failing result is set, the initializers\nstruct will be set to nil and the object is considered as initialized and visible to all\nclients.\n+patchMergeKey=name\n+patchStrategy=merge", + "items": { + "$ref": "#/definitions/v1Initializer" + } + }, + "result": { + "$ref": "#/definitions/v1Status" + } + } + }, + "v1ListMeta": { + "description": "ListMeta describes metadata that synthetic resources must have, including lists and\nvarious status objects. A resource may have only one of {ObjectMeta, ListMeta}.", + "type": "object", + "properties": { + "continue": { + "description": "continue may be set if the user set a limit on the number of items returned, and indicates that\nthe server has more data available. The value is opaque and may be used to issue another request\nto the endpoint that served this list to retrieve the next set of available objects. Continuing a\nconsistent list may not be possible if the server configuration has changed or more than a few\nminutes have passed. The resourceVersion field returned when using this continue value will be\nidentical to the value in the first response, unless you have received this token from an error\nmessage.", + "type": "string" + }, + "resourceVersion": { + "type": "string", + "title": "String that identifies the server's internal version of this object that\ncan be used by clients to determine when objects have changed.\nValue must be treated as opaque by clients and passed unmodified back to the server.\nPopulated by the system.\nRead-only.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\n+optional" + }, + "selfLink": { + "type": "string", + "title": "selfLink is a URL representing this object.\nPopulated by the system.\nRead-only.\n+optional" + } + } + }, + "v1LoadBalancerIngress": { + "description": "LoadBalancerIngress represents the status of a load-balancer ingress point:\ntraffic intended for the service should be sent to an ingress point.", + "type": "object", + "properties": { + "hostname": { + "type": "string", + "title": "Hostname is set for load-balancer ingress points that are DNS based\n(typically AWS load-balancers)\n+optional" + }, + "ip": { + "type": "string", + "title": "IP is set for load-balancer ingress points that are IP based\n(typically GCE or OpenStack load-balancers)\n+optional" + } + } + }, + "v1ManagedFieldsEntry": { + "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource\nthat the fieldset applies to.", + "type": "object", + "properties": { + "apiVersion": { + "description": "APIVersion defines the version of this resource that this field set\napplies to. The format is \"group/version\" just like the top-level\nAPIVersion field. It is necessary to track the version of a field\nset because it cannot be automatically converted.", + "type": "string" + }, + "fields": { + "$ref": "#/definitions/v1Fields" + }, + "manager": { + "description": "Manager is an identifier of the workflow managing these fields.", + "type": "string" + }, + "operation": { + "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created.\nThe only valid values for this field are 'Apply' and 'Update'.", + "type": "string" + }, + "time": { + "$ref": "#/definitions/v1Time" + } + } + }, + "v1MicroTime": { + "description": "MicroTime is version of Time with microsecond level precision.\n\n+protobuf.options.marshal=false\n+protobuf.as=Timestamp\n+protobuf.options.(gogoproto.goproto_stringer)=false", + "type": "object", + "properties": { + "nanos": { + "description": "Non-negative fractions of a second at nanosecond resolution. Negative\nsecond values with fractions must still have non-negative nanos values\nthat count forward in time. Must be from 0 to 999,999,999\ninclusive. This field may be limited in precision depending on context.", + "type": "integer", + "format": "int32" + }, + "seconds": { + "description": "Represents seconds of UTC time since Unix epoch\n1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to\n9999-12-31T23:59:59Z inclusive.", + "type": "string", + "format": "int64" + } + } + }, + "v1ObjectMeta": { + "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects\nusers must create.", + "type": "object", + "properties": { + "annotations": { + "type": "object", + "title": "Annotations is an unstructured key value map stored with a resource that may be\nset by external tools to store and retrieve arbitrary metadata. They are not\nqueryable and should be preserved when modifying objects.\nMore info: http://kubernetes.io/docs/user-guide/annotations\n+optional", + "additionalProperties": { + "type": "string" + } + }, + "clusterName": { + "type": "string", + "title": "The name of the cluster which the object belongs to.\nThis is used to distinguish resources with same name and namespace in different clusters.\nThis field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.\n+optional" + }, + "creationTimestamp": { + "$ref": "#/definitions/v1Time" + }, + "deletionGracePeriodSeconds": { + "type": "string", + "format": "int64", + "title": "Number of seconds allowed for this object to gracefully terminate before\nit will be removed from the system. Only set when deletionTimestamp is also set.\nMay only be shortened.\nRead-only.\n+optional" + }, + "deletionTimestamp": { + "$ref": "#/definitions/v1Time" + }, + "finalizers": { + "type": "array", + "title": "Must be empty before the object is deleted from the registry. Each entry\nis an identifier for the responsible component that will remove the entry\nfrom the list. If the deletionTimestamp of the object is non-nil, entries\nin this list can only be removed.\n+optional\n+patchStrategy=merge", + "items": { + "type": "string" + } + }, + "generateName": { + "description": "GenerateName is an optional prefix, used by the server, to generate a unique\nname ONLY IF the Name field has not been provided.\nIf this field is used, the name returned to the client will be different\nthan the name passed. This value will also be combined with a unique suffix.\nThe provided value has the same validation rules as the Name field,\nand may be truncated by the length of the suffix required to make the value\nunique on the server.\n\nIf this field is specified and the generated name exists, the server will\nNOT return a 409 - instead, it will either return 201 Created or 500 with Reason\nServerTimeout indicating a unique name could not be found in the time allotted, and the client\nshould retry (optionally after the time indicated in the Retry-After header).\n\nApplied only if Name is not specified.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency\n+optional", + "type": "string" + }, + "generation": { + "type": "string", + "format": "int64", + "title": "A sequence number representing a specific generation of the desired state.\nPopulated by the system. Read-only.\n+optional" + }, + "initializers": { + "$ref": "#/definitions/v1Initializers" + }, + "labels": { + "type": "object", + "title": "Map of string keys and values that can be used to organize and categorize\n(scope and select) objects. May match selectors of replication controllers\nand services.\nMore info: http://kubernetes.io/docs/user-guide/labels\n+optional", + "additionalProperties": { + "type": "string" + } + }, + "managedFields": { + "description": "ManagedFields maps workflow-id and version to the set of fields\nthat are managed by that workflow. This is mostly for internal\nhousekeeping, and users typically shouldn't need to set or\nunderstand this field. A workflow can be the user's name, a\ncontroller's name, or the name of a specific apply path like\n\"ci-cd\". The set of fields is always in the version that the\nworkflow used when modifying the object.\n\nThis field is alpha and can be changed or removed without notice.\n\n+optional", + "type": "array", + "items": { + "$ref": "#/definitions/v1ManagedFieldsEntry" + } + }, + "name": { + "type": "string", + "title": "Name must be unique within a namespace. Is required when creating resources, although\nsome resources may allow a client to request the generation of an appropriate name\nautomatically. Name is primarily intended for creation idempotence and configuration\ndefinition.\nCannot be updated.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#names\n+optional" + }, + "namespace": { + "description": "Namespace defines the space within each name must be unique. An empty namespace is\nequivalent to the \"default\" namespace, but \"default\" is the canonical representation.\nNot all objects are required to be scoped to a namespace - the value of this field for\nthose objects will be empty.\n\nMust be a DNS_LABEL.\nCannot be updated.\nMore info: http://kubernetes.io/docs/user-guide/namespaces\n+optional", + "type": "string" + }, + "ownerReferences": { + "type": "array", + "title": "List of objects depended by this object. If ALL objects in the list have\nbeen deleted, this object will be garbage collected. If this object is managed by a controller,\nthen an entry in this list will point to this controller, with the controller field set to true.\nThere cannot be more than one managing controller.\n+optional\n+patchMergeKey=uid\n+patchStrategy=merge", + "items": { + "$ref": "#/definitions/v1OwnerReference" + } + }, + "resourceVersion": { + "description": "An opaque value that represents the internal version of this object that can\nbe used by clients to determine when objects have changed. May be used for optimistic\nconcurrency, change detection, and the watch operation on a resource or set of resources.\nClients must treat these values as opaque and passed unmodified back to the server.\nThey may only be valid for a particular resource or set of resources.\n\nPopulated by the system.\nRead-only.\nValue must be treated as opaque by clients and .\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\n+optional", + "type": "string" + }, + "selfLink": { + "type": "string", + "title": "SelfLink is a URL representing this object.\nPopulated by the system.\nRead-only.\n+optional" + }, + "uid": { + "description": "UID is the unique in time and space value for this object. It is typically generated by\nthe server on successful creation of a resource and is not allowed to change on PUT\noperations.\n\nPopulated by the system.\nRead-only.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#uids\n+optional", + "type": "string" + } + } + }, + "v1ObjectReference": { + "type": "object", + "title": "ObjectReference contains enough information to let you inspect or modify the referred object.\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", + "properties": { + "apiVersion": { + "type": "string", + "title": "API version of the referent.\n+optional" + }, + "fieldPath": { + "type": "string", + "title": "If referring to a piece of an object instead of an entire object, this string\nshould contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].\nFor example, if the object reference is to a container within a pod, this would take on a value like:\n\"spec.containers{name}\" (where \"name\" refers to the name of the container that triggered\nthe event) or if no container name is specified \"spec.containers[2]\" (container with\nindex 2 in this pod). This syntax is chosen only to have some well-defined way of\nreferencing a part of an object.\nTODO: this design is not final and this field is subject to change in the future.\n+optional" + }, + "kind": { + "type": "string", + "title": "Kind of the referent.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\n+optional" + }, + "name": { + "type": "string", + "title": "Name of the referent.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n+optional" + }, + "namespace": { + "type": "string", + "title": "Namespace of the referent.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/\n+optional" + }, + "resourceVersion": { + "type": "string", + "title": "Specific resourceVersion to which this reference is made, if any.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency\n+optional" + }, + "uid": { + "type": "string", + "title": "UID of the referent.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids\n+optional" + } + } + }, + "v1OwnerReference": { + "description": "OwnerReference contains enough information to let you identify an owning\nobject. An owning object must be in the same namespace as the dependent, or\nbe cluster-scoped, so there is no namespace field.", + "type": "object", + "properties": { + "apiVersion": { + "description": "API version of the referent.", + "type": "string" + }, + "blockOwnerDeletion": { + "type": "boolean", + "format": "boolean", + "title": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then\nthe owner cannot be deleted from the key-value store until this\nreference is removed.\nDefaults to false.\nTo set this field, a user needs \"delete\" permission of the owner,\notherwise 422 (Unprocessable Entity) will be returned.\n+optional" + }, + "controller": { + "type": "boolean", + "format": "boolean", + "title": "If true, this reference points to the managing controller.\n+optional" + }, + "kind": { + "type": "string", + "title": "Kind of the referent.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" + }, + "name": { + "type": "string", + "title": "Name of the referent.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#names" + }, + "uid": { + "type": "string", + "title": "UID of the referent.\nMore info: http://kubernetes.io/docs/user-guide/identifiers#uids" + } + } + }, + "v1Status": { + "description": "Status is a return value for calls that don't return other objects.", + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32", + "title": "Suggested HTTP return code for this status, 0 if not set.\n+optional" + }, + "details": { + "$ref": "#/definitions/v1StatusDetails" + }, + "message": { + "type": "string", + "title": "A human-readable description of the status of this operation.\n+optional" + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + }, + "reason": { + "type": "string", + "title": "A machine-readable description of why this operation is in the\n\"Failure\" status. If this value is empty there\nis no information available. A Reason clarifies an HTTP status\ncode but does not override it.\n+optional" + }, + "status": { + "type": "string", + "title": "Status of the operation.\nOne of: \"Success\" or \"Failure\".\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status\n+optional" + } + } + }, + "v1StatusCause": { + "description": "StatusCause provides more information about an api.Status failure, including\ncases when multiple errors are encountered.", + "type": "object", + "properties": { + "field": { + "description": "The field of the resource that has caused this error, as named by its JSON\nserialization. May include dot and postfix notation for nested attributes.\nArrays are zero-indexed. Fields may appear more than once in an array of\ncauses due to fields having multiple errors.\nOptional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"\n+optional", + "type": "string" + }, + "message": { + "type": "string", + "title": "A human-readable description of the cause of the error. This field may be\npresented as-is to a reader.\n+optional" + }, + "reason": { + "type": "string", + "title": "A machine-readable description of the cause of the error. If this value is\nempty there is no information available.\n+optional" + } + } + }, + "v1StatusDetails": { + "description": "StatusDetails is a set of additional properties that MAY be set by the\nserver to provide additional information about a response. The Reason\nfield of a Status object defines what attributes will be set. Clients\nmust ignore fields that do not match the defined type of each attribute,\nand should assume that any attribute may be empty, invalid, or under\ndefined.", + "type": "object", + "properties": { + "causes": { + "type": "array", + "title": "The Causes array includes more details associated with the StatusReason\nfailure. Not all StatusReasons may provide detailed causes.\n+optional", + "items": { + "$ref": "#/definitions/v1StatusCause" + } + }, + "group": { + "type": "string", + "title": "The group attribute of the resource associated with the status StatusReason.\n+optional" + }, + "kind": { + "type": "string", + "title": "The kind attribute of the resource associated with the status StatusReason.\nOn some operations may differ from the requested resource Kind.\nMore info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds\n+optional" + }, + "name": { + "type": "string", + "title": "The name attribute of the resource associated with the status StatusReason\n(when there is a single name which can be described).\n+optional" + }, + "retryAfterSeconds": { + "type": "integer", + "format": "int32", + "title": "If specified, the time in seconds before the operation should be retried. Some errors may indicate\nthe client must take an alternate action - for those errors this field may indicate how long to wait\nbefore taking the alternate action.\n+optional" + }, + "uid": { + "type": "string", + "title": "UID of the resource.\n(when there is a single resource which can be described).\nMore info: http://kubernetes.io/docs/user-guide/identifiers#uids\n+optional" + } + } + }, + "v1Time": { + "description": "Time is a wrapper around time.Time which supports correct\nmarshaling to YAML and JSON. Wrappers are provided for many\nof the factory methods that the time package offers.\n\n+protobuf.options.marshal=false\n+protobuf.as=Timestamp\n+protobuf.options.(gogoproto.goproto_stringer)=false", + "type": "object", + "properties": { + "nanos": { + "description": "Non-negative fractions of a second at nanosecond resolution. Negative\nsecond values with fractions must still have non-negative nanos values\nthat count forward in time. Must be from 0 to 999,999,999\ninclusive. This field may be limited in precision depending on context.", + "type": "integer", + "format": "int32" + }, + "seconds": { + "description": "Represents seconds of UTC time since Unix epoch\n1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to\n9999-12-31T23:59:59Z inclusive.", + "type": "string", + "format": "int64" + } + } + }, + "v1alpha1AWSAuthConfig": { + "type": "object", + "title": "AWSAuthConfig is an AWS IAM authentication configuration", + "properties": { + "clusterName": { + "type": "string", + "title": "ClusterName contains AWS cluster name" + }, + "roleARN": { + "description": "RoleARN contains optional role ARN. If set then AWS IAM Authenticator assume a role to perform cluster operations instead of the default AWS credential provider chain.", + "type": "string" + } + } + }, + "v1alpha1AppProject": { + "type": "object", + "title": "AppProject provides a logical grouping of applications, providing controls for:\n* where the apps may deploy to (cluster whitelist)\n* what may be deployed (repository whitelist, resource whitelist/blacklist)\n* who can access these applications (roles, OIDC group claims bindings)\n* and what they can do (RBAC policies)\n* automation access to these roles (JWT tokens)\n+genclient\n+genclient:noStatus\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n+kubebuilder:resource:path=appprojects,shortName=appproj;appprojs", + "properties": { + "metadata": { + "$ref": "#/definitions/v1ObjectMeta" + }, + "spec": { + "$ref": "#/definitions/v1alpha1AppProjectSpec" + } + } + }, + "v1alpha1AppProjectList": { + "type": "object", + "title": "AppProjectList is list of AppProject resources\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1AppProject" + } + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + } + } + }, + "v1alpha1AppProjectSpec": { + "type": "object", + "title": "AppProjectSpec is the specification of an AppProject", + "properties": { + "clusterResourceWhitelist": { + "type": "array", + "title": "ClusterResourceWhitelist contains list of whitelisted cluster level resources", + "items": { + "$ref": "#/definitions/v1GroupKind" + } + }, + "description": { + "type": "string", + "title": "Description contains optional project description" + }, + "destinations": { + "type": "array", + "title": "Destinations contains list of destinations available for deployment", + "items": { + "$ref": "#/definitions/v1alpha1ApplicationDestination" + } + }, + "namespaceResourceBlacklist": { + "type": "array", + "title": "NamespaceResourceBlacklist contains list of blacklisted namespace level resources", + "items": { + "$ref": "#/definitions/v1GroupKind" + } + }, + "roles": { + "type": "array", + "title": "Roles are user defined RBAC roles associated with this project", + "items": { + "$ref": "#/definitions/v1alpha1ProjectRole" + } + }, + "sourceRepos": { + "type": "array", + "title": "SourceRepos contains list of git repository URLs which can be used for deployment", + "items": { + "type": "string" + } + } + } + }, + "v1alpha1Application": { + "type": "object", + "title": "Application is a definition of Application resource.\n+genclient\n+genclient:noStatus\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object\n+kubebuilder:resource:path=applications,shortName=app;apps", + "properties": { + "metadata": { + "$ref": "#/definitions/v1ObjectMeta" + }, + "operation": { + "$ref": "#/definitions/v1alpha1Operation" + }, + "spec": { + "$ref": "#/definitions/v1alpha1ApplicationSpec" + }, + "status": { + "$ref": "#/definitions/v1alpha1ApplicationStatus" + } + } + }, + "v1alpha1ApplicationCondition": { + "type": "object", + "title": "ApplicationCondition contains details about current application condition", + "properties": { + "message": { + "type": "string", + "title": "Message contains human-readable message indicating details about condition" + }, + "type": { + "type": "string", + "title": "Type is an application condition type" + } + } + }, + "v1alpha1ApplicationDestination": { + "type": "object", + "title": "ApplicationDestination contains deployment destination information", + "properties": { + "namespace": { + "type": "string", + "title": "Namespace overrides the environment namespace value in the ksonnet app.yaml" + }, + "server": { + "type": "string", + "title": "Server overrides the environment server value in the ksonnet app.yaml" + } + } + }, + "v1alpha1ApplicationList": { + "type": "object", + "title": "ApplicationList is list of Application resources\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1Application" + } + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + } + } + }, + "v1alpha1ApplicationSource": { + "description": "ApplicationSource contains information about github repository, path within repository and target application environment.", + "type": "object", + "properties": { + "directory": { + "$ref": "#/definitions/v1alpha1ApplicationSourceDirectory" + }, + "helm": { + "$ref": "#/definitions/v1alpha1ApplicationSourceHelm" + }, + "ksonnet": { + "$ref": "#/definitions/v1alpha1ApplicationSourceKsonnet" + }, + "kustomize": { + "$ref": "#/definitions/v1alpha1ApplicationSourceKustomize" + }, + "path": { + "type": "string", + "title": "Path is a directory path within the repository containing a" + }, + "plugin": { + "$ref": "#/definitions/v1alpha1ApplicationSourcePlugin" + }, + "repoURL": { + "type": "string", + "title": "RepoURL is the git repository URL of the application manifests" + }, + "targetRevision": { + "type": "string", + "title": "TargetRevision defines the commit, tag, or branch in which to sync the application to.\nIf omitted, will sync to HEAD" + } + } + }, + "v1alpha1ApplicationSourceDirectory": { + "type": "object", + "properties": { + "jsonnet": { + "$ref": "#/definitions/v1alpha1ApplicationSourceJsonnet" + }, + "recurse": { + "type": "boolean", + "format": "boolean" + } + } + }, + "v1alpha1ApplicationSourceHelm": { + "type": "object", + "title": "ApplicationSourceHelm holds helm specific options", + "properties": { + "parameters": { + "type": "array", + "title": "Parameters are parameters to the helm template", + "items": { + "$ref": "#/definitions/v1alpha1HelmParameter" + } + }, + "releaseName": { + "type": "string", + "title": "The Helm release name. If omitted it will use the application name" + }, + "valueFiles": { + "type": "array", + "title": "ValuesFiles is a list of Helm value files to use when generating a template", + "items": { + "type": "string" + } + } + } + }, + "v1alpha1ApplicationSourceJsonnet": { + "type": "object", + "title": "ApplicationSourceJsonnet holds jsonnet specific options", + "properties": { + "extVars": { + "type": "array", + "title": "ExtVars is a list of Jsonnet External Variables", + "items": { + "$ref": "#/definitions/v1alpha1JsonnetVar" + } + }, + "tlas": { + "type": "array", + "title": "TLAS is a list of Jsonnet Top-level Arguments", + "items": { + "$ref": "#/definitions/v1alpha1JsonnetVar" + } + } + } + }, + "v1alpha1ApplicationSourceKsonnet": { + "type": "object", + "title": "ApplicationSourceKsonnet holds ksonnet specific options", + "properties": { + "environment": { + "type": "string", + "title": "Environment is a ksonnet application environment name" + }, + "parameters": { + "type": "array", + "title": "Parameters are a list of ksonnet component parameter override values", + "items": { + "$ref": "#/definitions/v1alpha1KsonnetParameter" + } + } + } + }, + "v1alpha1ApplicationSourceKustomize": { + "type": "object", + "title": "ApplicationSourceKustomize holds kustomize specific options", + "properties": { + "commonLabels": { + "type": "object", + "title": "CommonLabels adds additional kustomize commonLabels", + "additionalProperties": { + "type": "string" + } + }, + "images": { + "type": "array", + "title": "Images are kustomize image overrides", + "items": { + "type": "string" + } + }, + "namePrefix": { + "type": "string", + "title": "NamePrefix is a prefix appended to resources for kustomize apps" + } + } + }, + "v1alpha1ApplicationSourcePlugin": { + "type": "object", + "title": "ApplicationSourcePlugin holds config management plugin specific options", + "properties": { + "env": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1EnvEntry" + } + }, + "name": { + "type": "string" + } + } + }, + "v1alpha1ApplicationSpec": { + "description": "ApplicationSpec represents desired application state. Contains link to repository with application definition and additional parameters link definition revision.", + "type": "object", + "properties": { + "destination": { + "$ref": "#/definitions/v1alpha1ApplicationDestination" + }, + "ignoreDifferences": { + "type": "array", + "title": "IgnoreDifferences controls resources fields which should be ignored during comparison", + "items": { + "$ref": "#/definitions/v1alpha1ResourceIgnoreDifferences" + } + }, + "info": { + "type": "array", + "title": "Infos contains a list of useful information (URLs, email addresses, and plain text) that relates to the application", + "items": { + "$ref": "#/definitions/v1alpha1Info" + } + }, + "project": { + "description": "Project is a application project name. Empty name means that application belongs to 'default' project.", + "type": "string" + }, + "source": { + "$ref": "#/definitions/v1alpha1ApplicationSource" + }, + "syncPolicy": { + "$ref": "#/definitions/v1alpha1SyncPolicy" + } + } + }, + "v1alpha1ApplicationStatus": { + "type": "object", + "title": "ApplicationStatus contains information about application sync, health status", + "properties": { + "conditions": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ApplicationCondition" + } + }, + "health": { + "$ref": "#/definitions/v1alpha1HealthStatus" + }, + "history": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1RevisionHistory" + } + }, + "observedAt": { + "$ref": "#/definitions/v1Time" + }, + "operationState": { + "$ref": "#/definitions/v1alpha1OperationState" + }, + "reconciledAt": { + "$ref": "#/definitions/v1Time" + }, + "resources": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceStatus" + } + }, + "sourceType": { + "type": "string" + }, + "summary": { + "$ref": "#/definitions/v1alpha1ApplicationSummary" + }, + "sync": { + "$ref": "#/definitions/v1alpha1SyncStatus" + } + } + }, + "v1alpha1ApplicationSummary": { + "type": "object", + "properties": { + "externalURLs": { + "description": "ExternalURLs holds all external URLs of application child resources.", + "type": "array", + "items": { + "type": "string" + } + }, + "images": { + "description": "Images holds all images of application child resources.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "v1alpha1ApplicationTree": { + "type": "object", + "title": "ApplicationTree holds nodes which belongs to the application", + "properties": { + "nodes": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceNode" + } + } + } + }, + "v1alpha1ApplicationWatchEvent": { + "description": "ApplicationWatchEvent contains information about application change.", + "type": "object", + "properties": { + "application": { + "$ref": "#/definitions/v1alpha1Application" + }, + "type": { + "type": "string" + } + } + }, + "v1alpha1Cluster": { + "type": "object", + "title": "Cluster is the definition of a cluster resource", + "properties": { + "config": { + "$ref": "#/definitions/v1alpha1ClusterConfig" + }, + "connectionState": { + "$ref": "#/definitions/v1alpha1ConnectionState" + }, + "name": { + "type": "string", + "title": "Name of the cluster. If omitted, will use the server address" + }, + "server": { + "type": "string", + "title": "Server is the API server URL of the Kubernetes cluster" + } + } + }, + "v1alpha1ClusterConfig": { + "description": "ClusterConfig is the configuration attributes. This structure is subset of the go-client\nrest.Config with annotations added for marshalling.", + "type": "object", + "properties": { + "awsAuthConfig": { + "$ref": "#/definitions/v1alpha1AWSAuthConfig" + }, + "bearerToken": { + "description": "Server requires Bearer authentication. This client will not attempt to use\nrefresh tokens for an OAuth2 flow.\nTODO: demonstrate an OAuth2 compatible client.", + "type": "string" + }, + "password": { + "type": "string" + }, + "tlsClientConfig": { + "$ref": "#/definitions/v1alpha1TLSClientConfig" + }, + "username": { + "type": "string", + "title": "Server requires Basic authentication" + } + } + }, + "v1alpha1ClusterList": { + "description": "ClusterList is a collection of Clusters.", + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1Cluster" + } + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + } + } + }, + "v1alpha1ComparedTo": { + "type": "object", + "title": "ComparedTo contains application source and target which was used for resources comparison", + "properties": { + "destination": { + "$ref": "#/definitions/v1alpha1ApplicationDestination" + }, + "source": { + "$ref": "#/definitions/v1alpha1ApplicationSource" + } + } + }, + "v1alpha1ConnectionState": { + "type": "object", + "title": "ConnectionState contains information about remote resource connection state", + "properties": { + "attemptedAt": { + "$ref": "#/definitions/v1Time" + }, + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "v1alpha1EnvEntry": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "the name, usually uppercase" + }, + "value": { + "type": "string", + "title": "the value" + } + } + }, + "v1alpha1HealthStatus": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "v1alpha1HelmParameter": { + "type": "object", + "title": "HelmParameter is a parameter to a helm template", + "properties": { + "forceString": { + "type": "boolean", + "format": "boolean", + "title": "ForceString determines whether to tell Helm to interpret booleans and numbers as strings" + }, + "name": { + "type": "string", + "title": "Name is the name of the helm parameter" + }, + "value": { + "type": "string", + "title": "Value is the value for the helm parameter" + } + } + }, + "v1alpha1Info": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "v1alpha1InfoItem": { + "type": "object", + "title": "InfoItem contains human readable information about object", + "properties": { + "name": { + "description": "Name is a human readable title for this piece of information.", + "type": "string" + }, + "value": { + "description": "Value is human readable content.", + "type": "string" + } + } + }, + "v1alpha1JWTToken": { + "type": "object", + "title": "JWTToken holds the issuedAt and expiresAt values of a token", + "properties": { + "exp": { + "type": "string", + "format": "int64" + }, + "iat": { + "type": "string", + "format": "int64" + } + } + }, + "v1alpha1JsonnetVar": { + "type": "object", + "title": "JsonnetVar is a jsonnet variable", + "properties": { + "code": { + "type": "boolean", + "format": "boolean" + }, + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "v1alpha1KsonnetParameter": { + "type": "object", + "title": "KsonnetParameter is a ksonnet component parameter", + "properties": { + "component": { + "type": "string" + }, + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "v1alpha1KustomizeOptions": { + "type": "object", + "title": "KustomizeOptions are options for kustomize to use when building manifests", + "properties": { + "buildOptions": { + "type": "string", + "title": "BuildOptions is a string of build parameters to use when calling `kustomize build`" + } + } + }, + "v1alpha1Operation": { + "description": "Operation contains requested operation parameters.", + "type": "object", + "properties": { + "sync": { + "$ref": "#/definitions/v1alpha1SyncOperation" + } + } + }, + "v1alpha1OperationState": { + "description": "OperationState contains information about state of currently performing operation on application.", + "type": "object", + "properties": { + "finishedAt": { + "$ref": "#/definitions/v1Time" + }, + "message": { + "description": "Message hold any pertinent messages when attempting to perform operation (typically errors).", + "type": "string" + }, + "operation": { + "$ref": "#/definitions/v1alpha1Operation" + }, + "phase": { + "type": "string", + "title": "Phase is the current phase of the operation" + }, + "startedAt": { + "$ref": "#/definitions/v1Time" + }, + "syncResult": { + "$ref": "#/definitions/v1alpha1SyncOperationResult" + } + } + }, + "v1alpha1ProjectRole": { + "type": "object", + "title": "ProjectRole represents a role that has access to a project", + "properties": { + "description": { + "type": "string", + "title": "Description is a description of the role" + }, + "groups": { + "type": "array", + "title": "Groups are a list of OIDC group claims bound to this role", + "items": { + "type": "string" + } + }, + "jwtTokens": { + "type": "array", + "title": "JWTTokens are a list of generated JWT tokens bound to this role", + "items": { + "$ref": "#/definitions/v1alpha1JWTToken" + } + }, + "name": { + "type": "string", + "title": "Name is a name for this role" + }, + "policies": { + "type": "array", + "title": "Policies Stores a list of casbin formated strings that define access policies for the role in the project", + "items": { + "type": "string" + } + } + } + }, + "v1alpha1Repository": { + "type": "object", + "title": "Repository is a Git repository holding application configurations", + "properties": { + "connectionState": { + "$ref": "#/definitions/v1alpha1ConnectionState" + }, + "enableLfs": { + "type": "boolean", + "format": "boolean", + "title": "Whether git-lfs support should be enabled for this repo" + }, + "insecure": { + "type": "boolean", + "format": "boolean", + "title": "Whether the repo is insecure" + }, + "insecureIgnoreHostKey": { + "type": "boolean", + "format": "boolean", + "title": "InsecureIgnoreHostKey should not be used anymore, Insecure is favoured" + }, + "password": { + "type": "string", + "title": "Password for authenticating at the repo server" + }, + "repo": { + "type": "string", + "title": "URL of the repo" + }, + "sshPrivateKey": { + "type": "string", + "title": "SSH private key data for authenticating at the repo server" + }, + "tlsClientCertData": { + "type": "string", + "title": "TLS client cert data for authenticating at the repo server" + }, + "tlsClientCertKey": { + "type": "string", + "title": "TLS client cert key for authenticating at the repo server" + }, + "username": { + "type": "string", + "title": "Username for authenticating at the repo server" + } + } + }, + "v1alpha1RepositoryCertificate": { + "type": "object", + "title": "A RepositoryCertificate is either SSH known hosts entry or TLS certificate", + "properties": { + "certData": { + "type": "string", + "format": "byte", + "title": "Actual certificate data, protocol dependent" + }, + "certInfo": { + "type": "string", + "title": "Additional certificate info (e.g. SSH fingerprint, X509 CommonName)" + }, + "certSubType": { + "type": "string", + "title": "The sub type of the cert, i.e. \"ssh-rsa\"" + }, + "certType": { + "type": "string", + "title": "Type of certificate - currently \"https\" or \"ssh\"" + }, + "serverName": { + "type": "string", + "title": "Name of the server the certificate is intended for" + } + } + }, + "v1alpha1RepositoryCertificateList": { + "type": "object", + "title": "RepositoryCertificateList is a collection of RepositoryCertificates", + "properties": { + "items": { + "type": "array", + "title": "List of certificates to be processed", + "items": { + "$ref": "#/definitions/v1alpha1RepositoryCertificate" + } + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + } + } + }, + "v1alpha1RepositoryList": { + "description": "RepositoryList is a collection of Repositories.", + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1Repository" + } + }, + "metadata": { + "$ref": "#/definitions/v1ListMeta" + } + } + }, + "v1alpha1ResourceAction": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "params": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceActionParam" + } + } + } + }, + "v1alpha1ResourceActionParam": { + "type": "object", + "properties": { + "default": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "v1alpha1ResourceDiff": { + "type": "object", + "title": "ResourceDiff holds the diff of a live and target resource object", + "properties": { + "diff": { + "type": "string" + }, + "group": { + "type": "string" + }, + "hook": { + "type": "boolean", + "format": "boolean" + }, + "kind": { + "type": "string" + }, + "liveState": { + "type": "string" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "targetState": { + "type": "string" + } + } + }, + "v1alpha1ResourceIgnoreDifferences": { + "description": "ResourceIgnoreDifferences contains resource filter and list of json paths which should be ignored during comparison with live state.", + "type": "object", + "properties": { + "group": { + "type": "string" + }, + "jsonPointers": { + "type": "array", + "items": { + "type": "string" + } + }, + "kind": { + "type": "string" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + } + }, + "v1alpha1ResourceNetworkingInfo": { + "type": "object", + "title": "ResourceNetworkingInfo holds networking resource related information", + "properties": { + "externalURLs": { + "description": "ExternalURLs holds list of URLs which should be available externally. List is populated for ingress resources using rules hostnames.", + "type": "array", + "items": { + "type": "string" + } + }, + "ingress": { + "type": "array", + "items": { + "$ref": "#/definitions/v1LoadBalancerIngress" + } + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "targetLabels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "targetRefs": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceRef" + } + } + } + }, + "v1alpha1ResourceNode": { + "type": "object", + "title": "ResourceNode contains information about live resource and its children", + "properties": { + "health": { + "$ref": "#/definitions/v1alpha1HealthStatus" + }, + "images": { + "type": "array", + "items": { + "type": "string" + } + }, + "info": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1InfoItem" + } + }, + "networkingInfo": { + "$ref": "#/definitions/v1alpha1ResourceNetworkingInfo" + }, + "parentRefs": { + "type": "array", + "items": { + "$ref": "#/definitions/v1alpha1ResourceRef" + } + }, + "resourceRef": { + "$ref": "#/definitions/v1alpha1ResourceRef" + }, + "resourceVersion": { + "type": "string" + } + } + }, + "v1alpha1ResourceOverride": { + "type": "object", + "title": "ResourceOverride holds configuration to customize resource diffing and health assessment", + "properties": { + "actions": { + "type": "string" + }, + "healthLua": { + "type": "string" + }, + "ignoreDifferences": { + "type": "string" + } + } + }, + "v1alpha1ResourceRef": { + "type": "object", + "title": "ResourceRef includes fields which unique identify resource", + "properties": { + "group": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "v1alpha1ResourceResult": { + "type": "object", + "title": "ResourceResult holds the operation result details of a specific resource", + "properties": { + "group": { + "type": "string" + }, + "hookPhase": { + "type": "string", + "title": "the state of any operation associated with this resource OR hook\nnote: can contain values for non-hook resources" + }, + "hookType": { + "type": "string", + "title": "the type of the hook, empty for non-hook resources" + }, + "kind": { + "type": "string" + }, + "message": { + "type": "string", + "title": "message for the last sync OR operation" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "status": { + "type": "string", + "title": "the final result of the sync, this is be empty if the resources is yet to be applied/pruned and is always zero-value for hooks" + }, + "syncPhase": { + "type": "string", + "title": "indicates the particular phase of the sync that this is for" + }, + "version": { + "type": "string" + } + } + }, + "v1alpha1ResourceStatus": { + "type": "object", + "title": "ResourceStatus holds the current sync and health status of a resource", + "properties": { + "group": { + "type": "string" + }, + "health": { + "$ref": "#/definitions/v1alpha1HealthStatus" + }, + "hook": { + "type": "boolean", + "format": "boolean" + }, + "kind": { + "type": "string" + }, + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "requiresPruning": { + "type": "boolean", + "format": "boolean" + }, + "status": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "v1alpha1RevisionHistory": { + "type": "object", + "title": "RevisionHistory contains information relevant to an application deployment", + "properties": { + "deployedAt": { + "$ref": "#/definitions/v1Time" + }, + "id": { + "type": "string", + "format": "int64" + }, + "revision": { + "type": "string" + }, + "source": { + "$ref": "#/definitions/v1alpha1ApplicationSource" + } + } + }, + "v1alpha1RevisionMetadata": { + "type": "object", + "title": "data about a specific revision within a repo", + "properties": { + "author": { + "type": "string", + "title": "who authored this revision,\ntypically their name and email, e.g. \"John Doe \",\nbut might not match this example" + }, + "date": { + "$ref": "#/definitions/v1Time" + }, + "message": { + "type": "string", + "title": "the message associated with the revision,\nprobably the commit message,\nthis is truncated to the first newline or 64 characters (which ever comes first)" + }, + "tags": { + "type": "array", + "title": "tags on the revision,\nnote - tags can move from one revision to another", + "items": { + "type": "string" + } + } + } + }, + "v1alpha1SyncOperation": { + "description": "SyncOperation contains sync operation details.", + "type": "object", + "properties": { + "dryRun": { + "type": "boolean", + "format": "boolean", + "title": "DryRun will perform a `kubectl apply --dry-run` without actually performing the sync" + }, + "manifests": { + "type": "array", + "title": "Manifests is an optional field that overrides sync source with a local directory for development", + "items": { + "type": "string" + } + }, + "prune": { + "type": "boolean", + "format": "boolean", + "title": "Prune deletes resources that are no longer tracked in git" + }, + "resources": { + "type": "array", + "title": "Resources describes which resources to sync", + "items": { + "$ref": "#/definitions/v1alpha1SyncOperationResource" + } + }, + "revision": { + "description": "Revision is the git revision in which to sync the application to.\nIf omitted, will use the revision specified in app spec.", + "type": "string" + }, + "source": { + "$ref": "#/definitions/v1alpha1ApplicationSource" + }, + "syncStrategy": { + "$ref": "#/definitions/v1alpha1SyncStrategy" + } + } + }, + "v1alpha1SyncOperationResource": { + "description": "SyncOperationResource contains resources to sync.", + "type": "object", + "properties": { + "group": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "v1alpha1SyncOperationResult": { + "type": "object", + "title": "SyncOperationResult represent result of sync operation", + "properties": { + "resources": { + "type": "array", + "title": "Resources holds the sync result of each individual resource", + "items": { + "$ref": "#/definitions/v1alpha1ResourceResult" + } + }, + "revision": { + "type": "string", + "title": "Revision holds the git commit SHA of the sync" + }, + "source": { + "$ref": "#/definitions/v1alpha1ApplicationSource" + } + } + }, + "v1alpha1SyncPolicy": { + "type": "object", + "title": "SyncPolicy controls when a sync will be performed in response to updates in git", + "properties": { + "automated": { + "$ref": "#/definitions/v1alpha1SyncPolicyAutomated" + } + } + }, + "v1alpha1SyncPolicyAutomated": { + "type": "object", + "title": "SyncPolicyAutomated controls the behavior of an automated sync", + "properties": { + "prune": { + "type": "boolean", + "format": "boolean", + "title": "Prune will prune resources automatically as part of automated sync (default: false)" + }, + "selfHeal": { + "type": "boolean", + "format": "boolean", + "title": "SelfHeal enables auto-syncing if (default: false)" + } + } + }, + "v1alpha1SyncStatus": { + "description": "SyncStatus is a comparison result of application spec and deployed application.", + "type": "object", + "properties": { + "comparedTo": { + "$ref": "#/definitions/v1alpha1ComparedTo" + }, + "revision": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "v1alpha1SyncStrategy": { + "type": "object", + "title": "SyncStrategy controls the manner in which a sync is performed", + "properties": { + "apply": { + "$ref": "#/definitions/v1alpha1SyncStrategyApply" + }, + "hook": { + "$ref": "#/definitions/v1alpha1SyncStrategyHook" + } + } + }, + "v1alpha1SyncStrategyApply": { + "type": "object", + "title": "SyncStrategyApply uses `kubectl apply` to perform the apply", + "properties": { + "force": { + "description": "Force indicates whether or not to supply the --force flag to `kubectl apply`.\nThe --force flag deletes and re-create the resource, when PATCH encounters conflict and has\nretried for 5 times.", + "type": "boolean", + "format": "boolean" + } + } + }, + "v1alpha1SyncStrategyHook": { + "description": "SyncStrategyHook will perform a sync using hooks annotations.\nIf no hook annotation is specified falls back to `kubectl apply`.", + "type": "object", + "properties": { + "syncStrategyApply": { + "$ref": "#/definitions/v1alpha1SyncStrategyApply" + } + } + }, + "v1alpha1TLSClientConfig": { + "type": "object", + "title": "TLSClientConfig contains settings to enable transport layer security", + "properties": { + "caData": { + "type": "string", + "format": "byte", + "title": "CAData holds PEM-encoded bytes (typically read from a root certificates bundle).\nCAData takes precedence over CAFile" + }, + "certData": { + "type": "string", + "format": "byte", + "title": "CertData holds PEM-encoded bytes (typically read from a client certificate file).\nCertData takes precedence over CertFile" + }, + "insecure": { + "description": "Server should be accessed without verifying the TLS certificate. For testing only.", + "type": "boolean", + "format": "boolean" + }, + "keyData": { + "type": "string", + "format": "byte", + "title": "KeyData holds PEM-encoded bytes (typically read from a client certificate key file).\nKeyData takes precedence over KeyFile" + }, + "serverName": { + "description": "ServerName is passed to the server for SNI and is used in the client to check server\ncertificates against. If ServerName is empty, the hostname used to contact the\nserver is used.", + "type": "string" + } + } + }, + "versionVersionMessage": { + "type": "object", + "title": "VersionMessage represents version of the Argo CD API server", + "properties": { + "BuildDate": { + "type": "string" + }, + "Compiler": { + "type": "string" + }, + "GitCommit": { + "type": "string" + }, + "GitTag": { + "type": "string" + }, + "GitTreeState": { + "type": "string" + }, + "GoVersion": { + "type": "string" + }, + "KsonnetVersion": { + "type": "string" + }, + "Platform": { + "type": "string" + }, + "Version": { + "type": "string" + } + } + } + } +} \ No newline at end of file From 7d2f5f7c31e843ff9202098b66f59546fe9a95ff Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 15:47:10 +0530 Subject: [PATCH 036/118] Copied the earlier deleted files --- internal/sql/repository/AppListingRepository.go | 6 +++--- internal/sql/repository/app/AppRepository.go | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 43188c7420..bbcc2a8ec9 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -85,9 +85,9 @@ func NewAppListingRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB, func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) { var jobContainers []*bean.JobListingContainer - //if len(appIds) == 0 { - // return jobContainers, nil - //} + if len(appIds) == 0 { + return jobContainers, nil + } jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(appIds, statuses) impl.Logger.Debugw("basic app detail query: ", jobsQuery) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 63ea66a180..ed7f88fb04 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -359,11 +359,14 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL Id int `json:"id"` } var appIds []AppId - whereCondition := " where display_name like ? and active = true and app_store = 2 " + whereCondition := " where active = true and app_store = 2 " if len(jobListingFilter.Teams) > 0 { whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" } + if len(jobListingFilter.AppNameSearch) > 0 { + whereCondition += " and display_name like '%" + jobListingFilter.AppNameSearch + "%' " + } orderByCondition := " order by display_name " if jobListingFilter.SortOrder == "DESC" { orderByCondition += string(jobListingFilter.SortOrder) @@ -371,8 +374,8 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL orderByCondition += " limit ? offset ? " query := "select id " + "from app " + whereCondition + orderByCondition - appName := "%" + jobListingFilter.AppNameSearch + "%" - _, err := repo.dbConnection.Query(&appIds, query, appName, jobListingFilter.Size, jobListingFilter.Offset) + + _, err := repo.dbConnection.Query(&appIds, query, jobListingFilter.Size, jobListingFilter.Offset) appIdsResult := make([]int, 0) for _, id := range appIds { appIdsResult = append(appIdsResult, id.Id) From de20607c2d11a2b2681258aec0eb2ee0da182a09 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Wed, 22 Feb 2023 17:24:07 +0530 Subject: [PATCH 037/118] Update jobs.yaml --- specs/jobs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 52ae664b69..d06fbf8ddb 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -21,7 +21,7 @@ paths: schema: $ref: "#/components/schemas/ActionResponse" /orchestrator/app/jobs/list: - get: + post: description: Get the list of all the jobs by applying filter requestBody: required: true From 9eea012e2b7153f8bffc37794cc87a2fd9ca4632 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 17:51:27 +0530 Subject: [PATCH 038/118] Updated the api end points and wrote the api specs. --- Wire.go | 3 + api/restHandler/AppListingRestHandler.go | 10 +- .../app/BuildPipelineRestHandler.go | 6 +- .../app/PipelineConfigRestHandler.go | 1 + api/router/AppListingRouter.go | 4 - api/router/JobsRouter.go | 29 ++++ api/router/router.go | 8 +- .../sql/repository/AppListingRepository.go | 6 +- pkg/app/AppListingService.go | 6 +- pkg/bean/app.go | 2 +- pkg/pipeline/PipelineBuilder.go | 3 + specs/jobs.yaml | 161 +++++++++++++++++- wire_gen.go | 3 +- 13 files changed, 217 insertions(+), 25 deletions(-) create mode 100644 api/router/JobsRouter.go diff --git a/Wire.go b/Wire.go index ddd2736895..8d4139fa02 100644 --- a/Wire.go +++ b/Wire.go @@ -248,6 +248,9 @@ func InitializeApp() (*App, error) { repository.NewAppListingRepositoryImpl, wire.Bind(new(repository.AppListingRepository), new(*repository.AppListingRepositoryImpl)), + router.NewJobRouterImpl, + wire.Bind(new(router.JobRouter), new(*router.JobRouterImpl)), + pipelineConfig.NewPipelineRepositoryImpl, wire.Bind(new(pipelineConfig.PipelineRepository), new(*pipelineConfig.PipelineRepositoryImpl)), pipeline.NewPropertiesConfigServiceImpl, diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 19fcc874c7..55269e1dee 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -57,7 +57,7 @@ import ( type AppListingRestHandler interface { FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) FetchJobs(w http.ResponseWriter, r *http.Request) - FetchOverviewCiPipeline(w http.ResponseWriter, r *http.Request) + FetchOverviewCiPipelines(w http.ResponseWriter, r *http.Request) FetchAppDetails(w http.ResponseWriter, r *http.Request) FetchAllDevtronManagedApps(w http.ResponseWriter, r *http.Request) FetchAppTriggerView(w http.ResponseWriter, r *http.Request) @@ -197,7 +197,7 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) } -func (handler AppListingRestHandlerImpl) FetchOverviewCiPipeline(w http.ResponseWriter, r *http.Request) { +func (handler AppListingRestHandlerImpl) FetchOverviewCiPipelines(w http.ResponseWriter, r *http.Request) { userId, err := handler.userService.GetLoggedInUser(r) if userId == 0 || err != nil { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) @@ -205,14 +205,14 @@ func (handler AppListingRestHandlerImpl) FetchOverviewCiPipeline(w http.Response } vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) + jobId, err := strconv.Atoi(vars["jobId"]) if err != nil { - handler.logger.Errorw("request err, GetAppMetaInfo", "err", err, "appId", appId) + handler.logger.Errorw("request err, GetAppMetaInfo", "err", err, "jobId", jobId) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - jobCi, err := handler.appListingService.FetchOverviewCiPipeline(appId) + jobCi, err := handler.appListingService.FetchOverviewCiPipelines(jobId) common.WriteJsonResp(w, err, jobCi, http.StatusOK) } diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index a0d5e95f5c..871a2c3906 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -243,8 +243,10 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri return } + ciConf, err := handler.pipelineBuilder.GetCiPipeline(patchRequest.AppId) + var emptyDockerRegistry string - if patchRequest.IsJob { + if patchRequest.IsJob && ciConf == nil { ciConfigRequest := bean.CiConfigRequest{} ciConfigRequest.DockerRegistry = emptyDockerRegistry ciConfigRequest.AppId = patchRequest.AppId @@ -287,11 +289,13 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri } } createResp, err := handler.pipelineBuilder.PatchCiPipeline(&patchRequest) + createResp.AppName = app.AppName if err != nil { handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } + common.WriteJsonResp(w, err, createResp, http.StatusOK) } diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 2a2929861a..b4852f399f 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -327,6 +327,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r //} var createResp *bean.CreateAppDTO err = nil + createRequest.IsJob = true if createRequest.TemplateId == 0 { createResp, err = handler.pipelineBuilder.CreateApp(&createRequest) } else { diff --git a/api/router/AppListingRouter.go b/api/router/AppListingRouter.go index b97ff50612..0f51031911 100644 --- a/api/router/AppListingRouter.go +++ b/api/router/AppListingRouter.go @@ -52,10 +52,6 @@ func (router AppListingRouterImpl) initAppListingRouter(appListingRouter *mux.Ro HandlerFunc(router.appListingRestHandler.FetchJobs). Methods("POST") - appListingRouter.Path("/jobs/list/{appId}"). - HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipeline). - Methods("GET") - //This API used for fetch app details, not deployment details appListingRouter.Path("/detail").Queries("app-id", "{app-id}").Queries("env-id", "{env-id}"). HandlerFunc(router.appListingRestHandler.FetchAppDetails). diff --git a/api/router/JobsRouter.go b/api/router/JobsRouter.go new file mode 100644 index 0000000000..834bee1c65 --- /dev/null +++ b/api/router/JobsRouter.go @@ -0,0 +1,29 @@ +package router + +import ( + "github.com/devtron-labs/devtron/api/restHandler" + "github.com/devtron-labs/devtron/api/restHandler/app" + "github.com/gorilla/mux" +) + +type JobRouter interface { + InitJobRouter(router *mux.Router) +} +type JobRouterImpl struct { + pipelineConfigRestHandler app.PipelineConfigRestHandler + appListingRestHandler restHandler.AppListingRestHandler +} + +func NewJobRouterImpl(pipelineConfigRestHandler app.PipelineConfigRestHandler, appListingRestHandler restHandler.AppListingRestHandler) *JobRouterImpl { + router := &JobRouterImpl{ + appListingRestHandler: appListingRestHandler, + pipelineConfigRestHandler: pipelineConfigRestHandler, + } + return router +} +func (router JobRouterImpl) InitJobRouter(jobRouter *mux.Router) { + jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateJob).Methods("POST") + jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchCiPipelines).Methods("POST") + jobRouter.Path("/list").HandlerFunc(router.appListingRestHandler.FetchJobs).Methods("POST") + jobRouter.Path("/ci-pipeline/list/{jobId}").HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipelines).Methods("GET") +} diff --git a/api/router/router.go b/api/router/router.go index 2300652679..be065245c8 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -55,6 +55,7 @@ type MuxRouter struct { Router *mux.Router HelmRouter PipelineTriggerRouter PipelineConfigRouter PipelineConfigRouter + JobRouter JobRouter MigrateDbRouter MigrateDbRouter EnvironmentClusterMappingsRouter cluster.EnvironmentRouter AppListingRouter AppListingRouter @@ -143,7 +144,8 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P serverRouter server.ServerRouter, apiTokenRouter apiToken.ApiTokenRouter, helmApplicationStatusUpdateHandler cron.CdApplicationStatusUpdateHandler, k8sCapacityRouter k8s.K8sCapacityRouter, webhookHelmRouter webhookHelm.WebhookHelmRouter, globalCMCSRouter GlobalCMCSRouter, - userTerminalAccessRouter terminal2.UserTerminalAccessRouter, ciStatusUpdateCron cron.CiStatusUpdateCron) *MuxRouter { + userTerminalAccessRouter terminal2.UserTerminalAccessRouter, + jobRouter JobRouterImpl, ciStatusUpdateCron cron.CiStatusUpdateCron) *MuxRouter { r := &MuxRouter{ Router: mux.NewRouter(), HelmRouter: HelmRouter, @@ -210,6 +212,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P globalCMCSRouter: globalCMCSRouter, userTerminalAccessRouter: userTerminalAccessRouter, ciStatusUpdateCron: ciStatusUpdateCron, + JobRouter: jobRouter, } return r } @@ -258,6 +261,9 @@ func (r MuxRouter) Init() { r.HelmRouter.initPipelineTriggerRouter(pipelineConfigRouter) r.appRouter.InitAppRouter(pipelineConfigRouter) + jobConfigRouter := r.Router.PathPrefix("/orchestrator/job").Subrouter() + r.JobRouter.InitJobRouter(jobConfigRouter) + migrateRouter := r.Router.PathPrefix("/orchestrator/migrate").Subrouter() r.MigrateDbRouter.InitMigrateDbRouter(migrateRouter) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index bbcc2a8ec9..4192944f94 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -38,7 +38,7 @@ import ( type AppListingRepository interface { FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) - FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) + FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) FetchJobsLastSucceededOn(ciPipelineIDs []int) ([]*bean.CiPipelineLastSucceededTime, error) DeploymentDetailsByAppIdAndEnvId(ctx context.Context, appId int, envId int) (bean.DeploymentDetailContainer, error) FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -98,11 +98,11 @@ func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string) } return jobContainers, nil } -func (impl AppListingRepositoryImpl) FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) { +func (impl AppListingRepositoryImpl) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) { var jobContainers []*bean.JobListingContainer jobsQuery := impl.appListingRepositoryQueryBuilder.OverviewCiPipelineQuery() impl.Logger.Debugw("basic app detail query: ", jobsQuery) - _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, appId) + _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery, jobId) if appsErr != nil { impl.Logger.Error(appsErr) return jobContainers, appsErr diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 7e7aa795b3..ecf735cf92 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -54,7 +54,7 @@ import ( type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) - FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) + FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) FetchAppDetails(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) @@ -249,8 +249,8 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi return jobContainers, nil } -func (impl AppListingServiceImpl) FetchOverviewCiPipeline(appId int) ([]*bean.JobListingContainer, error) { - jobCiContainers, err := impl.appListingRepository.FetchOverviewCiPipeline(appId) +func (impl AppListingServiceImpl) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) { + jobCiContainers, err := impl.appListingRepository.FetchOverviewCiPipelines(jobId) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) return []*bean.JobListingContainer{}, err diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 955a4e67b7..47d8b2a703 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -46,7 +46,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` - IsJob bool `json:"isJob"` + IsJob bool `json:"-"` } type CreateMaterialDTO struct { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 781c4f7ea3..9d1e5932b1 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -381,6 +381,9 @@ func (impl PipelineBuilderImpl) GetApp(appId int) (application *bean.CreateAppDT } gitMaterials := impl.GetMaterialsForAppId(appId) + if app.AppStore == 2 { + app.AppName = app.DisplayName + } application = &bean.CreateAppDTO{ Id: app.Id, AppName: app.AppName, diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 52ae664b69..807347019d 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -3,7 +3,7 @@ info: title: Jobs version: "1.0" paths: - /orchestrator/app/jobs: + /orchestrator/job: post: description: Create and clones a job requestBody: @@ -20,8 +20,8 @@ paths: application/json: schema: $ref: "#/components/schemas/ActionResponse" - /orchestrator/app/jobs/list: - get: + /orchestrator/job/list: + post: description: Get the list of all the jobs by applying filter requestBody: required: true @@ -37,6 +37,51 @@ paths: schema: items: $ref: "#/components/schemas/JobListResponse" + /orchestrator/job/ci-pipeline/list/{jobId}: + get: + description: fetch details of job ci-pipelines for the overview page + parameters: + - name: jobId + in: path + required: true + schema: + type: integer + /orchestrator/job//ci-pipeline/patch: + post: + description: Create/Update ci pipeline. + operationId: PatchCiPipeline + requestBody: + description: A JSON object containing the pipeline configuration + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CiPatchRequest" + responses: + "200": + description: Successfully return a message stating the operation is successful. + content: + application/json: + schema: + type: string + "400": + description: Bad Request. Validation error/wrong request body. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + description: Internal Server Error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "403": + description: Unauthorized User + content: + application/json: + schema: + $ref: "#/components/schemas/Error" #components components: @@ -176,8 +221,112 @@ components: type: time.Time jobCount: type: integer - - - + Error: + required: + - code + - message + properties: + code: + type: integer + description: Error code + message: + type: string + description: Error message + CiPatchRequest: + type: object + properties: + action: + type: integer + appId: + type: integer + isJob: + type: boolean + example: true + appWorkflowId: + type: integer + ciPipeline: + $ref: "#/components/schemas/CiPipelineDetails" + CiPipelineDetails: + type: object + properties: + isDockerConfigOverridden: + type: boolean + dockerConfigOverride: + $ref: "#/components/schemas/DockerBuildConfig" + name: + type: string + isManual: + type: boolean + scanEnabled: + type: boolean + isExternal: + type: boolean + parentAppId: + type: integer + parentCiPipeline: + type: integer + linkedCount: + type: integer + ciMaterials: + type: array + items: + $ref: "#/components/schemas/CiPipelineMaterialConfig" + dockerArgs: + type: array + items: + type: object + properties: + Key: + type: string + Value: + type: string + description: map of docker arguments, i.e. key-value pairs + beforeDockerBuildScripts: + type: array + items: + $ref: "#/components/schemas/BuildScript" + afterDockerbuildScripts: + type: array + items: + $ref: "#/components/schemas/BuildScript" + DockerBuildConfig: + type: object + properties: + gitMaterialId: + type: integer + dockerfileRelativePath: + type: string + targetPlatform: + type: string + args: + type: array + items: + type: object + properties: + Key: + type: string + Value: + type: string + description: map of docker arguments, i.e. key-value pairs + CiPipelineMaterialConfig: + type: object + properties: + type: + type: string + value: + type: string + checkoutPath: + type: string + BuildScript: + type: object + properties: + index: + type: integer + name: + type: string + script: + type: string + reportDirectoryPath: + type: string diff --git a/wire_gen.go b/wire_gen.go index 71f1772f19..84f8cd7a17 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -456,6 +456,7 @@ func InitializeApp() (*App, error) { cdApplicationStatusUpdateHandlerImpl := cron.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, cdHandlerImpl, appStatusConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl) appListingRestHandlerImpl := restHandler.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, clusterServiceImplExtended, helmAppServiceImpl, argoUserServiceImpl, k8sApplicationServiceImpl, installedAppServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl) appListingRouterImpl := router.NewAppListingRouterImpl(appListingRestHandlerImpl) + jobListingRouterImpl := router.NewJobRouterImpl(pipelineConfigRestHandlerImpl, appListingRestHandlerImpl) chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sUtil, clusterServiceImplExtended, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) deleteServiceExtendedImpl := delete2.NewDeleteServiceExtendedImpl(sugaredLogger, teamServiceImpl, clusterServiceImplExtended, environmentServiceImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, chartRepositoryServiceImpl, installedAppRepositoryImpl) environmentRestHandlerImpl := cluster3.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl) @@ -683,7 +684,7 @@ func InitializeApp() (*App, error) { return nil, err } ciStatusUpdateCronImpl := cron.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, ciHandlerImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusUpdateHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, ciStatusUpdateCronImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusUpdateHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, *jobListingRouterImpl, ciStatusUpdateCronImpl) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient) return mainApp, nil } From 27b9a0bb5c0093a81361b9612b0d81a59028fab2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 18:02:31 +0530 Subject: [PATCH 039/118] Updated the api end points and wrote the api specs. --- specs/jobs.yaml | 159 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 155 insertions(+), 4 deletions(-) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 52ae664b69..65264da17b 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -3,7 +3,7 @@ info: title: Jobs version: "1.0" paths: - /orchestrator/app/jobs: + /orchestrator/job: post: description: Create and clones a job requestBody: @@ -20,8 +20,8 @@ paths: application/json: schema: $ref: "#/components/schemas/ActionResponse" - /orchestrator/app/jobs/list: - get: + /orchestrator/job/list: + post: description: Get the list of all the jobs by applying filter requestBody: required: true @@ -37,7 +37,51 @@ paths: schema: items: $ref: "#/components/schemas/JobListResponse" - + /orchestrator/job/ci-pipeline/list/{jobId}: + get: + description: fetch details of job ci-pipelines for the overview page + parameters: + - name: jobId + in: path + required: true + schema: + type: integer + /orchestrator/job//ci-pipeline/patch: + post: + description: Create/Update ci pipeline. + operationId: PatchCiPipeline + requestBody: + description: A JSON object containing the pipeline configuration + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CiPatchRequest" + responses: + "200": + description: Successfully return a message stating the operation is successful. + content: + application/json: + schema: + type: string + "400": + description: Bad Request. Validation error/wrong request body. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + description: Internal Server Error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "403": + description: Unauthorized User + content: + application/json: + schema: + $ref: "#/components/schemas/Error" #components components: schemas: @@ -176,6 +220,113 @@ components: type: time.Time jobCount: type: integer + Error: + required: + - code + - message + properties: + code: + type: integer + description: Error code + message: + type: string + description: Error message + CiPatchRequest: + type: object + properties: + action: + type: integer + appId: + type: integer + isJob: + type: boolean + example: true + appWorkflowId: + type: integer + ciPipeline: + $ref: "#/components/schemas/CiPipelineDetails" + CiPipelineDetails: + type: object + properties: + isDockerConfigOverridden: + type: boolean + dockerConfigOverride: + $ref: "#/components/schemas/DockerBuildConfig" + name: + type: string + isManual: + type: boolean + scanEnabled: + type: boolean + isExternal: + type: boolean + parentAppId: + type: integer + parentCiPipeline: + type: integer + linkedCount: + type: integer + ciMaterials: + type: array + items: + $ref: "#/components/schemas/CiPipelineMaterialConfig" + dockerArgs: + type: array + items: + type: object + properties: + Key: + type: string + Value: + type: string + description: map of docker arguments, i.e. key-value pairs + beforeDockerBuildScripts: + type: array + items: + $ref: "#/components/schemas/BuildScript" + afterDockerbuildScripts: + type: array + items: + $ref: "#/components/schemas/BuildScript" + DockerBuildConfig: + type: object + properties: + gitMaterialId: + type: integer + dockerfileRelativePath: + type: string + targetPlatform: + type: string + args: + type: array + items: + type: object + properties: + Key: + type: string + Value: + type: string + description: map of docker arguments, i.e. key-value pairs + CiPipelineMaterialConfig: + type: object + properties: + type: + type: string + value: + type: string + checkoutPath: + type: string + BuildScript: + type: object + properties: + index: + type: integer + name: + type: string + script: + type: string + reportDirectoryPath: + type: string From 818497c22fec99b3036cfe4980a571e1c3af59d5 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 18:11:47 +0530 Subject: [PATCH 040/118] Updated the api end points and wrote the api specs. --- scripts/sql/115_jobs.down.sql | 4 +++- scripts/sql/115_jobs.up.sql | 4 +++- scripts/sql/116_job_description.down.sql | 2 -- scripts/sql/116_job_description.up.sql | 14 -------------- 4 files changed, 6 insertions(+), 18 deletions(-) delete mode 100644 scripts/sql/116_job_description.down.sql delete mode 100644 scripts/sql/116_job_description.up.sql diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/115_jobs.down.sql index 02ab78bd81..cbc3124e66 100644 --- a/scripts/sql/115_jobs.down.sql +++ b/scripts/sql/115_jobs.down.sql @@ -1,3 +1,5 @@ ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=1 THEN true ELSE false end; -ALTER TABLE app ALTER COLUMN app_store SET DEFAULT FALSE; \ No newline at end of file +ALTER TABLE app ALTER COLUMN app_store SET DEFAULT FALSE; +ALTER TABLE app DROP COLUMN display_name; +ALTER TABLE app DROP COLUMN description; \ No newline at end of file diff --git a/scripts/sql/115_jobs.up.sql b/scripts/sql/115_jobs.up.sql index f97f801545..6bae49696d 100644 --- a/scripts/sql/115_jobs.up.sql +++ b/scripts/sql/115_jobs.up.sql @@ -1,3 +1,5 @@ ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=true THEN 1 ELSE 0 end; -ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; \ No newline at end of file +ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; +ALTER TABLE app ADD COLUMN display_name varchar(250); +ALTER TABLE app ADD COLUMN description text; diff --git a/scripts/sql/116_job_description.down.sql b/scripts/sql/116_job_description.down.sql deleted file mode 100644 index 3b8b4cf26d..0000000000 --- a/scripts/sql/116_job_description.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE app DROP COLUMN display_name; -ALTER TABLE app DROP COLUMN description; \ No newline at end of file diff --git a/scripts/sql/116_job_description.up.sql b/scripts/sql/116_job_description.up.sql deleted file mode 100644 index 90a5d4bfc0..0000000000 --- a/scripts/sql/116_job_description.up.sql +++ /dev/null @@ -1,14 +0,0 @@ -ALTER TABLE app ADD COLUMN display_name varchar(250); -ALTER TABLE app ADD COLUMN description text; - - - - - - - - - - - - From 0adfb31e9488ce882c2ab95a265595b224c657ff Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 19:09:02 +0530 Subject: [PATCH 041/118] Updated the api end points and wrote the api specs. --- internal/sql/repository/app/AppRepository.go | 4 ++-- pkg/pipeline/CiCdPipelineOrchestrator.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index ed7f88fb04..2b13a2369a 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -46,7 +46,7 @@ type AppRepository interface { Update(app *App) error UpdateWithTxn(app *App, tx *pg.Tx) error FindActiveByName(appName string) (pipelineGroup *App, err error) - FindJobByName(appName string) (pipelineGroup *App, err error) + FindJobByDisplayName(appName string) (pipelineGroup *App, err error) FindActiveListByName(appName string) ([]*App, error) FindById(id int) (pipelineGroup *App, err error) FindAppsByTeamId(teamId int) ([]*App, error) @@ -122,7 +122,7 @@ func (repo AppRepositoryImpl) FindActiveByName(appName string) (*App, error) { // there is only single active app will be present in db with a same name. return pipelineGroup, err } -func (repo AppRepositoryImpl) FindJobByName(appName string) (*App, error) { +func (repo AppRepositoryImpl) FindJobByDisplayName(appName string) (*App, error) { pipelineGroup := &App{} err := repo.dbConnection. Model(pipelineGroup). diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index a544924c19..aa05a7ad41 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -1012,7 +1012,7 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 return nil, err } if isJob { - job, err := impl.appRepository.FindJobByName(name) + job, err := impl.appRepository.FindJobByDisplayName(name) if err != nil && err != pg.ErrNoRows { return nil, err } From e71d643d7b964abdc681af6ceb182e873043cc8f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 22 Feb 2023 20:31:07 +0530 Subject: [PATCH 042/118] wire err fix --- api/router/JobsRouter.go | 4 ++-- api/router/router.go | 2 +- cmd/external-app/wire_gen.go | 6 ++++-- wire_gen.go | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/api/router/JobsRouter.go b/api/router/JobsRouter.go index 834bee1c65..7624dd1543 100644 --- a/api/router/JobsRouter.go +++ b/api/router/JobsRouter.go @@ -15,11 +15,11 @@ type JobRouterImpl struct { } func NewJobRouterImpl(pipelineConfigRestHandler app.PipelineConfigRestHandler, appListingRestHandler restHandler.AppListingRestHandler) *JobRouterImpl { - router := &JobRouterImpl{ + return &JobRouterImpl{ appListingRestHandler: appListingRestHandler, pipelineConfigRestHandler: pipelineConfigRestHandler, } - return router + //return router } func (router JobRouterImpl) InitJobRouter(jobRouter *mux.Router) { jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateJob).Methods("POST") diff --git a/api/router/router.go b/api/router/router.go index be065245c8..575021fce2 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -145,7 +145,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter PipelineTriggerRouter, P helmApplicationStatusUpdateHandler cron.CdApplicationStatusUpdateHandler, k8sCapacityRouter k8s.K8sCapacityRouter, webhookHelmRouter webhookHelm.WebhookHelmRouter, globalCMCSRouter GlobalCMCSRouter, userTerminalAccessRouter terminal2.UserTerminalAccessRouter, - jobRouter JobRouterImpl, ciStatusUpdateCron cron.CiStatusUpdateCron) *MuxRouter { + jobRouter JobRouter, ciStatusUpdateCron cron.CiStatusUpdateCron) *MuxRouter { r := &MuxRouter{ Router: mux.NewRouter(), HelmRouter: HelmRouter, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 28838b9eb9..14055e8a11 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -34,6 +34,7 @@ import ( "github.com/devtron-labs/devtron/client/telemetry" repository4 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/apiToken" @@ -149,7 +150,8 @@ func InitializeApp() (*App, error) { v := informer.NewGlobalMapClusterNamespace() k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, runtimeConfig) clusterServiceImpl := cluster.NewClusterServiceImpl(clusterRepositoryImpl, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) - environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db) + appStatusRepositoryImpl:= appStatus.NewAppStatusRepositoryImpl(db,sugaredLogger) + environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db, sugaredLogger, appStatusRepositoryImpl) environmentServiceImpl := cluster.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImpl, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthServiceImpl) chartRepoRepositoryImpl := chartRepoRepository.NewChartRepoRepositoryImpl(db) acdAuthConfig, err := util3.GetACDAuthConfig() @@ -193,7 +195,7 @@ func InitializeApp() (*App, error) { serverDataStoreServerDataStore := serverDataStore.InitServerDataStore() appStoreApplicationVersionRepositoryImpl := appStoreDiscoverRepository.NewAppStoreApplicationVersionRepositoryImpl(sugaredLogger, db) pipelineRepositoryImpl := pipelineConfig.NewPipelineRepositoryImpl(db, sugaredLogger) - helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl) + helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl,clusterRepositoryImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, installedAppRepositoryImpl) attributesRepositoryImpl := repository4.NewAttributesRepositoryImpl(db) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) diff --git a/wire_gen.go b/wire_gen.go index 84f8cd7a17..cade56923d 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -456,7 +456,6 @@ func InitializeApp() (*App, error) { cdApplicationStatusUpdateHandlerImpl := cron.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppServiceImpl, cdHandlerImpl, appStatusConfig, pubSubClientServiceImpl, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl) appListingRestHandlerImpl := restHandler.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, clusterServiceImplExtended, helmAppServiceImpl, argoUserServiceImpl, k8sApplicationServiceImpl, installedAppServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl) appListingRouterImpl := router.NewAppListingRouterImpl(appListingRestHandlerImpl) - jobListingRouterImpl := router.NewJobRouterImpl(pipelineConfigRestHandlerImpl, appListingRestHandlerImpl) chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sUtil, clusterServiceImplExtended, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) deleteServiceExtendedImpl := delete2.NewDeleteServiceExtendedImpl(sugaredLogger, teamServiceImpl, clusterServiceImplExtended, environmentServiceImpl, appRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, chartRepositoryServiceImpl, installedAppRepositoryImpl) environmentRestHandlerImpl := cluster3.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl) @@ -679,12 +678,13 @@ func InitializeApp() (*App, error) { } userTerminalAccessRestHandlerImpl := terminal2.NewUserTerminalAccessRestHandlerImpl(sugaredLogger, userTerminalAccessServiceImpl, enforcerImpl, userServiceImpl, validate) userTerminalAccessRouterImpl := terminal2.NewUserTerminalAccessRouterImpl(userTerminalAccessRestHandlerImpl) + jobRouterImpl := router.NewJobRouterImpl(pipelineConfigRestHandlerImpl, appListingRestHandlerImpl) ciWorkflowStatusUpdateConfig, err := cron.GetCiWorkflowStatusUpdateConfig() if err != nil { return nil, err } ciStatusUpdateCronImpl := cron.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, ciHandlerImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusUpdateHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, *jobListingRouterImpl, ciStatusUpdateCronImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, pipelineTriggerRouterImpl, pipelineConfigRouterImpl, migrateDbRouterImpl, appListingRouterImpl, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, applicationRouterImpl, cdRouterImpl, projectManagementRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusUpdateHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, testSuitRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient) return mainApp, nil } From 7f018612f228d6c7a275f25dcdddeecc8b3a4904 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 10:52:45 +0530 Subject: [PATCH 043/118] Wrote the sql migrate down command. --- scripts/sql/115_jobs.down.sql | 100 +++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 2 deletions(-) diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/115_jobs.down.sql index cbc3124e66..ec531d6b0d 100644 --- a/scripts/sql/115_jobs.down.sql +++ b/scripts/sql/115_jobs.down.sql @@ -1,5 +1,101 @@ +ALTER TABLE app_workflow DROP CONSTRAINT app_workflow_app_id_fkey, ADD CONSTRAINT app_workflow_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; + +ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_app_id_fkey, ADD CONSTRAINT ci_pipeline_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; + +ALTER TABLE ci_template DROP CONSTRAINT ci_template_app_id_fkey, ADD CONSTRAINT ci_template_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; + +ALTER TABLE git_material DROP CONSTRAINT git_material_app_id_fkey, ADD CONSTRAINT git_material_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; + +ALTER TABLE app_label DROP CONSTRAINT app_label_app_id_fkey, ADD CONSTRAINT app_label_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; + +ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_history_app_id_fkey, ADD CONSTRAINT ci_template_history_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; + +ALTER TABLE app_workflow_mapping DROP CONSTRAINT app_workflow_mapping_app_workflow_id_fkey, ADD CONSTRAINT app_workflow_mapping_app_workflow_id_fkey FOREIGN KEY (app_workflow_id) REFERENCES app_workflow(id) ON DELETE CASCADE; + +ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey, ADD CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id) ON DELETE CASCADE; + +ALTER TABLE ci_pipeline_history DROP CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk, ADD CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id) ON DELETE CASCADE; + +ALTER TABLE git_material_history DROP CONSTRAINT git_material_history_git_material_id_fkey, ADD CONSTRAINT git_material_history_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; + +ALTER TABLE ci_workflow DROP CONSTRAINT ci_workflow_ci_pipeline_id_fkey, ADD CONSTRAINT ci_workflow_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id) ON DELETE CASCADE; + +ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_git_material_id_fkey, ADD CONSTRAINT ci_pipeline_material_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; + +ALTER TABLE ci_template DROP CONSTRAINT ci_template_git_material_id_fkey, ADD CONSTRAINT ci_template_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; + +ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_git_material_history_id_fkey, ADD CONSTRAINT ci_template_git_material_history_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; + +ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_ci_template_id_fkey, ADD CONSTRAINT ci_pipeline_ci_template_id_fkey FOREIGN KEY (ci_template_id) REFERENCES ci_template(id) ON DELETE CASCADE; + +DELETE FROM app WHERE app_store = 2; + ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; -ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=1 THEN true ELSE false end; + +ALTER TABLE app ALTER app_store TYPE boolean USING CASE WHEN app_store=1 THEN true ELSE false end; + ALTER TABLE app ALTER COLUMN app_store SET DEFAULT FALSE; + ALTER TABLE app DROP COLUMN display_name; -ALTER TABLE app DROP COLUMN description; \ No newline at end of file + +ALTER TABLE app DROP COLUMN description; + +ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_ci_template_id_fkey; + +ALTER TABLE ci_pipeline ADD CONSTRAINT ci_pipeline_ci_template_id_fkey FOREIGN KEY (ci_template_id) REFERENCES ci_template(id); + +ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_git_material_history_id_fkey; + +ALTER TABLE ci_template_history ADD CONSTRAINT ci_template_git_material_history_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); + +ALTER TABLE ci_template DROP CONSTRAINT ci_template_git_material_id_fkey; + +ALTER TABLE ci_template ADD CONSTRAINT ci_template_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); + +ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_git_material_id_fkey; + +ALTER TABLE ci_pipeline_material ADD CONSTRAINT ci_pipeline_material_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); + +ALTER TABLE ci_workflow DROP CONSTRAINT ci_workflow_ci_pipeline_id_fkey; + +ALTER TABLE ci_workflow ADD CONSTRAINT ci_workflow_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id); + +ALTER TABLE git_material_history DROP CONSTRAINT git_material_history_git_material_id_fkey; + +ALTER TABLE git_material_history ADD CONSTRAINT git_material_history_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); + +ALTER TABLE ci_pipeline_history DROP CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk; + +ALTER TABLE ci_pipeline_history ADD CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id); + +ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey; + +ALTER TABLE ci_pipeline_material ADD CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id); + +ALTER TABLE app_workflow_mapping DROP CONSTRAINT app_workflow_mapping_app_workflow_id_fkey; + +ALTER TABLE app_workflow_mapping ADD CONSTRAINT app_workflow_mapping_app_workflow_id_fkey FOREIGN KEY (app_workflow_id) REFERENCES app_workflow(id); + +ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_history_app_id_fkey; + +ALTER TABLE ci_template_history ADD CONSTRAINT ci_template_history_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); + +ALTER TABLE app_label DROP CONSTRAINT app_label_app_id_fkey; + +ALTER TABLE app_label ADD CONSTRAINT app_label_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); + +ALTER TABLE git_material DROP CONSTRAINT git_material_app_id_fkey; + +ALTER TABLE git_material ADD CONSTRAINT git_material_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); + +ALTER TABLE ci_template DROP CONSTRAINT ci_template_app_id_fkey; + +ALTER TABLE git_material ADD CONSTRAINT ci_template_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); + +ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_app_id_fkey; + +ALTER TABLE ci_pipeline ADD CONSTRAINT ci_pipeline_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); + +ALTER TABLE app_workflow DROP CONSTRAINT app_workflow_app_id_fkey; + +ALTER TABLE app_workflow ADD CONSTRAINT app_workflow_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); \ No newline at end of file From b12d9d7a762385ba366bbaeb7aadf6930e98c047 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 11:02:28 +0530 Subject: [PATCH 044/118] Removed the extra api endpoints. --- api/router/AppListingRouter.go | 4 ---- api/router/PipelineConfigRouter.go | 1 - 2 files changed, 5 deletions(-) diff --git a/api/router/AppListingRouter.go b/api/router/AppListingRouter.go index 0f51031911..de115f96eb 100644 --- a/api/router/AppListingRouter.go +++ b/api/router/AppListingRouter.go @@ -48,10 +48,6 @@ func (router AppListingRouterImpl) initAppListingRouter(appListingRouter *mux.Ro HandlerFunc(router.appListingRestHandler.FetchAppsByEnvironment). Methods("POST") - appListingRouter.Path("/jobs/list"). - HandlerFunc(router.appListingRestHandler.FetchJobs). - Methods("POST") - //This API used for fetch app details, not deployment details appListingRouter.Path("/detail").Queries("app-id", "{app-id}").Queries("env-id", "{env-id}"). HandlerFunc(router.appListingRestHandler.FetchAppDetails). diff --git a/api/router/PipelineConfigRouter.go b/api/router/PipelineConfigRouter.go index 79942d3e9a..9f7ad84c80 100644 --- a/api/router/PipelineConfigRouter.go +++ b/api/router/PipelineConfigRouter.go @@ -51,7 +51,6 @@ func NewPipelineRouterImpl(restHandler app.PipelineConfigRestHandler, func (router PipelineConfigRouterImpl) initPipelineConfigRouter(configRouter *mux.Router) { configRouter.Path("").HandlerFunc(router.restHandler.CreateApp).Methods("POST") - configRouter.Path("/jobs").HandlerFunc(router.restHandler.CreateJob).Methods("POST") configRouter.Path("/{appId}").HandlerFunc(router.restHandler.DeleteApp).Methods("DELETE") configRouter.Path("/material").HandlerFunc(router.restHandler.CreateMaterial).Methods("POST") configRouter.Path("/material").HandlerFunc(router.restHandler.UpdateMaterial).Methods("PUT") From 7a3425e710e7ba0ba39e0eb82bd1a971744850d0 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 11:10:47 +0530 Subject: [PATCH 045/118] Removed the commented code --- api/bean/AppView.go | 1 - api/restHandler/AppListingRestHandler.go | 10 ---------- api/restHandler/app/PipelineConfigRestHandler.go | 10 ---------- 3 files changed, 21 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index e4da0e82d2..df2ffb8c1d 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -81,7 +81,6 @@ type JobListingContainer struct { CiPipelineName string `json:"ci_pipeline_name"` Status string `json:"status"` StartedOn time.Time `json:"started_on"` - //LastSuccessAt time.Time `json:"-"` } type CiPipelineLastSucceededTime struct { diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 55269e1dee..e02963ab58 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -180,16 +180,6 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt return } - //jobsCount := len(jobs) - //offset := fetchJobListingRequest.Offset - //limit := fetchJobListingRequest.Size - // - //if offset+limit <= len(jobs) { - // fetchJob = fetchJob[offset : offset+limit] - //} else { - // fetchJob = fetchJob[offset:] - //} - jobContainerResponse := bean.JobContainerResponse{ JobContainers: jobs, JobCount: len(jobs), diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index b4852f399f..d6b39745bc 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -315,16 +315,6 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r return } - //project, err := handler.teamService.FetchOne(createRequest.TeamId) - //if err != nil { - // common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - // return - //} - //// with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) - //if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, fmt.Sprintf("%s/%s", strings.ToLower(project.Name), "*")); !ok { - // common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - // return - //} var createResp *bean.CreateAppDTO err = nil createRequest.IsJob = true From 168fae6ac1aa0bfe34c295a6aa870a8aee0bc466 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 11:12:50 +0530 Subject: [PATCH 046/118] Removed the commented code. --- pkg/appClone/AppCloneService.go | 90 --------------------------------- 1 file changed, 90 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 5f923f11d7..1a214b0dcc 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -39,7 +39,6 @@ import ( type AppCloneService interface { CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) - //CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) } type AppCloneServiceImpl struct { logger *zap.SugaredLogger @@ -215,95 +214,6 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context return app, nil } -//func (impl *AppCloneServiceImpl) CloneJob(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { -// //validate template job -// templateApp, err := impl.appRepository.FindById(createReq.TemplateId) -// if err != nil && err != pg.ErrNoRows { -// return nil, err -// } -// if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppStore != 2) { -// impl.logger.Warnw("template job does not exist", "id", createReq.TemplateId) -// err = &util.ApiError{ -// Code: constants.AppDoesNotExist.Code, -// InternalMessage: "job does not exist", -// UserMessage: constants.AppAlreadyExists.UserMessage(createReq.TemplateId), -// } -// return nil, err -// } -// //create new job -// -// cloneReq := &CloneRequest{ -// RefAppId: createReq.TemplateId, -// Name: createReq.AppName, -// ProjectId: createReq.TeamId, -// AppLabels: createReq.AppLabels, -// } -// userId := createReq.UserId -// appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) -// if err != nil { -// return nil, err -// } -// refApp, err := impl.pipelineBuilder.GetApp(cloneReq.RefAppId) -// if err != nil { -// return nil, err -// } -// isSameProject := refApp.TeamId == cloneReq.ProjectId -// /* appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId)) -// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(1, "MATERIAL", materialExists)) -// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId)) -// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(3, "CI_PIPELINE", stages.CiPipelineId)) -// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(4, "CHART", stages.ChartId)) -// appStageStatus = append(appStageStatus, impl.makeAppStageStatus(5, "CD_PIPELINE", stages.PipelineId)) -// */ -// refAppStatus := make(map[string]bool) -// for _, as := range appStatus { -// refAppStatus[as.StageName] = as.Status -// } -// -// //TODO check stage of current app -// if !refAppStatus["APP"] { -// impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) -// return nil, nil -// } -// app, err := impl.CreateApp(cloneReq, userId) -// if err != nil { -// impl.logger.Errorw("error in creating app", "req", cloneReq, "err", err) -// return nil, err -// } -// newAppId := app.Id -// if !refAppStatus["MATERIAL"] { -// impl.logger.Errorw("status not", "MATERIAL", cloneReq.RefAppId) -// return app, nil -// } -// _, gitMaerialMap, err := impl.CloneGitRepo(cloneReq.RefAppId, newAppId, userId) -// if err != nil { -// impl.logger.Errorw("error in cloning git", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) -// return nil, err -// } -// -// _, err = impl.CreateCiTemplate(cloneReq.RefAppId, newAppId, userId) -// if err != nil { -// impl.logger.Errorw("error in cloning docker template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) -// return nil, err -// } -// if !refAppStatus["TEMPLATE"] { -// impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) -// return app, nil -// } -// if !refAppStatus["CHART"] { -// impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) -// return app, nil -// } -// -// _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) -// if err != nil { -// impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) -// return nil, err -// } -// -// return app, nil -//} - func (impl *AppCloneServiceImpl) CreateApp(cloneReq *CloneRequest, userId int32) (*bean.CreateAppDTO, error) { createAppReq := &bean.CreateAppDTO{ AppName: cloneReq.Name, From c5ab938d934517ff012c0cf0627f9f1262629efc Mon Sep 17 00:00:00 2001 From: ShashwatDadhich <92629050+ShashwatDadhich@users.noreply.github.com> Date: Thu, 23 Feb 2023 11:28:17 +0530 Subject: [PATCH 047/118] Update version.go --- util/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/version.go b/util/version.go index 417b3fb010..f64cb32956 100644 --- a/util/version.go +++ b/util/version.go @@ -31,7 +31,7 @@ type ServerVersion struct { } func GetDevtronVersion() *ServerVersion { - return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: SERVER_MODE_FULL} + return &ServerVersion{BuildTime: BuildTime, GitCommit: GitCommit, ServerMode: ServerMode} } func IsBaseStack() bool { From 2421d844fe16fc1183180a131e3db0f7906920c8 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 11:45:24 +0530 Subject: [PATCH 048/118] Added logger wherever required. --- api/restHandler/AppListingRestHandler.go | 7 +++++++ .../app/PipelineConfigRestHandler.go | 4 ++++ pkg/pipeline/CiService.go | 19 +++---------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index e02963ab58..3ee6a18209 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -161,6 +161,7 @@ func (handler AppListingRestHandlerImpl) FetchAllDevtronManagedApps(w http.Respo func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *http.Request) { userId, err := handler.userService.GetLoggedInUser(r) if userId == 0 || err != nil { + handler.logger.Errorw("request err, userId", "err", err, "payload", userId) common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } @@ -190,6 +191,7 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt func (handler AppListingRestHandlerImpl) FetchOverviewCiPipelines(w http.ResponseWriter, r *http.Request) { userId, err := handler.userService.GetLoggedInUser(r) if userId == 0 || err != nil { + handler.logger.Errorw("request err, userId", "err", err, "payload", userId) common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } @@ -203,6 +205,11 @@ func (handler AppListingRestHandlerImpl) FetchOverviewCiPipelines(w http.Respons } jobCi, err := handler.appListingService.FetchOverviewCiPipelines(jobId) + if err != nil { + handler.logger.Errorw("request err, GetJobCi", "err", jobCi, "jobCi", jobCi) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } common.WriteJsonResp(w, err, jobCi, http.StatusOK) } diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index d6b39745bc..3d6c8d4941 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -288,12 +288,16 @@ func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { + handler.Logger.Errorw("request err, GetUserId", "err", userId, "userId", userId) common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index f54e79eee7..3463aad8f6 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -443,22 +443,9 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) } workflowRequest := &WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - //DockerRegistryId: dockerRegistry.Id, - //DockerRegistryType: string(dockerRegistry.RegistryType), - //DockerImageTag: dockerImageTag, - //DockerRegistryURL: dockerRegistry.RegistryURL, - //DockerRepository: dockerRepository, - //CheckoutPath: checkoutPath, - //DockerUsername: dockerRegistry.Username, - //DockerPassword: dockerRegistry.Password, - //AwsRegion: dockerRegistry.AWSRegion, - //AccessKey: dockerRegistry.AWSAccessKeyId, - //SecretKey: dockerRegistry.AWSSecretAccessKey, - //DockerConnection: dockerRegistry.Connection, - //DockerCert: dockerRegistry.Cert, + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", CiProjectDetails: ciProjectDetails, Namespace: ciWorkflowConfig.Namespace, From 4639c989c6438b2a1b14f4a073eff964dc2837af Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 12:02:52 +0530 Subject: [PATCH 049/118] Fixed a bug. --- pkg/pipeline/CiCdPipelineOrchestrator.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index aa05a7ad41..cf43a55d04 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -1002,14 +1002,16 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 if err != nil && err != pg.ErrNoRows { return nil, err } - if app != nil && app.Id > 0 { - impl.logger.Warnw("app already exists", "name", name) - err = &util.ApiError{ - Code: constants.AppAlreadyExists.Code, - InternalMessage: "app already exists", - UserMessage: constants.AppAlreadyExists.UserMessage(name), + if !isJob { + if app != nil && app.Id > 0 { + impl.logger.Warnw("app already exists", "name", name) + err = &util.ApiError{ + Code: constants.AppAlreadyExists.Code, + InternalMessage: "app already exists", + UserMessage: constants.AppAlreadyExists.UserMessage(name), + } + return nil, err } - return nil, err } if isJob { job, err := impl.appRepository.FindJobByDisplayName(name) From 1002ad061ea8740570d3854ec684e7d23a11d02c Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 23 Feb 2023 12:19:15 +0530 Subject: [PATCH 050/118] Updated the api specs. --- specs/jobs.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 65264da17b..3e5e1febdc 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -116,10 +116,6 @@ components: type: string description: Used to give the description of the job once it is made. example: "This is my first Job" - isJob: - type: boolean - description: Used to state whether it is an app or a job - example: true ActionResponse: name: result type: object From f9e3c148d99b4cba0b06ffa9283105798c8af600 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 12:18:32 +0530 Subject: [PATCH 051/118] Added the user authentications --- api/restHandler/AppListingRestHandler.go | 18 ++++++++++++++++-- .../app/BuildPipelineRestHandler.go | 8 ++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 3ee6a18209..7b978c49c1 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -165,7 +165,14 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - + isSuperAdmin, err := handler.userService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } var fetchJobListingRequest app.FetchAppListingRequest decoder := json.NewDecoder(r.Body) err = decoder.Decode(&fetchJobListingRequest) @@ -195,7 +202,14 @@ func (handler AppListingRestHandlerImpl) FetchOverviewCiPipelines(w http.Respons common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - + isSuperAdmin, err := handler.userService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } vars := mux.Vars(r) jobId, err := strconv.Atoi(vars["jobId"]) if err != nil { diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 871a2c3906..fdfdc24186 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -207,6 +207,14 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } var patchRequest bean.CiPatchRequest err = decoder.Decode(&patchRequest) patchRequest.UserId = userId From efdd49672a82cc3ec73d86e01a1682db20fc739f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 15:12:36 +0530 Subject: [PATCH 052/118] Added the user authentications --- .../app/BuildPipelineRestHandler.go | 348 ++++++++++++++++-- api/router/JobsRouter.go | 6 +- 2 files changed, 330 insertions(+), 24 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index fdfdc24186..3e851eea3e 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -31,7 +31,9 @@ type DevtronAppBuildRestHandler interface { GetExternalCi(w http.ResponseWriter, r *http.Request) GetExternalCiById(w http.ResponseWriter, r *http.Request) PatchCiPipelines(w http.ResponseWriter, r *http.Request) + PatchJobCiPipelines(w http.ResponseWriter, r *http.Request) TriggerCiPipeline(w http.ResponseWriter, r *http.Request) + TriggerJobCiPipeline(w http.ResponseWriter, r *http.Request) GetCiPipelineMin(w http.ResponseWriter, r *http.Request) GetCIPipelineById(w http.ResponseWriter, r *http.Request) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) @@ -45,12 +47,15 @@ type DevtronAppBuildRestHandler interface { type DevtronAppBuildMaterialRestHandler interface { CreateMaterial(w http.ResponseWriter, r *http.Request) + CreateJobMaterial(w http.ResponseWriter, r *http.Request) UpdateMaterial(w http.ResponseWriter, r *http.Request) + UpdateJobMaterial(w http.ResponseWriter, r *http.Request) FetchMaterials(w http.ResponseWriter, r *http.Request) RefreshMaterials(w http.ResponseWriter, r *http.Request) FetchMaterialInfo(w http.ResponseWriter, r *http.Request) FetchChanges(w http.ResponseWriter, r *http.Request) DeleteMaterial(w http.ResponseWriter, r *http.Request) + DeleteJobMaterial(w http.ResponseWriter, r *http.Request) GetCommitMetadataForPipelineMaterial(w http.ResponseWriter, r *http.Request) } @@ -201,6 +206,74 @@ func (handler PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w } func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWriter, r *http.Request) { + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var patchRequest bean.CiPatchRequest + err = decoder.Decode(&patchRequest) + patchRequest.UserId = userId + if err != nil { + handler.Logger.Errorw("request err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.Logger.Infow("request payload, PatchCiPipelines", "PatchCiPipelines", patchRequest) + err = handler.validator.Struct(patchRequest) + if err != nil { + handler.Logger.Errorw("validation err", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.Logger.Debugw("update request ", "req", patchRequest) + token := r.Header.Get("token") + app, err := handler.pipelineBuilder.GetApp(patchRequest.AppId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) + var ok bool + if patchRequest.IsJob { + ok = handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionCreate, resourceName) + } else { + ok = handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName) + } + if !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } + + pipelineData, err := handler.pipelineRepository.FindActiveByAppIdAndPipelineId(patchRequest.AppId, patchRequest.CiPipeline.Id) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + var environmentIds []int + for _, pipeline := range pipelineData { + environmentIds = append(environmentIds, pipeline.EnvironmentId) + } + if handler.appWorkflowService.CheckCdPipelineByCiPipelineId(patchRequest.CiPipeline.Id) { + for _, envId := range environmentIds { + envObject := handler.enforcerUtil.GetEnvRBACNameByCiPipelineIdAndEnvId(patchRequest.CiPipeline.Id, envId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, envObject); !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return + } + } + } + createResp, err := handler.pipelineBuilder.PatchCiPipeline(&patchRequest) + if err != nil { + handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, createResp, http.StatusOK) +} + +func (handler PipelineConfigRestHandlerImpl) PatchJobCiPipelines(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { @@ -223,7 +296,6 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - handler.Logger.Infow("request payload, PatchCiPipelines", "PatchCiPipelines", patchRequest) err = handler.validator.Struct(patchRequest) if err != nil { @@ -238,7 +310,6 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) var ok bool if patchRequest.IsJob { @@ -276,26 +347,6 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri } } - if !patchRequest.IsJob { - pipelineData, err := handler.pipelineRepository.FindActiveByAppIdAndPipelineId(patchRequest.AppId, patchRequest.CiPipeline.Id) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - var environmentIds []int - for _, pipeline := range pipelineData { - environmentIds = append(environmentIds, pipeline.EnvironmentId) - } - if handler.appWorkflowService.CheckCdPipelineByCiPipelineId(patchRequest.CiPipeline.Id) { - for _, envId := range environmentIds { - envObject := handler.enforcerUtil.GetEnvRBACNameByCiPipelineIdAndEnvId(patchRequest.CiPipeline.Id, envId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, envObject); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - } - } - } createResp, err := handler.pipelineBuilder.PatchCiPipeline(&patchRequest) createResp.AppName = app.AppName if err != nil { @@ -303,7 +354,6 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - common.WriteJsonResp(w, err, createResp, http.StatusOK) } @@ -481,6 +531,93 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, response, http.StatusOK) } +func (handler PipelineConfigRestHandlerImpl) TriggerJobCiPipeline(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + decoder := json.NewDecoder(r.Body) + var ciTriggerRequest bean.CiTriggerRequest + err = decoder.Decode(&ciTriggerRequest) + if err != nil { + handler.Logger.Errorw("request err, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + if !handler.validForMultiMaterial(ciTriggerRequest) { + handler.Logger.Errorw("invalid req, commit hash not present for multi-git", "payload", ciTriggerRequest) + common.WriteJsonResp(w, errors.New("invalid req, commit hash not present for multi-git"), + nil, http.StatusBadRequest) + } + ciTriggerRequest.TriggeredBy = userId + handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) + + //RBAC CHECK CD PIPELINE - FOR USER + pipelines, err := handler.pipelineRepository.FindByCiPipelineId(ciTriggerRequest.PipelineId) + if err != nil { + handler.Logger.Errorw("error in finding ccd pipelines by ciPipelineId", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + var authorizedPipelines []pipelineConfig.Pipeline + var unauthorizedPipelines []pipelineConfig.Pipeline + token := r.Header.Get("token") + for _, p := range pipelines { + object := handler.enforcerUtil.GetAppRBACNameByAppId(p.AppId) + appRbacOk := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionTrigger, object) + object = handler.enforcerUtil.GetAppRBACByAppIdAndPipelineId(p.AppId, p.Id) + envRbacOk := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionTrigger, object) + if appRbacOk && envRbacOk { + authorizedPipelines = append(authorizedPipelines, *p) + } else { + unauthorizedPipelines = append(unauthorizedPipelines, *p) + } + } + resMessage := "allowed for all pipelines" + response := make(map[string]string) + if len(unauthorizedPipelines) > 0 { + resMessage = "some pipelines not authorized" + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + if len(authorizedPipelines) == 0 { + //user has no cd pipeline + ciPipeline, err := handler.ciPipelineRepository.FindById(ciTriggerRequest.PipelineId) + if err != nil { + handler.Logger.Errorw("err in finding ci pipeline, TriggerCiPipeline", "err", err, "ciPipelineId", ciTriggerRequest.PipelineId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionTrigger, object); !ok { + handler.Logger.Debug(fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + } + //RBAC CHECK CD PIPELINE - FOR USER + + resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) + if err != nil { + handler.Logger.Errorw("service err, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) + common.WriteJsonResp(w, err, response, http.StatusInternalServerError) + return + } + response["apiResponse"] = strconv.Itoa(resp) + response["authStatus"] = resMessage + + common.WriteJsonResp(w, err, response, http.StatusOK) +} + func (handler PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWriter, r *http.Request) { userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { @@ -944,6 +1081,66 @@ func (handler PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, createResp, http.StatusOK) } +func (handler PipelineConfigRestHandlerImpl) CreateJobMaterial(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("token") + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var createMaterialDto bean.CreateMaterialDTO + err = decoder.Decode(&createMaterialDto) + createMaterialDto.UserId = userId + if err != nil { + handler.Logger.Errorw("request err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.Logger.Infow("request payload, CreateMaterial", "CreateMaterial", createMaterialDto) + err = handler.validator.Struct(createMaterialDto) + if err != nil { + handler.Logger.Errorw("validation err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + for _, gitMaterial := range createMaterialDto.Material { + validationResult, err := handler.ValidateGitMaterialUrl(gitMaterial.GitProviderId, gitMaterial.Url) + if err != nil { + handler.Logger.Errorw("service err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } else { + if !validationResult { + handler.Logger.Errorw("validation err, CreateMaterial : invalid git material url", "err", err, "gitMaterialUrl", gitMaterial.Url, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, fmt.Errorf("validation for url failed"), nil, http.StatusBadRequest) + return + } + } + } + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(createMaterialDto.AppId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + + createResp, err := handler.pipelineBuilder.CreateMaterialsForApp(&createMaterialDto) + if err != nil { + handler.Logger.Errorw("service err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, createResp, http.StatusOK) +} + func (handler PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") decoder := json.NewDecoder(r.Body) @@ -994,6 +1191,64 @@ func (handler PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, createResp, http.StatusOK) } +func (handler PipelineConfigRestHandlerImpl) UpdateJobMaterial(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("token") + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var updateMaterialDto bean.UpdateMaterialDTO + err = decoder.Decode(&updateMaterialDto) + updateMaterialDto.UserId = userId + if err != nil { + handler.Logger.Errorw("request err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.Logger.Infow("request payload, UpdateMaterial", "UpdateMaterial", updateMaterialDto) + err = handler.validator.Struct(updateMaterialDto) + if err != nil { + handler.Logger.Errorw("validation err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + validationResult, err := handler.ValidateGitMaterialUrl(updateMaterialDto.Material.GitProviderId, updateMaterialDto.Material.Url) + if err != nil { + handler.Logger.Errorw("service err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } else { + if !validationResult { + handler.Logger.Errorw("validation err, UpdateMaterial : invalid git material url", "err", err, "gitMaterialUrl", updateMaterialDto.Material.Url, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + } + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(updateMaterialDto.AppId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + + createResp, err := handler.pipelineBuilder.UpdateMaterialsForApp(&updateMaterialDto) + if err != nil { + handler.Logger.Errorw("service err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, createResp, http.StatusOK) +} + func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) @@ -1033,6 +1288,53 @@ func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, GIT_MATERIAL_DELETE_SUCCESS_RESP, http.StatusOK) } +func (handler PipelineConfigRestHandlerImpl) DeleteJobMaterial(w http.ResponseWriter, r *http.Request) { + decoder := json.NewDecoder(r.Body) + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + var deleteMaterial bean.UpdateMaterialDTO + err = decoder.Decode(&deleteMaterial) + deleteMaterial.UserId = userId + if err != nil { + handler.Logger.Errorw("request err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.Logger.Infow("request payload, DeleteMaterial", "DeleteMaterial", deleteMaterial) + err = handler.validator.Struct(deleteMaterial) + if err != nil { + handler.Logger.Errorw("validation err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + //rbac starts + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(deleteMaterial.AppId) + token := r.Header.Get("token") + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + //rbac ends + err = handler.pipelineBuilder.DeleteMaterial(&deleteMaterial) + if err != nil { + handler.Logger.Errorw("service err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, GIT_MATERIAL_DELETE_SUCCESS_RESP, http.StatusOK) +} + func (handler PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) var wfUpdateReq v1alpha1.WorkflowStatus diff --git a/api/router/JobsRouter.go b/api/router/JobsRouter.go index 7624dd1543..5adf0a2b61 100644 --- a/api/router/JobsRouter.go +++ b/api/router/JobsRouter.go @@ -23,7 +23,11 @@ func NewJobRouterImpl(pipelineConfigRestHandler app.PipelineConfigRestHandler, a } func (router JobRouterImpl) InitJobRouter(jobRouter *mux.Router) { jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateJob).Methods("POST") - jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchCiPipelines).Methods("POST") + jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchJobCiPipelines).Methods("POST") jobRouter.Path("/list").HandlerFunc(router.appListingRestHandler.FetchJobs).Methods("POST") jobRouter.Path("/ci-pipeline/list/{jobId}").HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipelines).Methods("GET") + jobRouter.Path("/material").HandlerFunc(router.pipelineConfigRestHandler.CreateJobMaterial).Methods("POST") + jobRouter.Path("/material").HandlerFunc(router.pipelineConfigRestHandler.UpdateJobMaterial).Methods("PUT") + jobRouter.Path("/material/delete").HandlerFunc(router.pipelineConfigRestHandler.DeleteJobMaterial).Methods("DELETE") + jobRouter.Path("/ci-pipeline/trigger").HandlerFunc(router.pipelineConfigRestHandler.TriggerJobCiPipeline).Methods("POST") } From 8b3471d342a933f6e3e7d767deb680bb5905ca9c Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 15:15:44 +0530 Subject: [PATCH 053/118] Added the user authentications --- api/restHandler/app/BuildPipelineRestHandler.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 3e851eea3e..1db88cfcba 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -235,13 +235,7 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri return } resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - var ok bool - if patchRequest.IsJob { - ok = handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionCreate, resourceName) - } else { - ok = handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName) - } - if !ok { + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) return } @@ -264,6 +258,7 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri } } } + createResp, err := handler.pipelineBuilder.PatchCiPipeline(&patchRequest) if err != nil { handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) From d0faf6089faeee97db199015e173e3dd40a4dfa2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 16:00:56 +0530 Subject: [PATCH 054/118] Added the user authentications --- .../app/BuildPipelineRestHandler.go | 251 +++--------------- api/router/JobsRouter.go | 6 +- 2 files changed, 35 insertions(+), 222 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 1db88cfcba..52ff1c2ae6 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -31,7 +31,6 @@ type DevtronAppBuildRestHandler interface { GetExternalCi(w http.ResponseWriter, r *http.Request) GetExternalCiById(w http.ResponseWriter, r *http.Request) PatchCiPipelines(w http.ResponseWriter, r *http.Request) - PatchJobCiPipelines(w http.ResponseWriter, r *http.Request) TriggerCiPipeline(w http.ResponseWriter, r *http.Request) TriggerJobCiPipeline(w http.ResponseWriter, r *http.Request) GetCiPipelineMin(w http.ResponseWriter, r *http.Request) @@ -47,15 +46,12 @@ type DevtronAppBuildRestHandler interface { type DevtronAppBuildMaterialRestHandler interface { CreateMaterial(w http.ResponseWriter, r *http.Request) - CreateJobMaterial(w http.ResponseWriter, r *http.Request) UpdateMaterial(w http.ResponseWriter, r *http.Request) - UpdateJobMaterial(w http.ResponseWriter, r *http.Request) FetchMaterials(w http.ResponseWriter, r *http.Request) RefreshMaterials(w http.ResponseWriter, r *http.Request) FetchMaterialInfo(w http.ResponseWriter, r *http.Request) FetchChanges(w http.ResponseWriter, r *http.Request) DeleteMaterial(w http.ResponseWriter, r *http.Request) - DeleteJobMaterial(w http.ResponseWriter, r *http.Request) GetCommitMetadataForPipelineMaterial(w http.ResponseWriter, r *http.Request) } @@ -234,77 +230,16 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - - pipelineData, err := handler.pipelineRepository.FindActiveByAppIdAndPipelineId(patchRequest.AppId, patchRequest.CiPipeline.Id) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - var environmentIds []int - for _, pipeline := range pipelineData { - environmentIds = append(environmentIds, pipeline.EnvironmentId) - } - if handler.appWorkflowService.CheckCdPipelineByCiPipelineId(patchRequest.CiPipeline.Id) { - for _, envId := range environmentIds { - envObject := handler.enforcerUtil.GetEnvRBACNameByCiPipelineIdAndEnvId(patchRequest.CiPipeline.Id, envId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionUpdate, envObject); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return } } - - createResp, err := handler.pipelineBuilder.PatchCiPipeline(&patchRequest) - if err != nil { - handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - -func (handler PipelineConfigRestHandlerImpl) PatchJobCiPipelines(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var patchRequest bean.CiPatchRequest - err = decoder.Decode(&patchRequest) - patchRequest.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, PatchCiPipelines", "PatchCiPipelines", patchRequest) - err = handler.validator.Struct(patchRequest) - if err != nil { - handler.Logger.Errorw("validation err", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Debugw("update request ", "req", patchRequest) - token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(patchRequest.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) var ok bool if patchRequest.IsJob { @@ -1034,66 +969,21 @@ func (handler PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrite } var createMaterialDto bean.CreateMaterialDTO err = decoder.Decode(&createMaterialDto) - createMaterialDto.UserId = userId + app, err := handler.pipelineBuilder.GetApp(createMaterialDto.AppId) if err != nil { - handler.Logger.Errorw("request err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - handler.Logger.Infow("request payload, CreateMaterial", "CreateMaterial", createMaterialDto) - err = handler.validator.Struct(createMaterialDto) - if err != nil { - handler.Logger.Errorw("validation err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - for _, gitMaterial := range createMaterialDto.Material { - validationResult, err := handler.ValidateGitMaterialUrl(gitMaterial.GitProviderId, gitMaterial.Url) - if err != nil { - handler.Logger.Errorw("service err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } else { - if !validationResult { - handler.Logger.Errorw("validation err, CreateMaterial : invalid git material url", "err", err, "gitMaterialUrl", gitMaterial.Url, "CreateMaterial", createMaterialDto) - common.WriteJsonResp(w, fmt.Errorf("validation for url failed"), nil, http.StatusBadRequest) - return + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return } } - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(createMaterialDto.AppId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - - createResp, err := handler.pipelineBuilder.CreateMaterialsForApp(&createMaterialDto) - if err != nil { - handler.Logger.Errorw("service err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - -func (handler PipelineConfigRestHandlerImpl) CreateJobMaterial(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get("token") - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var createMaterialDto bean.CreateMaterialDTO - err = decoder.Decode(&createMaterialDto) createMaterialDto.UserId = userId if err != nil { handler.Logger.Errorw("request err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) @@ -1146,64 +1036,21 @@ func (handler PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWrite } var updateMaterialDto bean.UpdateMaterialDTO err = decoder.Decode(&updateMaterialDto) - updateMaterialDto.UserId = userId + app, err := handler.pipelineBuilder.GetApp(updateMaterialDto.AppId) if err != nil { - handler.Logger.Errorw("request err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - handler.Logger.Infow("request payload, UpdateMaterial", "UpdateMaterial", updateMaterialDto) - err = handler.validator.Struct(updateMaterialDto) - if err != nil { - handler.Logger.Errorw("validation err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - validationResult, err := handler.ValidateGitMaterialUrl(updateMaterialDto.Material.GitProviderId, updateMaterialDto.Material.Url) - if err != nil { - handler.Logger.Errorw("service err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } else { - if !validationResult { - handler.Logger.Errorw("validation err, UpdateMaterial : invalid git material url", "err", err, "gitMaterialUrl", updateMaterialDto.Material.Url, "UpdateMaterial", updateMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } } - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(updateMaterialDto.AppId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - - createResp, err := handler.pipelineBuilder.UpdateMaterialsForApp(&updateMaterialDto) - if err != nil { - handler.Logger.Errorw("service err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - -func (handler PipelineConfigRestHandlerImpl) UpdateJobMaterial(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get("token") - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - var updateMaterialDto bean.UpdateMaterialDTO - err = decoder.Decode(&updateMaterialDto) updateMaterialDto.UserId = userId if err != nil { handler.Logger.Errorw("request err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) @@ -1253,53 +1100,23 @@ func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrite } var deleteMaterial bean.UpdateMaterialDTO err = decoder.Decode(&deleteMaterial) - deleteMaterial.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, DeleteMaterial", "DeleteMaterial", deleteMaterial) - err = handler.validator.Struct(deleteMaterial) + var updateMaterialDto bean.UpdateMaterialDTO + err = decoder.Decode(&updateMaterialDto) + app, err := handler.pipelineBuilder.GetApp(deleteMaterial.AppId) if err != nil { - handler.Logger.Errorw("validation err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - //rbac starts - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(deleteMaterial.AppId) - token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - //rbac ends - err = handler.pipelineBuilder.DeleteMaterial(&deleteMaterial) - if err != nil { - handler.Logger.Errorw("service err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, GIT_MATERIAL_DELETE_SUCCESS_RESP, http.StatusOK) -} - -func (handler PipelineConfigRestHandlerImpl) DeleteJobMaterial(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return } - var deleteMaterial bean.UpdateMaterialDTO - err = decoder.Decode(&deleteMaterial) deleteMaterial.UserId = userId if err != nil { handler.Logger.Errorw("request err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) diff --git a/api/router/JobsRouter.go b/api/router/JobsRouter.go index 5adf0a2b61..7624dd1543 100644 --- a/api/router/JobsRouter.go +++ b/api/router/JobsRouter.go @@ -23,11 +23,7 @@ func NewJobRouterImpl(pipelineConfigRestHandler app.PipelineConfigRestHandler, a } func (router JobRouterImpl) InitJobRouter(jobRouter *mux.Router) { jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateJob).Methods("POST") - jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchJobCiPipelines).Methods("POST") + jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchCiPipelines).Methods("POST") jobRouter.Path("/list").HandlerFunc(router.appListingRestHandler.FetchJobs).Methods("POST") jobRouter.Path("/ci-pipeline/list/{jobId}").HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipelines).Methods("GET") - jobRouter.Path("/material").HandlerFunc(router.pipelineConfigRestHandler.CreateJobMaterial).Methods("POST") - jobRouter.Path("/material").HandlerFunc(router.pipelineConfigRestHandler.UpdateJobMaterial).Methods("PUT") - jobRouter.Path("/material/delete").HandlerFunc(router.pipelineConfigRestHandler.DeleteJobMaterial).Methods("DELETE") - jobRouter.Path("/ci-pipeline/trigger").HandlerFunc(router.pipelineConfigRestHandler.TriggerJobCiPipeline).Methods("POST") } From d833dbf02306966f7d911420461edf785804151f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 16:57:05 +0530 Subject: [PATCH 055/118] Added the user authentications --- api/bean/AppView.go | 8 ++++---- .../helper/AppListingRepositoryQueryBuilder.go | 2 +- pkg/app/AppListingService.go | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index df2ffb8c1d..e90f42fccd 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -59,8 +59,8 @@ type CiMaterialDTO struct { } type JobContainer struct { - AppId int `json:"appId"` - AppName string `json:"appName""` + JobId int `json:"jobId"` + JobName string `json:"jobName""` Description string `json:"description"` JobCiPipelines []JobCIPipeline `json:"ciPipelines"'` } @@ -74,8 +74,8 @@ type JobCIPipeline struct { } type JobListingContainer struct { - AppId int `json:"app_id"` - AppName string `json:"app_name"` + JobId int `json:"job_id"` + JobName string `json:"job_name"` Description string `json:"description"` CiPipelineID int `json:"ci_pipeline_id"` CiPipelineName string `json:"ci_pipeline_name"` diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 1edd621176..2b23158d43 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -61,7 +61,7 @@ const ( ) func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, statuses []string) string { - query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as app_id,app.display_name as app_name, app.description ,cwr.started_on,cwr.status " + + query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as job_id,app.display_name as job_name, app.description ,cwr.started_on,cwr.status " + "from ci_pipeline left join " + "(select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw " + "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index ecf735cf92..33c2c2445b 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -393,12 +393,12 @@ func BuildJobListingResponse(jobContainers []*bean.JobListingContainer, JobsLast //Storing the sequence in appIds array for _, jobContainer := range jobContainers { - val, ok := jobContainersMapping[jobContainer.AppId] + val, ok := jobContainersMapping[jobContainer.JobId] if !ok { - appIds = append(appIds, jobContainer.AppId) + appIds = append(appIds, jobContainer.JobId) val = bean.JobContainer{} - val.AppId = jobContainer.AppId - val.AppName = jobContainer.AppName + val.JobId = jobContainer.JobId + val.JobName = jobContainer.JobName val.Description = jobContainer.Description } @@ -420,7 +420,7 @@ func BuildJobListingResponse(jobContainers []*bean.JobListingContainer, JobsLast val.JobCiPipelines = append(val.JobCiPipelines, ciPipelineObj) } - jobContainersMapping[jobContainer.AppId] = val + jobContainersMapping[jobContainer.JobId] = val } From fc0902b2463f258b2f565066a9b72c8519d2b259 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 17:00:53 +0530 Subject: [PATCH 056/118] Added the user authentications --- specs/jobs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 3e5e1febdc..272079f78e 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -196,9 +196,9 @@ components: type: [] jobContainer jobContainer: properties: - appId: + jobId: type: integer - appName: + jobName: type: string description: type: string From 9b3a4981e6d9a4c212983f78bfbabc406959528f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 17:24:10 +0530 Subject: [PATCH 057/118] Added the user authentications for triggerCiPipeline --- .../app/BuildPipelineRestHandler.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 52ff1c2ae6..94535b7017 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -411,6 +411,25 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } + var appId int + if pipelines[0] != nil { + appId = pipelines[0].AppId + } + app, err := handler.pipelineBuilder.GetApp(appId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + } var authorizedPipelines []pipelineConfig.Pipeline var unauthorizedPipelines []pipelineConfig.Pipeline token := r.Header.Get("token") From 42a0c85cc99c6d59a99785c59a49bf620b111894 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 18:24:32 +0530 Subject: [PATCH 058/118] Added the user authentications for triggerCiPipeline --- .../app/BuildPipelineRestHandler.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 94535b7017..52ff1c2ae6 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -411,25 +411,6 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - var appId int - if pipelines[0] != nil { - appId = pipelines[0].AppId - } - app, err := handler.pipelineBuilder.GetApp(appId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - if app.IsJob { - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - } var authorizedPipelines []pipelineConfig.Pipeline var unauthorizedPipelines []pipelineConfig.Pipeline token := r.Header.Get("token") From 78dd576179cb6a958787c12f11ad329e21407777 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 21:15:44 +0530 Subject: [PATCH 059/118] Updated the api specs and rbac checks --- .../app/BuildPipelineRestHandler.go | 123 ++++++++++-------- .../app/PipelineConfigRestHandler.go | 106 +++++---------- api/router/JobsRouter.go | 2 +- pkg/bean/app.go | 3 +- specs/jobs.yaml | 4 + 5 files changed, 104 insertions(+), 134 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 52ff1c2ae6..e4c3693b69 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -405,6 +405,16 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) //RBAC CHECK CD PIPELINE - FOR USER + if ciTriggerRequest.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + } pipelines, err := handler.pipelineRepository.FindByCiPipelineId(ciTriggerRequest.PipelineId) if err != nil { handler.Logger.Errorw("error in finding ccd pipelines by ciPipelineId", "err", err) @@ -969,6 +979,19 @@ func (handler PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrite } var createMaterialDto bean.CreateMaterialDTO err = decoder.Decode(&createMaterialDto) + createMaterialDto.UserId = userId + if err != nil { + handler.Logger.Errorw("request err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + handler.Logger.Infow("request payload, CreateMaterial", "CreateMaterial", createMaterialDto) + err = handler.validator.Struct(createMaterialDto) + if err != nil { + handler.Logger.Errorw("validation err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } app, err := handler.pipelineBuilder.GetApp(createMaterialDto.AppId) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -983,19 +1006,12 @@ func (handler PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } - } - createMaterialDto.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - handler.Logger.Infow("request payload, CreateMaterial", "CreateMaterial", createMaterialDto) - err = handler.validator.Struct(createMaterialDto) - if err != nil { - handler.Logger.Errorw("validation err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return + } else { + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(createMaterialDto.AppId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } } for _, gitMaterial := range createMaterialDto.Material { validationResult, err := handler.ValidateGitMaterialUrl(gitMaterial.GitProviderId, gitMaterial.Url) @@ -1011,11 +1027,6 @@ func (handler PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrite } } } - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(createMaterialDto.AppId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } createResp, err := handler.pipelineBuilder.CreateMaterialsForApp(&createMaterialDto) if err != nil { @@ -1036,21 +1047,6 @@ func (handler PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWrite } var updateMaterialDto bean.UpdateMaterialDTO err = decoder.Decode(&updateMaterialDto) - app, err := handler.pipelineBuilder.GetApp(updateMaterialDto.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - if app.IsJob { - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - } updateMaterialDto.UserId = userId if err != nil { handler.Logger.Errorw("request err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) @@ -1076,11 +1072,27 @@ func (handler PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWrite return } } - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(updateMaterialDto.AppId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + app, err := handler.pipelineBuilder.GetApp(updateMaterialDto.AppId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + } else { + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(updateMaterialDto.AppId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + } createResp, err := handler.pipelineBuilder.UpdateMaterialsForApp(&updateMaterialDto) if err != nil { @@ -1102,21 +1114,6 @@ func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrite err = decoder.Decode(&deleteMaterial) var updateMaterialDto bean.UpdateMaterialDTO err = decoder.Decode(&updateMaterialDto) - app, err := handler.pipelineBuilder.GetApp(deleteMaterial.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - if app.IsJob { - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - } deleteMaterial.UserId = userId if err != nil { handler.Logger.Errorw("request err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) @@ -1131,12 +1128,28 @@ func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrite return } //rbac starts - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(deleteMaterial.AppId) - token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + app, err := handler.pipelineBuilder.GetApp(deleteMaterial.AppId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + } else { + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(deleteMaterial.AppId) + token := r.Header.Get("token") + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + } //rbac ends err = handler.pipelineBuilder.DeleteMaterial(&deleteMaterial) if err != nil { diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 3d6c8d4941..0b89febeb5 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -59,7 +59,6 @@ import ( type DevtronAppRestHandler interface { CreateApp(w http.ResponseWriter, r *http.Request) - CreateJob(w http.ResponseWriter, r *http.Request) DeleteApp(w http.ResponseWriter, r *http.Request) GetApp(w http.ResponseWriter, r *http.Request) FindAppsByTeamId(w http.ResponseWriter, r *http.Request) @@ -199,13 +198,27 @@ func (handler PipelineConfigRestHandlerImpl) DeleteApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - - resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(appId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionDelete, resourceObject); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + app, err := handler.pipelineBuilder.GetApp(appId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - + if app.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + } else { + resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(appId) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionDelete, resourceObject); !ok { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + return + } + } err = handler.pipelineBuilder.DeleteApp(appId, userId) if err != nil { handler.Logger.Errorw("service error, delete app", "err", err, "appId", appId) @@ -231,7 +244,16 @@ func (handler PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - + if createRequest.IsJob { + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if !isSuperAdmin || err != nil { + if err != nil { + handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + } + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + } handler.Logger.Infow("request payload, CreateApp", "CreateApp", createRequest) err = handler.validator.Struct(createRequest) if err != nil { @@ -283,76 +305,6 @@ func (handler PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, createResp, http.StatusOK) } -func (handler PipelineConfigRestHandlerImpl) CreateJob(w http.ResponseWriter, r *http.Request) { - //token := r.Header.Get("token") - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - handler.Logger.Errorw("request err, GetUserId", "err", userId, "userId", userId) - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - - var createRequest bean.CreateAppDTO - err = decoder.Decode(&createRequest) - createRequest.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, CreateApp", "err", err, "CreateApp", createRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - handler.Logger.Infow("request payload, CreateApp", "CreateApp", createRequest) - err = handler.validator.Struct(createRequest) - if err != nil { - handler.Logger.Errorw("validation err, CreateApp", "err", err, "CreateApp", createRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - - var createResp *bean.CreateAppDTO - err = nil - createRequest.IsJob = true - if createRequest.TemplateId == 0 { - createResp, err = handler.pipelineBuilder.CreateApp(&createRequest) - } else { - ctx, cancel := context.WithCancel(r.Context()) - if cn, ok := w.(http.CloseNotifier); ok { - go func(done <-chan struct{}, closed <-chan bool) { - select { - case <-done: - case <-closed: - cancel() - } - }(ctx.Done(), cn.CloseNotify()) - } - var acdToken string - acdToken, err = handler.argoUserService.GetLatestDevtronArgoCdUserToken() - if err != nil { - handler.Logger.Errorw("error in getting acd token", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - ctx = context.WithValue(r.Context(), "token", acdToken) - createResp, err = handler.appCloneService.CloneApp(&createRequest, ctx) - } - if err != nil { - handler.Logger.Errorw("service err, CreateApp", "err", err, "CreateApp", createRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) -} - func (handler PipelineConfigRestHandlerImpl) GetApp(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") vars := mux.Vars(r) diff --git a/api/router/JobsRouter.go b/api/router/JobsRouter.go index 7624dd1543..e87508fef1 100644 --- a/api/router/JobsRouter.go +++ b/api/router/JobsRouter.go @@ -22,7 +22,7 @@ func NewJobRouterImpl(pipelineConfigRestHandler app.PipelineConfigRestHandler, a //return router } func (router JobRouterImpl) InitJobRouter(jobRouter *mux.Router) { - jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateJob).Methods("POST") + jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateApp).Methods("POST") jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchCiPipelines).Methods("POST") jobRouter.Path("/list").HandlerFunc(router.appListingRestHandler.FetchJobs).Methods("POST") jobRouter.Path("/ci-pipeline/list/{jobId}").HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipelines).Methods("GET") diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 47d8b2a703..a2cbe08c36 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -46,7 +46,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` - IsJob bool `json:"-"` + IsJob bool `json:"isJob"` } type CreateMaterialDTO struct { @@ -258,6 +258,7 @@ type CiTriggerRequest struct { CiPipelineMaterial []CiPipelineMaterial `json:"ciPipelineMaterials" validate:"required"` TriggeredBy int32 `json:"triggeredBy"` InvalidateCache bool `json:"invalidateCache"` + IsJob bool `json:"isJob"` } type CiTrigger struct { diff --git a/specs/jobs.yaml b/specs/jobs.yaml index 272079f78e..e0491a2768 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -92,6 +92,10 @@ components: type: string description: Used to give the name of the job example: "my-job-1" + isJob: + type: boolean + description: States whether its a job or an app + example: true teamId: type: integer description: Used to give team id From 1b9a862b609d8102ddeb8428c48fd4b5d38f5738 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 27 Feb 2023 22:44:00 +0530 Subject: [PATCH 060/118] Updated the api specs and rbac checks --- specs/jobs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/jobs.yaml b/specs/jobs.yaml index e0491a2768..ce47951eb8 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -46,7 +46,7 @@ paths: required: true schema: type: integer - /orchestrator/job//ci-pipeline/patch: + /orchestrator/job/ci-pipeline/patch: post: description: Create/Update ci pipeline. operationId: PatchCiPipeline From bcd9d2621d37527f61af5730fc5e1ccfc45c4c9d Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 28 Feb 2023 13:05:28 +0530 Subject: [PATCH 061/118] Updated the stage status api in case of jobs. --- api/restHandler/AppListingRestHandler.go | 6 +++++- internal/sql/repository/AppListingRepository.go | 13 ++++++++----- pkg/app/AppListingService.go | 14 +++++++++++--- pkg/appClone/AppCloneService.go | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 7b978c49c1..8e48d4ce07 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -580,6 +580,10 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + v := r.URL.Query() + isJobParam := v.Get("isJob") + isJob := isJobParam == "true" + handler.logger.Infow("request payload, FetchAppStageStatus", "appId", appId) token := r.Header.Get("token") app, err := handler.pipeline.GetApp(appId) @@ -597,7 +601,7 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit } //RBAC enforcer Ends - triggerView, err := handler.appListingService.FetchAppStageStatus(appId) + triggerView, err := handler.appListingService.FetchAppStageStatus(appId, isJob) if err != nil { handler.logger.Errorw("service err, FetchAppStageStatus", "err", err, "appId", appId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 4192944f94..66bd685ce5 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -44,7 +44,7 @@ type AppListingRepository interface { FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) FetchAppTriggerView(appId int) ([]bean.TriggerView, error) - FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) + FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) //Not in used PrometheusApiByEnvId(id int) (*string, error) @@ -409,7 +409,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig return triggerViewResponse, nil } -func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) { +func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) { impl.Logger.Debug("reached at AppListingRepository:") var appStageStatus []bean.AppStageStatus @@ -432,11 +432,14 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppS " LEFT JOIN charts ch on ch.app_id=app.id" + " LEFT JOIN pipeline p on p.app_id=app.id" + " LEFT JOIN chart_env_config_override ceco on ceco.chart_id=ch.id" + - " WHERE app.id=? and app.app_store = 0;" + " WHERE app.id=? and app.app_store = ?;" impl.Logger.Debugw("last app stages status query:", "query", query) - - _, err := impl.dbConnection.Query(&stages, query, appId) + appStore := 0 + if isJob { + appStore = 2 + } + _, err := impl.dbConnection.Query(&stages, query, appId, appStore) if err != nil { impl.Logger.Errorw("error:", err) return appStageStatus, err diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 33c2c2445b..e6d0f47b09 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -80,7 +80,7 @@ type AppListingService interface { GraphAPI(appId int, envId int) error FetchAppTriggerView(appId int) ([]bean.TriggerView, error) - FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) + FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) FetchOtherEnvironment(ctx context.Context, appId int) ([]*bean.Environment, error) RedirectToLinkouts(Id int, appId int, envId int, podName string, containerName string) (string, error) @@ -1532,8 +1532,16 @@ func (impl AppListingServiceImpl) FetchAppTriggerView(appId int) ([]bean.Trigger return impl.appListingRepository.FetchAppTriggerView(appId) } -func (impl AppListingServiceImpl) FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) { - return impl.appListingRepository.FetchAppStageStatus(appId) +func (impl AppListingServiceImpl) FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) { + appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId, isJob) + if isJob { + for i := range appStageStatuses { + if appStageStatuses[i].StageName == "TEMPLATE" || appStageStatuses[i].StageName == "CHART" || appStageStatuses[i].StageName == "CD_PIPELINE" || appStageStatuses[i].StageName == "CHART_ENV_CONFIG" { + appStageStatuses[i].Status = true + } + } + } + return appStageStatuses, err } func (impl AppListingServiceImpl) FetchOtherEnvironment(ctx context.Context, appId int) ([]*bean.Environment, error) { diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 1a214b0dcc..118f0a5614 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -114,7 +114,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context Description: createReq.Description, } userId := createReq.UserId - appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) + appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId, cloneReq.isJob) if err != nil { return nil, err } From c9111d15f4ea8daf9e93b5c8841f777efac73581 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 28 Feb 2023 13:12:14 +0530 Subject: [PATCH 062/118] Updated the job autocomplete api. --- pkg/pipeline/PipelineBuilder.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 9d1e5932b1..52d676c0e3 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -2844,7 +2844,11 @@ func (impl PipelineBuilderImpl) FindAllMatchesByAppName(appName string, isJob bo return nil, err } for _, app := range apps { - appsRes = append(appsRes, &AppBean{Id: app.Id, Name: app.AppName}) + name := app.AppName + if isJob { + name = app.DisplayName + } + appsRes = append(appsRes, &AppBean{Id: app.Id, Name: name}) } return appsRes, err } From 4a874c5e5717aee424155df6f786855d5ff0a6dc Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 28 Feb 2023 16:22:39 +0530 Subject: [PATCH 063/118] Made the status changes --- .../app/BuildPipelineRestHandler.go | 89 ------------------- pkg/app/AppListingService.go | 4 +- 2 files changed, 3 insertions(+), 90 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index e4c3693b69..9638ebda90 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -32,7 +32,6 @@ type DevtronAppBuildRestHandler interface { GetExternalCiById(w http.ResponseWriter, r *http.Request) PatchCiPipelines(w http.ResponseWriter, r *http.Request) TriggerCiPipeline(w http.ResponseWriter, r *http.Request) - TriggerJobCiPipeline(w http.ResponseWriter, r *http.Request) GetCiPipelineMin(w http.ResponseWriter, r *http.Request) GetCIPipelineById(w http.ResponseWriter, r *http.Request) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) @@ -470,94 +469,6 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, response, http.StatusOK) } - -func (handler PipelineConfigRestHandlerImpl) TriggerJobCiPipeline(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - decoder := json.NewDecoder(r.Body) - var ciTriggerRequest bean.CiTriggerRequest - err = decoder.Decode(&ciTriggerRequest) - if err != nil { - handler.Logger.Errorw("request err, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - if !handler.validForMultiMaterial(ciTriggerRequest) { - handler.Logger.Errorw("invalid req, commit hash not present for multi-git", "payload", ciTriggerRequest) - common.WriteJsonResp(w, errors.New("invalid req, commit hash not present for multi-git"), - nil, http.StatusBadRequest) - } - ciTriggerRequest.TriggeredBy = userId - handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) - - //RBAC CHECK CD PIPELINE - FOR USER - pipelines, err := handler.pipelineRepository.FindByCiPipelineId(ciTriggerRequest.PipelineId) - if err != nil { - handler.Logger.Errorw("error in finding ccd pipelines by ciPipelineId", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - var authorizedPipelines []pipelineConfig.Pipeline - var unauthorizedPipelines []pipelineConfig.Pipeline - token := r.Header.Get("token") - for _, p := range pipelines { - object := handler.enforcerUtil.GetAppRBACNameByAppId(p.AppId) - appRbacOk := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionTrigger, object) - object = handler.enforcerUtil.GetAppRBACByAppIdAndPipelineId(p.AppId, p.Id) - envRbacOk := handler.enforcer.Enforce(token, casbin.ResourceEnvironment, casbin.ActionTrigger, object) - if appRbacOk && envRbacOk { - authorizedPipelines = append(authorizedPipelines, *p) - } else { - unauthorizedPipelines = append(unauthorizedPipelines, *p) - } - } - resMessage := "allowed for all pipelines" - response := make(map[string]string) - if len(unauthorizedPipelines) > 0 { - resMessage = "some pipelines not authorized" - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - if len(authorizedPipelines) == 0 { - //user has no cd pipeline - ciPipeline, err := handler.ciPipelineRepository.FindById(ciTriggerRequest.PipelineId) - if err != nil { - handler.Logger.Errorw("err in finding ci pipeline, TriggerCiPipeline", "err", err, "ciPipelineId", ciTriggerRequest.PipelineId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionTrigger, object); !ok { - handler.Logger.Debug(fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) - return - } - } - //RBAC CHECK CD PIPELINE - FOR USER - - resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) - if err != nil { - handler.Logger.Errorw("service err, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) - common.WriteJsonResp(w, err, response, http.StatusInternalServerError) - return - } - response["apiResponse"] = strconv.Itoa(resp) - response["authStatus"] = resMessage - - common.WriteJsonResp(w, err, response, http.StatusOK) -} - func (handler PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWriter, r *http.Request) { userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index e6d0f47b09..f88dd015b3 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -1536,8 +1536,10 @@ func (impl AppListingServiceImpl) FetchAppStageStatus(appId int, isJob bool) ([] appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId, isJob) if isJob { for i := range appStageStatuses { - if appStageStatuses[i].StageName == "TEMPLATE" || appStageStatuses[i].StageName == "CHART" || appStageStatuses[i].StageName == "CD_PIPELINE" || appStageStatuses[i].StageName == "CHART_ENV_CONFIG" { + if appStageStatuses[i].StageName == "TEMPLATE" || appStageStatuses[i].StageName == "CHART" || appStageStatuses[i].StageName == "CHART_ENV_CONFIG" { appStageStatuses[i].Status = true + } else if appStageStatuses[i].StageName == "APP" || appStageStatuses[i].StageName == "CD_PIPELINE" { + appStageStatuses[i].Status = false } } } From cc872160d312f1563328f6800f110ac012305829 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 1 Mar 2023 12:02:49 +0530 Subject: [PATCH 064/118] Fixed the bug of last succeeded time --- api/bean/AppView.go | 4 ++-- pkg/app/AppListingService.go | 2 +- pkg/pipeline/WebhookService.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/bean/AppView.go b/api/bean/AppView.go index e90f42fccd..f976216439 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -84,8 +84,8 @@ type JobListingContainer struct { } type CiPipelineLastSucceededTime struct { - CiPipelineID int `json:"ci_pipeline_id"` - LastSuccessOn time.Time `json:"last_succeeded_on"` + CiPipelineID int `json:"ci_pipeline_id"` + LastSucceededOn time.Time `json:"last_succeeded_on"` } type AppEnvironmentContainer struct { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index f88dd015b3..4ea692d1d9 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -388,7 +388,7 @@ func BuildJobListingResponse(jobContainers []*bean.JobListingContainer, JobsLast lastSucceededTimeMapping := make(map[int]time.Time) for _, lastSuccessTime := range JobsLastSucceededOnTime { - lastSucceededTimeMapping[lastSuccessTime.CiPipelineID] = lastSuccessTime.LastSuccessOn + lastSucceededTimeMapping[lastSuccessTime.CiPipelineID] = lastSuccessTime.LastSucceededOn } //Storing the sequence in appIds array diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index db2b81611e..d7712d2758 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -123,6 +123,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } savedWorkflow.Status = string(v1alpha1.NodeSucceeded) + savedWorkflow.FinishedOn = time.Now() impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) if err != nil { From 666bc01ca90c42ed87c8da0a9319145483114b5f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 1 Mar 2023 17:15:55 +0530 Subject: [PATCH 065/118] undid the reverted changes --- pkg/appClone/AppCloneService.go | 65 +++++++++++++++++---------------- pkg/pipeline/WebhookService.go | 2 +- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 118f0a5614..e477965806 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -163,48 +163,49 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context impl.logger.Errorw("error in cloning docker template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err } - if !refAppStatus["TEMPLATE"] { - impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) - return app, nil - } - if !refAppStatus["CHART"] { - impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) - return app, nil - } - _, err = impl.CreateDeploymentTemplate(cloneReq.RefAppId, newAppId, userId, context) - if err != nil { - impl.logger.Errorw("error in creating deployment template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) - return nil, err - } - _, err = impl.CreateGlobalCM(cloneReq.RefAppId, newAppId, userId) - - if err != nil { - impl.logger.Errorw("error in creating global cm", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) - return nil, err - } - _, err = impl.CreateGlobalSecret(cloneReq.RefAppId, newAppId, userId) - if err != nil { - impl.logger.Errorw("error in creating global secret", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) - return nil, err - } - if isSameProject { - _, err = impl.CreateEnvCm(context, cloneReq.RefAppId, newAppId, userId) + if !createReq.IsJob { + if !refAppStatus["TEMPLATE"] { + impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) + return app, nil + } + if !refAppStatus["CHART"] { + impl.logger.Errorw("status not", "CHART", cloneReq.RefAppId) + return app, nil + } + _, err = impl.CreateDeploymentTemplate(cloneReq.RefAppId, newAppId, userId, context) if err != nil { - impl.logger.Errorw("error in creating env cm", "err", err) + impl.logger.Errorw("error in creating deployment template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err } - _, err = impl.CreateEnvSecret(context, cloneReq.RefAppId, newAppId, userId) + _, err = impl.CreateGlobalCM(cloneReq.RefAppId, newAppId, userId) + if err != nil { - impl.logger.Errorw("error in creating env secret", "err", err) + impl.logger.Errorw("error in creating global cm", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err } - _, err = impl.createEnvOverride(cloneReq.RefAppId, newAppId, userId, context) + _, err = impl.CreateGlobalSecret(cloneReq.RefAppId, newAppId, userId) if err != nil { - impl.logger.Errorw("error in cloning env override", "err", err) + impl.logger.Errorw("error in creating global secret", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err } + if isSameProject { + _, err = impl.CreateEnvCm(context, cloneReq.RefAppId, newAppId, userId) + if err != nil { + impl.logger.Errorw("error in creating env cm", "err", err) + return nil, err + } + _, err = impl.CreateEnvSecret(context, cloneReq.RefAppId, newAppId, userId) + if err != nil { + impl.logger.Errorw("error in creating env secret", "err", err) + return nil, err + } + _, err = impl.createEnvOverride(cloneReq.RefAppId, newAppId, userId, context) + if err != nil { + impl.logger.Errorw("error in cloning env override", "err", err) + return nil, err + } + } } - _, err = impl.CreateWf(cloneReq.RefAppId, newAppId, userId, gitMaerialMap, context, isSameProject) if err != nil { impl.logger.Errorw("error in creating wf", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index d7712d2758..f72c1466d5 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -123,7 +123,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } savedWorkflow.Status = string(v1alpha1.NodeSucceeded) - savedWorkflow.FinishedOn = time.Now() + //savedWorkflow.FinishedOn = time.Now() impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) if err != nil { From 44ff4ffb2eafaf9331f13b975328c62ed2433b28 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 1 Mar 2023 18:41:20 +0530 Subject: [PATCH 066/118] Job count added --- api/restHandler/AppListingRestHandler.go | 4 ++-- internal/sql/repository/app/AppRepository.go | 17 +++++++++++++++++ pkg/app/AppListingService.go | 11 ++++++----- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index f360ffa461..7d3c112431 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -182,7 +182,7 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - jobs, err := handler.appListingService.FetchJobs(fetchJobListingRequest) + jobs, jobCount, err := handler.appListingService.FetchJobs(fetchJobListingRequest) if err != nil { handler.logger.Errorw("service err, FetchJobs", "err", err, "payload", fetchJobListingRequest) common.WriteJsonResp(w, err, "", http.StatusInternalServerError) @@ -191,7 +191,7 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt jobContainerResponse := bean.JobContainerResponse{ JobContainers: jobs, - JobCount: len(jobs), + JobCount: jobCount, } common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 2b13a2369a..bb0335c3cb 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -69,6 +69,7 @@ type AppRepository interface { FetchAllActiveDevtronAppsWithAppIdAndName() ([]*App, error) FindEnvironmentIdForInstalledApp(appId int) (int, error) FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) + FetchAllJobs() ([]int, error) } const DevtronApp = "DevtronApp" @@ -382,3 +383,19 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL } return appIdsResult, err } + +func (repo AppRepositoryImpl) FetchAllJobs() ([]int, error) { + type JobId struct { + Id int `json:"id"` + } + var appIds []JobId + whereCondition := " where active = true and app_store = 2 " + query := "select id " + "from app " + whereCondition + + _, err := repo.dbConnection.Query(&appIds, query) + jobIdsResult := make([]int, 0) + for _, id := range appIds { + jobIdsResult = append(jobIdsResult, id.Id) + } + return jobIdsResult, err +} diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 4ea692d1d9..c7ae61db87 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -53,7 +53,7 @@ import ( type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) - FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) + FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, int, error) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) @@ -222,7 +222,7 @@ func (impl AppListingServiceImpl) FetchAllDevtronManagedApps() ([]AppNameTypeIdC } return apps, nil } -func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) { +func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, int, error) { jobListingFilter := helper.AppListingFilter{ Teams: fetchJobListingRequest.Teams, @@ -236,17 +236,18 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi appIds, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) if err != nil { impl.Logger.Errorw("error in fetching app ids list", "error", err) - return []*bean.JobContainer{}, err + return []*bean.JobContainer{}, 0, err } jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds, jobListingFilter.AppStatuses) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) - return []*bean.JobContainer{}, err + return []*bean.JobContainer{}, 0, err } + jobIds, err := impl.appRepository.FetchAllJobs() CiPipelineIDs := GetCIPipelineIDs(jobListingContainers) JobsLastSucceededOnTime, err := impl.appListingRepository.FetchJobsLastSucceededOn(CiPipelineIDs) jobContainers := BuildJobListingResponse(jobListingContainers, JobsLastSucceededOnTime) - return jobContainers, nil + return jobContainers, len(jobIds), nil } func (impl AppListingServiceImpl) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) { From c76256061a083cfd1fc390dace4308c7f0c10a85 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 2 Mar 2023 13:17:13 +0530 Subject: [PATCH 067/118] Job count corrected --- internal/sql/repository/app/AppRepository.go | 27 +++++++------------- pkg/app/AppListingService.go | 5 ++-- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index bb0335c3cb..fdc6b51494 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -68,8 +68,7 @@ type AppRepository interface { FetchAllActiveInstalledAppsWithAppIdAndName() ([]*App, error) FetchAllActiveDevtronAppsWithAppIdAndName() ([]*App, error) FindEnvironmentIdForInstalledApp(appId int) (int, error) - FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) - FetchAllJobs() ([]int, error) + FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, int, error) } const DevtronApp = "DevtronApp" @@ -355,11 +354,12 @@ func (repo AppRepositoryImpl) FindEnvironmentIdForInstalledApp(appId int) (int, _, err := repo.dbConnection.Query(&res, query, appId) return res.envId, err } -func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) { +func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, int, error) { type AppId struct { Id int `json:"id"` } var appIds []AppId + var jobIds []AppId whereCondition := " where active = true and app_store = 2 " if len(jobListingFilter.Teams) > 0 { whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" @@ -381,21 +381,12 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL for _, id := range appIds { appIdsResult = append(appIdsResult, id.Id) } - return appIdsResult, err -} - -func (repo AppRepositoryImpl) FetchAllJobs() ([]int, error) { - type JobId struct { - Id int `json:"id"` - } - var appIds []JobId - whereCondition := " where active = true and app_store = 2 " - query := "select id " + "from app " + whereCondition + query2 := "select id " + "from app " + whereCondition - _, err := repo.dbConnection.Query(&appIds, query) - jobIdsResult := make([]int, 0) - for _, id := range appIds { - jobIdsResult = append(jobIdsResult, id.Id) + _, err = repo.dbConnection.Query(&jobIds, query2) + appCounts := make([]int, 0) + for _, id := range jobIds { + appCounts = append(appCounts, id.Id) } - return jobIdsResult, err + return appIdsResult, len(appCounts), err } diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index c7ae61db87..34c1729b90 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -233,7 +233,7 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi Size: fetchJobListingRequest.Size, AppStatuses: fetchJobListingRequest.AppStatuses, } - appIds, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) + appIds, jobIdsLen, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) if err != nil { impl.Logger.Errorw("error in fetching app ids list", "error", err) return []*bean.JobContainer{}, 0, err @@ -243,11 +243,10 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi impl.Logger.Errorw("error in fetching app list", "error", err) return []*bean.JobContainer{}, 0, err } - jobIds, err := impl.appRepository.FetchAllJobs() CiPipelineIDs := GetCIPipelineIDs(jobListingContainers) JobsLastSucceededOnTime, err := impl.appListingRepository.FetchJobsLastSucceededOn(CiPipelineIDs) jobContainers := BuildJobListingResponse(jobListingContainers, JobsLastSucceededOnTime) - return jobContainers, len(jobIds), nil + return jobContainers, jobIdsLen, nil } func (impl AppListingServiceImpl) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) { From 528a8f25b492695d587c017078093b0e78c432ba Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 2 Mar 2023 14:09:12 +0530 Subject: [PATCH 068/118] Job count bug fixed --- api/restHandler/AppListingRestHandler.go | 15 +++++++++++++-- internal/sql/repository/app/AppRepository.go | 18 ++++-------------- pkg/app/AppListingService.go | 12 ++++++------ 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 7d3c112431..6a556a9303 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -182,16 +182,27 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - jobs, jobCount, err := handler.appListingService.FetchJobs(fetchJobListingRequest) + jobs, err := handler.appListingService.FetchJobs(fetchJobListingRequest) if err != nil { handler.logger.Errorw("service err, FetchJobs", "err", err, "payload", fetchJobListingRequest) common.WriteJsonResp(w, err, "", http.StatusInternalServerError) return } + // Apply pagination + jobsCount := len(jobs) + offset := fetchJobListingRequest.Offset + limit := fetchJobListingRequest.Size + if limit > 0 { + if offset+limit <= len(jobs) { + jobs = jobs[offset : offset+limit] + } else { + jobs = jobs[offset:] + } + } jobContainerResponse := bean.JobContainerResponse{ JobContainers: jobs, - JobCount: jobCount, + JobCount: jobsCount, } common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index fdc6b51494..cf135822b7 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -68,7 +68,7 @@ type AppRepository interface { FetchAllActiveInstalledAppsWithAppIdAndName() ([]*App, error) FetchAllActiveDevtronAppsWithAppIdAndName() ([]*App, error) FindEnvironmentIdForInstalledApp(appId int) (int, error) - FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, int, error) + FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) } const DevtronApp = "DevtronApp" @@ -354,11 +354,10 @@ func (repo AppRepositoryImpl) FindEnvironmentIdForInstalledApp(appId int) (int, _, err := repo.dbConnection.Query(&res, query, appId) return res.envId, err } -func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, int, error) { +func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppListingFilter) ([]int, error) { type AppId struct { Id int `json:"id"` } - var appIds []AppId var jobIds []AppId whereCondition := " where active = true and app_store = 2 " if len(jobListingFilter.Teams) > 0 { @@ -372,21 +371,12 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL if jobListingFilter.SortOrder == "DESC" { orderByCondition += string(jobListingFilter.SortOrder) } - orderByCondition += " limit ? offset ? " - query := "select id " + "from app " + whereCondition + orderByCondition - _, err := repo.dbConnection.Query(&appIds, query, jobListingFilter.Size, jobListingFilter.Offset) - appIdsResult := make([]int, 0) - for _, id := range appIds { - appIdsResult = append(appIdsResult, id.Id) - } - query2 := "select id " + "from app " + whereCondition - - _, err = repo.dbConnection.Query(&jobIds, query2) + _, err := repo.dbConnection.Query(&jobIds, query) appCounts := make([]int, 0) for _, id := range jobIds { appCounts = append(appCounts, id.Id) } - return appIdsResult, len(appCounts), err + return appCounts, err } diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 34c1729b90..4ea692d1d9 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -53,7 +53,7 @@ import ( type AppListingService interface { FetchAppsByEnvironment(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*bean.AppEnvironmentContainer, error) - FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, int, error) + FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) BuildAppListingResponse(fetchAppListingRequest FetchAppListingRequest, envContainers []*bean.AppEnvironmentContainer) ([]*bean.AppContainer, error) FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) @@ -222,7 +222,7 @@ func (impl AppListingServiceImpl) FetchAllDevtronManagedApps() ([]AppNameTypeIdC } return apps, nil } -func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, int, error) { +func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListingRequest) ([]*bean.JobContainer, error) { jobListingFilter := helper.AppListingFilter{ Teams: fetchJobListingRequest.Teams, @@ -233,20 +233,20 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi Size: fetchJobListingRequest.Size, AppStatuses: fetchJobListingRequest.AppStatuses, } - appIds, jobIdsLen, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) + appIds, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) if err != nil { impl.Logger.Errorw("error in fetching app ids list", "error", err) - return []*bean.JobContainer{}, 0, err + return []*bean.JobContainer{}, err } jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds, jobListingFilter.AppStatuses) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) - return []*bean.JobContainer{}, 0, err + return []*bean.JobContainer{}, err } CiPipelineIDs := GetCIPipelineIDs(jobListingContainers) JobsLastSucceededOnTime, err := impl.appListingRepository.FetchJobsLastSucceededOn(CiPipelineIDs) jobContainers := BuildJobListingResponse(jobListingContainers, JobsLastSucceededOnTime) - return jobContainers, jobIdsLen, nil + return jobContainers, nil } func (impl AppListingServiceImpl) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) { From 5009f0304861d06b8faf453e92cb27fbca8d3eb9 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 2 Mar 2023 19:23:26 +0530 Subject: [PATCH 069/118] Sorting order corrected --- .../sql/repository/helper/AppListingRepositoryQueryBuilder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 2b23158d43..a3cc4e24d2 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -74,6 +74,7 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, if len(statuses) > 0 { query += "and cwr.status IN (" + util.ProcessAppStatuses(statuses) + ") " } + query += " order by app.display_name" return query } func (impl AppListingRepositoryQueryBuilder) OverviewCiPipelineQuery() string { From 8f03388cd15dc5e47480a7c2f4fe6e7d11e65e97 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 2 Mar 2023 19:42:50 +0530 Subject: [PATCH 070/118] Sorting order DESC added --- internal/sql/repository/AppListingRepository.go | 6 +++--- .../repository/helper/AppListingRepositoryQueryBuilder.go | 5 ++++- pkg/app/AppListingService.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index e1fa4ed00a..9aa2e81352 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -37,7 +37,7 @@ import ( type AppListingRepository interface { FetchAppsByEnvironment(appListingFilter helper.AppListingFilter) ([]*bean.AppEnvironmentContainer, error) - FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) + FetchJobs(appIds []int, statuses []string, sortOrder string) ([]*bean.JobListingContainer, error) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) FetchJobsLastSucceededOn(ciPipelineIDs []int) ([]*bean.CiPipelineLastSucceededTime, error) DeploymentDetailsByAppIdAndEnvId(ctx context.Context, appId int, envId int) (bean.DeploymentDetailContainer, error) @@ -83,12 +83,12 @@ func NewAppListingRepositoryImpl(Logger *zap.SugaredLogger, dbConnection *pg.DB, return &AppListingRepositoryImpl{dbConnection: dbConnection, Logger: Logger, appListingRepositoryQueryBuilder: appListingRepositoryQueryBuilder} } -func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string) ([]*bean.JobListingContainer, error) { +func (impl AppListingRepositoryImpl) FetchJobs(appIds []int, statuses []string, sortOrder string) ([]*bean.JobListingContainer, error) { var jobContainers []*bean.JobListingContainer if len(appIds) == 0 { return jobContainers, nil } - jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(appIds, statuses) + jobsQuery := impl.appListingRepositoryQueryBuilder.BuildJobListingQuery(appIds, statuses, sortOrder) impl.Logger.Debugw("basic app detail query: ", jobsQuery) _, appsErr := impl.dbConnection.Query(&jobContainers, jobsQuery) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index a3cc4e24d2..02ac50c566 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -60,7 +60,7 @@ const ( AppNameSortBy SortBy = "appNameSort" ) -func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, statuses []string) string { +func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, statuses []string, sortOrder string) string { query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as job_id,app.display_name as job_name, app.description ,cwr.started_on,cwr.status " + "from ci_pipeline left join " + "(select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw " + @@ -75,6 +75,9 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, query += "and cwr.status IN (" + util.ProcessAppStatuses(statuses) + ") " } query += " order by app.display_name" + if sortOrder == "DESC" { + query += " DESC " + } return query } func (impl AppListingRepositoryQueryBuilder) OverviewCiPipelineQuery() string { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 4ea692d1d9..0e78f1a09b 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -238,7 +238,7 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi impl.Logger.Errorw("error in fetching app ids list", "error", err) return []*bean.JobContainer{}, err } - jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds, jobListingFilter.AppStatuses) + jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds, jobListingFilter.AppStatuses, string(jobListingFilter.SortOrder)) if err != nil { impl.Logger.Errorw("error in fetching app list", "error", err) return []*bean.JobContainer{}, err From 8eab28a1b4bad947dc9a49d9ec8613e04069decd Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 3 Mar 2023 18:23:34 +0530 Subject: [PATCH 071/118] Corrected the app overview page deleted pipeline bug. --- .../sql/repository/helper/AppListingRepositoryQueryBuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 02ac50c566..a39bf85d96 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -86,7 +86,7 @@ func (impl AppListingRepositoryQueryBuilder) OverviewCiPipelineQuery() string { " left join (select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw" + " inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id)" + " cws on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id)" + - " cwr on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true where ci_pipeline.app_id = ? ;" + " cwr on cwr.ci_pipeline_id = ci_pipeline.id where ci_pipeline.active = true and ci_pipeline.app_id = ? ;" return query } From a2c758c0919986b5b536422f65a6e65d218fcda7 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 4 Mar 2023 23:43:23 +0530 Subject: [PATCH 072/118] Corrected the app overview page deleted pipeline bug. --- api/router/pubsub/CiEventHandler.go | 34 ++++++++++--------- .../sql/repository/CiArtifactRepository.go | 1 + pkg/pipeline/CiHandler.go | 3 ++ pkg/pipeline/WebhookService.go | 34 ++++++++++--------- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index 3fd2f0db83..74d6477141 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -40,15 +40,16 @@ type CiEventHandlerImpl struct { } type CiCompleteEvent struct { - CiProjectDetails []pipeline.CiProjectDetails `json:"ciProjectDetails"` - DockerImage string `json:"dockerImage" validate:"required,image-validator"` - Digest string `json:"digest"` - PipelineId int `json:"pipelineId"` - WorkflowId *int `json:"workflowId"` - TriggeredBy int32 `json:"triggeredBy"` - PipelineName string `json:"pipelineName"` - DataSource string `json:"dataSource"` - MaterialType string `json:"materialType"` + CiProjectDetails []pipeline.CiProjectDetails `json:"ciProjectDetails"` + DockerImage string `json:"dockerImage" validate:"required,image-validator"` + Digest string `json:"digest"` + PipelineId int `json:"pipelineId"` + WorkflowId *int `json:"workflowId"` + TriggeredBy int32 `json:"triggeredBy"` + PipelineName string `json:"pipelineName"` + DataSource string `json:"dataSource"` + MaterialType string `json:"materialType"` + isArtifactUploaded bool `json:"isArtifactUploaded"` } func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService) *CiEventHandlerImpl { @@ -150,13 +151,14 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* } request := &pipeline.CiArtifactWebhookRequest{ - Image: event.DockerImage, - ImageDigest: event.Digest, - DataSource: event.DataSource, - PipelineName: event.PipelineName, - MaterialInfo: rawMaterialInfo, - UserId: event.TriggeredBy, - WorkflowId: event.WorkflowId, + Image: event.DockerImage, + ImageDigest: event.Digest, + DataSource: event.DataSource, + PipelineName: event.PipelineName, + MaterialInfo: rawMaterialInfo, + UserId: event.TriggeredBy, + WorkflowId: event.WorkflowId, + IsArtifactUploaded: event.isArtifactUploaded, } return request, nil } diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 307e24fee1..d58416fb38 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -43,6 +43,7 @@ type CiArtifact struct { ScanEnabled bool `sql:"scan_enabled,notnull"` Scanned bool `sql:"scanned,notnull"` ExternalCiPipelineId int `sql:"external_ci_pipeline_id"` + IsArtifactUploaded bool `sql:"is_artifact_uploaded"` DeployedTime time.Time `sql:"-"` Deployed bool `sql:"-"` Latest bool `sql:"-"` diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index ed83bde595..7e6c14e1d9 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -156,6 +156,7 @@ type WorkflowResponse struct { TriggeredByEmail string `json:"triggeredByEmail"` Stage string `json:"stage"` ArtifactId int `json:"artifactId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` } type GitTriggerInfoResponse struct { @@ -493,6 +494,8 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build TriggeredBy: workflow.TriggeredBy, TriggeredByEmail: triggeredByUser.EmailId, Artifact: ciArtifact.Image, + ArtifactId: ciArtifact.Id, + IsArtifactUploaded: ciArtifact.IsArtifactUploaded, } return workflowResponse, nil } diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index f72c1466d5..c215493fbe 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -38,13 +38,14 @@ import ( ) type CiArtifactWebhookRequest struct { - Image string `json:"image"` - ImageDigest string `json:"imageDigest"` - MaterialInfo json.RawMessage `json:"materialInfo"` - DataSource string `json:"dataSource"` - PipelineName string `json:"pipelineName"` - WorkflowId *int `json:"workflowId"` - UserId int32 `json:"userId"` + Image string `json:"image"` + ImageDigest string `json:"imageDigest"` + MaterialInfo json.RawMessage `json:"materialInfo"` + DataSource string `json:"dataSource"` + PipelineName string `json:"pipelineName"` + WorkflowId *int `json:"workflowId"` + UserId int32 `json:"userId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` } type WebhookService interface { @@ -155,15 +156,16 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C } materialJson = dst.Bytes() artifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: pipeline.Id, - WorkflowId: request.WorkflowId, - ScanEnabled: pipeline.ScanEnabled, - Scanned: false, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: pipeline.Id, + WorkflowId: request.WorkflowId, + ScanEnabled: pipeline.ScanEnabled, + Scanned: false, + IsArtifactUploaded: request.IsArtifactUploaded, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if pipeline.ScanEnabled { artifact.Scanned = true From 7df1925e50fb029bd6c058fff055ef823ab87ee0 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 4 Mar 2023 23:52:45 +0530 Subject: [PATCH 073/118] added column is_artifact_uploaded to the ci_artifact table. --- scripts/sql/115_jobs.down.sql | 4 +++- scripts/sql/115_jobs.up.sql | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/115_jobs.down.sql index ec531d6b0d..3458f020d1 100644 --- a/scripts/sql/115_jobs.down.sql +++ b/scripts/sql/115_jobs.down.sql @@ -98,4 +98,6 @@ ALTER TABLE ci_pipeline ADD CONSTRAINT ci_pipeline_app_id_fkey FOREIGN KEY (app_ ALTER TABLE app_workflow DROP CONSTRAINT app_workflow_app_id_fkey; -ALTER TABLE app_workflow ADD CONSTRAINT app_workflow_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); \ No newline at end of file +ALTER TABLE app_workflow ADD CONSTRAINT app_workflow_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); + +ALTER TABLE ci_artifact DROP COLUMN is_artifact_uploaded; diff --git a/scripts/sql/115_jobs.up.sql b/scripts/sql/115_jobs.up.sql index 6bae49696d..635caf44e8 100644 --- a/scripts/sql/115_jobs.up.sql +++ b/scripts/sql/115_jobs.up.sql @@ -3,3 +3,5 @@ ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=true THEN ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; ALTER TABLE app ADD COLUMN display_name varchar(250); ALTER TABLE app ADD COLUMN description text; +ALTER TABLE ci_artifact ADD COLUMN is_artifact_uploaded BOOLEAN NOT NULL DEFAULT FALSE; + From 7367ba8a72e522140f2932fd079a12defe6149c5 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sun, 5 Mar 2023 00:01:58 +0530 Subject: [PATCH 074/118] Added the check of deleted pipeline. --- .../sql/repository/helper/AppListingRepositoryQueryBuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index a39bf85d96..0a18cc4305 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -67,7 +67,7 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + "on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr" + " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + - "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 " + "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 and ci_pipeline.active = true " if len(appIDs) > 0 { query += "and app.id IN (" + GetCommaSepratedString(appIDs) + ") " } From 5bba36447749c5b7ec1b5160f2fd1514c7704487 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 6 Mar 2023 12:30:57 +0530 Subject: [PATCH 075/118] Added the check of deleted pipeline. --- .../sql/repository/helper/AppListingRepositoryQueryBuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 0a18cc4305..a39bf85d96 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -67,7 +67,7 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + "on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr" + " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + - "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 and ci_pipeline.active = true " + "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 " if len(appIDs) > 0 { query += "and app.id IN (" + GetCommaSepratedString(appIDs) + ") " } From 302066bbd32e372635c9de385e3f0aac53e577bc Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 6 Mar 2023 13:15:13 +0530 Subject: [PATCH 076/118] Added the check of deleted pipeline. --- .../helper/AppListingRepositoryQueryBuilder.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index a39bf85d96..3f6f06a573 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -61,13 +61,13 @@ const ( ) func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, statuses []string, sortOrder string) string { - query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as job_id,app.display_name as job_name, app.description ,cwr.started_on,cwr.status " + - "from ci_pipeline left join " + - "(select cw.ci_pipeline_id,cw.status,cw.started_on from ci_workflow cw " + - "inner join (SELECT ci_pipeline_id, MAX(started_on) max_started_on FROM ci_workflow GROUP BY ci_pipeline_id) cws " + - "on cw.ci_pipeline_id = cws.ci_pipeline_id and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr" + - " on cwr.ci_pipeline_id = ci_pipeline.id and ci_pipeline.active = true " + - "right join app on app.id = ci_pipeline.app_id where app.active = true and app.app_store = 2 " + query := "select ci_pipeline.name as ci_pipeline_name,ci_pipeline.id as ci_pipeline_id,app.id as job_id,app.display_name " + + "as job_name,app.description,cwr.started_on,cwr.status from app left join ci_pipeline on" + + " app.id = ci_pipeline.app_id and ci_pipeline.active=true left join (select cw.ci_pipeline_id, cw.status, cw.started_on " + + " from ci_workflow cw inner join (select ci_pipeline_id, MAX(started_on) max_started_on from ci_workflow group by ci_pipeline_id ) " + + "cws on cw.ci_pipeline_id = cws.ci_pipeline_id " + + "and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr on cwr.ci_pipeline_id = ci_pipeline.id" + + " where app.active = true and app.app_store = 2 " if len(appIDs) > 0 { query += "and app.id IN (" + GetCommaSepratedString(appIDs) + ") " } From a361f6382b9c6195e8c5b4c11e9791e5ca377cf8 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 6 Mar 2023 15:01:21 +0530 Subject: [PATCH 077/118] Clone app --- internal/sql/repository/app/AppRepository.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index cf135822b7..922ae29286 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -128,6 +128,7 @@ func (repo AppRepositoryImpl) FindJobByDisplayName(appName string) (*App, error) Model(pipelineGroup). Where("display_name = ?", appName). Where("active = ?", true). + Where("app_store = ?", 2). Order("id DESC").Limit(1). Select() // there is only single active app will be present in db with a same name. From ce0a0412c55d0ef92863f3f913ebee7b590b9644 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 6 Mar 2023 15:39:29 +0530 Subject: [PATCH 078/118] Multiple git repo patch --- api/restHandler/app/BuildPipelineRestHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index e41808163b..9c15ef7b9c 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -263,7 +263,7 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri ciConfigRequest.CiBuildConfig = &bean1.CiBuildConfigBean{} ciConfigRequest.CiBuildConfig.CiBuildType = "skip-build" ciConfigRequest.UserId = patchRequest.UserId - if patchRequest.CiPipeline == nil || patchRequest.CiPipeline.CiMaterial == nil || len(patchRequest.CiPipeline.CiMaterial) != 1 { + if patchRequest.CiPipeline == nil || patchRequest.CiPipeline.CiMaterial == nil { handler.Logger.Errorw("Invalid patch ci-pipeline request", "request", patchRequest, "err", "invalid CiPipeline data") common.WriteJsonResp(w, fmt.Errorf("invalid CiPipeline data"), nil, http.StatusBadRequest) return From 126fc153e7122119ae0520c9e9f7a5dff9f15f97 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 6 Mar 2023 19:50:43 +0530 Subject: [PATCH 079/118] removed the commented lines. --- pkg/pipeline/WebhookService.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index c215493fbe..9756331208 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -124,7 +124,6 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } savedWorkflow.Status = string(v1alpha1.NodeSucceeded) - //savedWorkflow.FinishedOn = time.Now() impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) if err != nil { From 0b94d6cf4f79dbddc3e3f4b1f4b21424080adf0b Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 9 Mar 2023 16:01:42 +0530 Subject: [PATCH 080/118] Minor bug fixes --- api/restHandler/AppListingRestHandler.go | 5 +++++ api/restHandler/app/BuildPipelineRestHandler.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 4132db0342..c99b180973 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -229,6 +229,11 @@ func (handler AppListingRestHandlerImpl) FetchOverviewCiPipelines(w http.Respons common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + _, err = handler.pipeline.GetApp(jobId) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } jobCi, err := handler.appListingService.FetchOverviewCiPipelines(jobId) if err != nil { diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 9c15ef7b9c..969e27c750 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -231,6 +231,13 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + appIsJob, err := handler.pipelineBuilder.GetApp(app.Id) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + app.IsJob = appIsJob.IsJob + if app.IsJob { isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { From 8efbdddf66bd254fdd68ec88cc75331229a3cea4 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 9 Mar 2023 16:04:38 +0530 Subject: [PATCH 081/118] Delete Git material bug fix. --- api/restHandler/app/BuildPipelineRestHandler.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 969e27c750..abdd0401c8 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -1032,8 +1032,6 @@ func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrite } var deleteMaterial bean.UpdateMaterialDTO err = decoder.Decode(&deleteMaterial) - var updateMaterialDto bean.UpdateMaterialDTO - err = decoder.Decode(&updateMaterialDto) deleteMaterial.UserId = userId if err != nil { handler.Logger.Errorw("request err, DeleteMaterial", "err", err, "DeleteMaterial", deleteMaterial) From a04a1fd04591d222c5338e51a1b1d40be10216c0 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 9 Mar 2023 18:32:01 +0530 Subject: [PATCH 082/118] Fixed minor bug --- api/restHandler/app/DeploymentPipelineRestHandler.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 26b9c70337..ebd3a1c248 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -177,7 +177,10 @@ func (handler PipelineConfigRestHandlerImpl) CreateCdPipeline(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - + if app.IsJob { + common.WriteJsonResp(w, fmt.Errorf("cannot create cd-pipeline for job"), "cannot create cd-pipeline for job", http.StatusBadRequest) + return + } //RBAC resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { @@ -710,6 +713,10 @@ func (handler PipelineConfigRestHandlerImpl) GetCdPipelines(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + if app.IsJob { + common.WriteJsonResp(w, fmt.Errorf("cd-pipeline for job does not exist"), "cd-pipeline for job does not exist", http.StatusBadRequest) + return + } resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !ok { common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) From 92850fed0b562c3e7585834a03b7e30c58556424 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 9 Mar 2023 18:44:25 +0530 Subject: [PATCH 083/118] Wrote the sql down command. --- scripts/sql/115_jobs.down.sql | 92 ++++++----------------------------- 1 file changed, 16 insertions(+), 76 deletions(-) diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/115_jobs.down.sql index 3458f020d1..53ddfedd54 100644 --- a/scripts/sql/115_jobs.down.sql +++ b/scripts/sql/115_jobs.down.sql @@ -1,32 +1,32 @@ -ALTER TABLE app_workflow DROP CONSTRAINT app_workflow_app_id_fkey, ADD CONSTRAINT app_workflow_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; +DELETE FROM ci_pipeline_material WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); -ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_app_id_fkey, ADD CONSTRAINT ci_pipeline_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; +DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2))); -ALTER TABLE ci_template DROP CONSTRAINT ci_template_app_id_fkey, ADD CONSTRAINT ci_template_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; +DELETE FROM ci_pipeline_history WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); -ALTER TABLE git_material DROP CONSTRAINT git_material_app_id_fkey, ADD CONSTRAINT git_material_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; +DELETE FROM ci_artifact WHERE ci_workflow_id IN (SELECT id FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2))))); -ALTER TABLE app_label DROP CONSTRAINT app_label_app_id_fkey, ADD CONSTRAINT app_label_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; +DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); -ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_history_app_id_fkey, ADD CONSTRAINT ci_template_history_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id) ON DELETE CASCADE; +DELETE FROM pipeline_stage WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); -ALTER TABLE app_workflow_mapping DROP CONSTRAINT app_workflow_mapping_app_workflow_id_fkey, ADD CONSTRAINT app_workflow_mapping_app_workflow_id_fkey FOREIGN KEY (app_workflow_id) REFERENCES app_workflow(id) ON DELETE CASCADE; +DELETE FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2))); -ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey, ADD CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id) ON DELETE CASCADE; +DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); -ALTER TABLE ci_pipeline_history DROP CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk, ADD CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id) ON DELETE CASCADE; +DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); -ALTER TABLE git_material_history DROP CONSTRAINT git_material_history_git_material_id_fkey, ADD CONSTRAINT git_material_history_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; +DELETE FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); -ALTER TABLE ci_workflow DROP CONSTRAINT ci_workflow_ci_pipeline_id_fkey, ADD CONSTRAINT ci_workflow_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id) ON DELETE CASCADE; +DELETE FROM git_material_history WHERE git_material_id IN(SELECT id FROM git_material WHERE app_id IN(SELECT id FROM app WHERE app_store = 2)); -ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_git_material_id_fkey, ADD CONSTRAINT ci_pipeline_material_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; +DELETE FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2); -ALTER TABLE ci_template DROP CONSTRAINT ci_template_git_material_id_fkey, ADD CONSTRAINT ci_template_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; +DELETE FROM app_label WHERE app_id IN (SELECT id FROM app WHERE app_store = 2); -ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_git_material_history_id_fkey, ADD CONSTRAINT ci_template_git_material_history_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id) ON DELETE CASCADE; +DELETE FROM app_workflow_mapping WHERE app_workflow_id IN(SELECT id FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); -ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_ci_template_id_fkey, ADD CONSTRAINT ci_pipeline_ci_template_id_fkey FOREIGN KEY (ci_template_id) REFERENCES ci_template(id) ON DELETE CASCADE; +DELETE FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_store = 2); DELETE FROM app WHERE app_store = 2; @@ -40,64 +40,4 @@ ALTER TABLE app DROP COLUMN display_name; ALTER TABLE app DROP COLUMN description; -ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_ci_template_id_fkey; - -ALTER TABLE ci_pipeline ADD CONSTRAINT ci_pipeline_ci_template_id_fkey FOREIGN KEY (ci_template_id) REFERENCES ci_template(id); - -ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_git_material_history_id_fkey; - -ALTER TABLE ci_template_history ADD CONSTRAINT ci_template_git_material_history_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); - -ALTER TABLE ci_template DROP CONSTRAINT ci_template_git_material_id_fkey; - -ALTER TABLE ci_template ADD CONSTRAINT ci_template_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); - -ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_git_material_id_fkey; - -ALTER TABLE ci_pipeline_material ADD CONSTRAINT ci_pipeline_material_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); - -ALTER TABLE ci_workflow DROP CONSTRAINT ci_workflow_ci_pipeline_id_fkey; - -ALTER TABLE ci_workflow ADD CONSTRAINT ci_workflow_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id); - -ALTER TABLE git_material_history DROP CONSTRAINT git_material_history_git_material_id_fkey; - -ALTER TABLE git_material_history ADD CONSTRAINT git_material_history_git_material_id_fkey FOREIGN KEY (git_material_id) REFERENCES git_material(id); - -ALTER TABLE ci_pipeline_history DROP CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk; - -ALTER TABLE ci_pipeline_history ADD CONSTRAINT ci_pipeline_history_ci_pipeline_id_fk FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id); - -ALTER TABLE ci_pipeline_material DROP CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey; - -ALTER TABLE ci_pipeline_material ADD CONSTRAINT ci_pipeline_material_ci_pipeline_id_fkey FOREIGN KEY (ci_pipeline_id) REFERENCES ci_pipeline(id); - -ALTER TABLE app_workflow_mapping DROP CONSTRAINT app_workflow_mapping_app_workflow_id_fkey; - -ALTER TABLE app_workflow_mapping ADD CONSTRAINT app_workflow_mapping_app_workflow_id_fkey FOREIGN KEY (app_workflow_id) REFERENCES app_workflow(id); - -ALTER TABLE ci_template_history DROP CONSTRAINT ci_template_history_app_id_fkey; - -ALTER TABLE ci_template_history ADD CONSTRAINT ci_template_history_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); - -ALTER TABLE app_label DROP CONSTRAINT app_label_app_id_fkey; - -ALTER TABLE app_label ADD CONSTRAINT app_label_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); - -ALTER TABLE git_material DROP CONSTRAINT git_material_app_id_fkey; - -ALTER TABLE git_material ADD CONSTRAINT git_material_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); - -ALTER TABLE ci_template DROP CONSTRAINT ci_template_app_id_fkey; - -ALTER TABLE git_material ADD CONSTRAINT ci_template_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); - -ALTER TABLE ci_pipeline DROP CONSTRAINT ci_pipeline_app_id_fkey; - -ALTER TABLE ci_pipeline ADD CONSTRAINT ci_pipeline_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); - -ALTER TABLE app_workflow DROP CONSTRAINT app_workflow_app_id_fkey; - -ALTER TABLE app_workflow ADD CONSTRAINT app_workflow_app_id_fkey FOREIGN KEY (app_id) REFERENCES app(id); - -ALTER TABLE ci_artifact DROP COLUMN is_artifact_uploaded; +ALTER TABLE ci_artifact DROP COLUMN is_artifact_uploaded; \ No newline at end of file From cd201620b208508a4e30739230724a48f1c8d727 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 9 Mar 2023 19:52:01 +0530 Subject: [PATCH 084/118] minor fix --- api/restHandler/AppListingRestHandler.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index c99b180973..c6a953e8ee 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -188,7 +188,6 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, "", http.StatusInternalServerError) return } - // Apply pagination jobsCount := len(jobs) offset := fetchJobListingRequest.Offset limit := fetchJobListingRequest.Size From 30ade14aae9c6a843d640d56464741f499ece2f8 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 10 Mar 2023 11:35:14 +0530 Subject: [PATCH 085/118] resolved the comments of PR. --- api/restHandler/app/BuildPipelineRestHandler.go | 10 ---------- internal/sql/repository/AppListingRepository.go | 3 +-- .../helper/AppListingRepositoryQueryBuilder.go | 2 +- pkg/app/AppListingService.go | 6 +++--- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index abdd0401c8..d4d66d3fcc 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -238,16 +238,6 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri } app.IsJob = appIsJob.IsJob - if app.IsJob { - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - } resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) var ok bool if patchRequest.IsJob { diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 9aa2e81352..97e161eee0 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -118,9 +118,8 @@ func (impl AppListingRepositoryImpl) FetchJobsLastSucceededOn(CiPipelineIDs []in jobsLastFinishedOnQuery := impl.appListingRepositoryQueryBuilder.JobsLastSucceededOnTimeQuery(CiPipelineIDs) impl.Logger.Debugw("basic app detail query: ", jobsLastFinishedOnQuery) _, appsErr := impl.dbConnection.Query(&lastSucceededTimeArray, jobsLastFinishedOnQuery) - impl.Logger.Debugw("basic app detail query: ", jobsLastFinishedOnQuery) if appsErr != nil { - impl.Logger.Error(appsErr) + impl.Logger.Errorw("error in fetching lastSucceededTimeArray", "error", appsErr, jobsLastFinishedOnQuery) return lastSucceededTimeArray, appsErr } return lastSucceededTimeArray, nil diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 3f6f06a573..2f0eb19c28 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -151,7 +151,7 @@ func (impl AppListingRepositoryQueryBuilder) buildJobListingWhereCondition(jobLi if jobListingFilter.AppNameSearch != "" { likeClause := "'%" + jobListingFilter.AppNameSearch + "%'" - whereCondition = whereCondition + "and a.app_name like " + likeClause + " " + whereCondition = whereCondition + "and a.display_name like " + likeClause + " " } // add job stats filter here if len(jobListingFilter.AppStatuses) > 0 { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 0e78f1a09b..9d9b7e1184 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -235,12 +235,12 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi } appIds, err := impl.appRepository.FetchAppIdsWithFilter(jobListingFilter) if err != nil { - impl.Logger.Errorw("error in fetching app ids list", "error", err) + impl.Logger.Errorw("error in fetching app ids list", "error", err, jobListingFilter) return []*bean.JobContainer{}, err } jobListingContainers, err := impl.appListingRepository.FetchJobs(appIds, jobListingFilter.AppStatuses, string(jobListingFilter.SortOrder)) if err != nil { - impl.Logger.Errorw("error in fetching app list", "error", err) + impl.Logger.Errorw("error in fetching job list", "error", err, jobListingFilter) return []*bean.JobContainer{}, err } CiPipelineIDs := GetCIPipelineIDs(jobListingContainers) @@ -252,7 +252,7 @@ func (impl AppListingServiceImpl) FetchJobs(fetchJobListingRequest FetchAppListi func (impl AppListingServiceImpl) FetchOverviewCiPipelines(jobId int) ([]*bean.JobListingContainer, error) { jobCiContainers, err := impl.appListingRepository.FetchOverviewCiPipelines(jobId) if err != nil { - impl.Logger.Errorw("error in fetching app list", "error", err) + impl.Logger.Errorw("error in fetching job container", "error", err, jobId) return []*bean.JobListingContainer{}, err } return jobCiContainers, nil From e8d5971a8c852fd4590ae0ecd66e1199c31866a0 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 10 Mar 2023 15:42:37 +0530 Subject: [PATCH 086/118] fixed a minor bug --- api/restHandler/app/DeploymentPipelineRestHandler.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index ebd3a1c248..8c5841f2e6 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -713,10 +713,6 @@ func (handler PipelineConfigRestHandlerImpl) GetCdPipelines(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if app.IsJob { - common.WriteJsonResp(w, fmt.Errorf("cd-pipeline for job does not exist"), "cd-pipeline for job does not exist", http.StatusBadRequest) - return - } resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !ok { common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) From fc670891d938f63402f1bf732adfefe579bfb8a9 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 10 Mar 2023 21:21:00 +0530 Subject: [PATCH 087/118] Changed from app_store to app_type --- api/restHandler/AppListingRestHandler.go | 11 +- .../app/AutoCompleteRestHandler.go | 12 +- .../app/BuildPipelineRestHandler.go | 24 +-- .../app/DeploymentPipelineRestHandler.go | 5 +- .../app/PipelineConfigRestHandler.go | 5 +- api/router/JobsRouter.go | 3 +- .../sql/repository/AppListingRepository.go | 20 +-- internal/sql/repository/app/AppRepository.go | 48 +++--- .../appStatus/AppStatusRepository.go | 2 +- .../AppListingRepositoryQueryBuilder.go | 14 +- .../repository/security/CveStoreRepository.go | 2 +- pkg/app/AppCrudOperationService.go | 11 +- pkg/app/AppListingService.go | 2 +- pkg/appClone/AppCloneService.go | 28 ++-- .../service/AppStoreDeploymentService.go | 3 +- pkg/bean/app.go | 3 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 18 +-- pkg/pipeline/PipelineBuilder.go | 13 +- pkg/security/ImageScanService.go | 9 +- pkg/security/policyService.go | 9 +- scripts/sql/115_jobs.down.sql | 40 ++--- scripts/sql/115_jobs.up.sql | 2 + specs/jobs.yaml | 146 +----------------- 23 files changed, 157 insertions(+), 273 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index c6a953e8ee..88f35be6da 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" @@ -58,7 +59,7 @@ import ( type AppListingRestHandler interface { FetchAppsByEnvironment(w http.ResponseWriter, r *http.Request) FetchJobs(w http.ResponseWriter, r *http.Request) - FetchOverviewCiPipelines(w http.ResponseWriter, r *http.Request) + FetchJobOverviewCiPipelines(w http.ResponseWriter, r *http.Request) FetchAppDetails(w http.ResponseWriter, r *http.Request) FetchAllDevtronManagedApps(w http.ResponseWriter, r *http.Request) FetchAppTriggerView(w http.ResponseWriter, r *http.Request) @@ -206,7 +207,7 @@ func (handler AppListingRestHandlerImpl) FetchJobs(w http.ResponseWriter, r *htt common.WriteJsonResp(w, err, jobContainerResponse, http.StatusOK) } -func (handler AppListingRestHandlerImpl) FetchOverviewCiPipelines(w http.ResponseWriter, r *http.Request) { +func (handler AppListingRestHandlerImpl) FetchJobOverviewCiPipelines(w http.ResponseWriter, r *http.Request) { userId, err := handler.userService.GetLoggedInUser(r) if userId == 0 || err != nil { handler.logger.Errorw("request err, userId", "err", err, "payload", userId) @@ -641,8 +642,8 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit return } v := r.URL.Query() - isJobParam := v.Get("isJob") - isJob := isJobParam == "true" + appTypeParam := v.Get("appType") + appType, err := strconv.Atoi(appTypeParam) handler.logger.Infow("request payload, FetchAppStageStatus", "appId", appId) token := r.Header.Get("token") @@ -661,7 +662,7 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit } //RBAC enforcer Ends - triggerView, err := handler.appListingService.FetchAppStageStatus(appId, isJob) + triggerView, err := handler.appListingService.FetchAppStageStatus(appId, appType == int(helper.Job)) if err != nil { handler.logger.Errorw("service err, FetchAppStageStatus", "err", err, "appId", appId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/app/AutoCompleteRestHandler.go b/api/restHandler/app/AutoCompleteRestHandler.go index 65035a5060..a6dfb216f7 100644 --- a/api/restHandler/app/AutoCompleteRestHandler.go +++ b/api/restHandler/app/AutoCompleteRestHandler.go @@ -2,6 +2,7 @@ package app import ( "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/user/casbin" "github.com/gorilla/mux" @@ -32,12 +33,17 @@ func (handler PipelineConfigRestHandlerImpl) GetAppListForAutocomplete(w http.Re v := r.URL.Query() teamId := v.Get("teamId") appName := v.Get("appName") - isJobParam := v.Get("isJob") - isJob := isJobParam == "true" + appTypeParam := v.Get("appType") + appType, err := strconv.Atoi(appTypeParam) + if err != nil { + handler.Logger.Errorw("service err, GetAppListForAutocomplete", "err", err, "teamId", teamId, "appTypeParam", appTypeParam) + common.WriteJsonResp(w, err, "Failed to parse appType param", http.StatusInternalServerError) + return + } handler.Logger.Infow("request payload, GetAppListForAutocomplete", "teamId", teamId) var apps []*pipeline.AppBean if len(teamId) == 0 { - apps, err = handler.pipelineBuilder.FindAllMatchesByAppName(appName, isJob) + apps, err = handler.pipelineBuilder.FindAllMatchesByAppName(appName, helper.AppType(appType)) if err != nil { handler.Logger.Errorw("service err, GetAppListForAutocomplete", "err", err, "teamId", teamId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index d4d66d3fcc..4c8c09beb9 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -9,6 +9,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" @@ -209,6 +210,11 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + if err != nil { + common.WriteJsonResp(w, err, "failed to check if user is super admin", http.StatusInternalServerError) + return + } var patchRequest bean.CiPatchRequest err = decoder.Decode(&patchRequest) patchRequest.UserId = userId @@ -231,17 +237,11 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - appIsJob, err := handler.pipelineBuilder.GetApp(app.Id) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - app.IsJob = appIsJob.IsJob resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) var ok bool - if patchRequest.IsJob { - ok = handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionCreate, resourceName) + if app.AppType == helper.Job { + ok = isSuperAdmin } else { ok = handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName) } @@ -253,7 +253,7 @@ func (handler PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWri ciConf, err := handler.pipelineBuilder.GetCiPipeline(patchRequest.AppId) var emptyDockerRegistry string - if patchRequest.IsJob && ciConf == nil { + if app.AppType == helper.Job && ciConf == nil { ciConfigRequest := bean.CiConfigRequest{} ciConfigRequest.DockerRegistry = emptyDockerRegistry ciConfigRequest.AppId = patchRequest.AppId @@ -907,7 +907,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if app.IsJob { + if app.AppType == helper.Job { isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { if err != nil { @@ -987,7 +987,7 @@ func (handler PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if app.IsJob { + if app.AppType == helper.Job { isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { if err != nil { @@ -1041,7 +1041,7 @@ func (handler PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if app.IsJob { + if app.AppType == helper.Job { isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { if err != nil { diff --git a/api/restHandler/app/DeploymentPipelineRestHandler.go b/api/restHandler/app/DeploymentPipelineRestHandler.go index 8c5841f2e6..3ec44cb14a 100644 --- a/api/restHandler/app/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/DeploymentPipelineRestHandler.go @@ -7,6 +7,7 @@ import ( "fmt" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" @@ -177,7 +178,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateCdPipeline(w http.ResponseWri common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if app.IsJob { + if app.AppType == helper.Job { common.WriteJsonResp(w, fmt.Errorf("cannot create cd-pipeline for job"), "cannot create cd-pipeline for job", http.StatusBadRequest) return } @@ -837,7 +838,7 @@ func (handler PipelineConfigRestHandlerImpl) GetArtifactsByCDPipeline(w http.Res } if len(digests) > 0 { vulnerableMap := make(map[string]bool) - cvePolicy, severityPolicy, err := handler.policyService.GetApplicablePolicy(pipelineModel.Environment.ClusterId, pipelineModel.EnvironmentId, pipelineModel.AppId, pipelineModel.App.AppStore == 1) + cvePolicy, severityPolicy, err := handler.policyService.GetApplicablePolicy(pipelineModel.Environment.ClusterId, pipelineModel.EnvironmentId, pipelineModel.AppId, pipelineModel.App.AppType == helper.ChartStoreApp) if err != nil { handler.Logger.Errorw("service err, GetArtifactsByCDPipeline", "err", err, "cdPipelineId", cdPipelineId, "stage", stage) } diff --git a/api/restHandler/app/PipelineConfigRestHandler.go b/api/restHandler/app/PipelineConfigRestHandler.go index 751255854a..098e4e241f 100644 --- a/api/restHandler/app/PipelineConfigRestHandler.go +++ b/api/restHandler/app/PipelineConfigRestHandler.go @@ -22,6 +22,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/util/argo" "io" @@ -207,7 +208,7 @@ func (handler PipelineConfigRestHandlerImpl) DeleteApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if app.IsJob { + if app.AppType == helper.Job { isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { if err != nil { @@ -248,7 +249,7 @@ func (handler PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if createRequest.IsJob { + if createRequest.AppType == helper.Job { isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) if !isSuperAdmin || err != nil { if err != nil { diff --git a/api/router/JobsRouter.go b/api/router/JobsRouter.go index e87508fef1..e6130ae93e 100644 --- a/api/router/JobsRouter.go +++ b/api/router/JobsRouter.go @@ -23,7 +23,6 @@ func NewJobRouterImpl(pipelineConfigRestHandler app.PipelineConfigRestHandler, a } func (router JobRouterImpl) InitJobRouter(jobRouter *mux.Router) { jobRouter.Path("").HandlerFunc(router.pipelineConfigRestHandler.CreateApp).Methods("POST") - jobRouter.Path("/ci-pipeline/patch").HandlerFunc(router.pipelineConfigRestHandler.PatchCiPipelines).Methods("POST") jobRouter.Path("/list").HandlerFunc(router.appListingRestHandler.FetchJobs).Methods("POST") - jobRouter.Path("/ci-pipeline/list/{jobId}").HandlerFunc(router.appListingRestHandler.FetchOverviewCiPipelines).Methods("GET") + jobRouter.Path("/ci-pipeline/list/{jobId}").HandlerFunc(router.appListingRestHandler.FetchJobOverviewCiPipelines).Methods("GET") } diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 97e161eee0..54d68244db 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -44,7 +44,7 @@ type AppListingRepository interface { FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) FetchAppTriggerView(appId int) ([]bean.TriggerView, error) - FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) + FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) //Not in used PrometheusApiByEnvId(id int) (*string, error) @@ -213,7 +213,7 @@ func (impl AppListingRepositoryImpl) DeploymentDetailsByAppIdAndEnvId(ctx contex " INNER JOIN ci_artifact cia on cia.id = pco.ci_artifact_id" + " INNER JOIN app a ON a.id=p.app_id" + " LEFT JOIN users u on u.id=pco.created_by" + - " WHERE a.app_store is false AND a.id=? AND env.id=? AND p.deleted = FALSE AND env.active = TRUE" + + " WHERE a.app_type = 0 AND a.id=? AND env.id=? AND p.deleted = FALSE AND env.active = TRUE" + " ORDER BY pco.created_on desc limit 1;" impl.Logger.Debugf("query:", query) _, err := impl.dbConnection.Query(&deploymentDetail, query, appId, envId) @@ -353,7 +353,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig " INNER JOIN ci_pipeline cp on cp.id = p.ci_pipeline_id" + " INNER JOIN app a ON a.id = p.app_id" + " INNER JOIN environment env on env.id = p.environment_id" + - " WHERE p.app_id=? and p.deleted=false and a.app_store = 0 AND env.active = TRUE;" + " WHERE p.app_id=? and p.deleted=false and a.app_type = 0 AND env.active = TRUE;" impl.Logger.Debugw("query", query) _, err := impl.dbConnection.Query(&triggerView, query, appId) @@ -409,7 +409,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig return triggerViewResponse, nil } -func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) { +func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) { impl.Logger.Debug("reached at AppListingRepository:") var appStageStatus []bean.AppStageStatus @@ -432,14 +432,14 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, isJob bool) " LEFT JOIN charts ch on ch.app_id=app.id" + " LEFT JOIN pipeline p on p.app_id=app.id" + " LEFT JOIN chart_env_config_override ceco on ceco.chart_id=ch.id" + - " WHERE app.id=? and app.app_store = ?;" + " WHERE app.id=? and app.app_type = ?;" impl.Logger.Debugw("last app stages status query:", "query", query) - appStore := 0 - if isJob { - appStore = 2 - } - _, err := impl.dbConnection.Query(&stages, query, appId, appStore) + appType := helper.CustomApp + //if isJob { + // appType = helper.Job + //} + _, err := impl.dbConnection.Query(&stages, query, appId, appType) if err != nil { impl.Logger.Errorw("error:", err) return appStageStatus, err diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 922ae29286..2ab72c7652 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -27,15 +27,15 @@ import ( ) type App struct { - tableName struct{} `sql:"app" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - AppName string `sql:"app_name,notnull"` //same as app name - DisplayName string `sql:"display_name"` - Active bool `sql:"active, notnull"` - TeamId int `sql:"team_id"` - AppStore int `sql:"app_store, notnull"` - AppOfferingMode string `sql:"app_offering_mode,notnull"` - Description string `sql:"description"` + tableName struct{} `sql:"app" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppName string `sql:"app_name,notnull"` //same as app name + DisplayName string `sql:"display_name"` + Active bool `sql:"active, notnull"` + TeamId int `sql:"team_id"` + AppType helper.AppType `sql:"app_type, notnull"` + AppOfferingMode string `sql:"app_offering_mode,notnull"` + Description string `sql:"description"` Team team.Team sql.AuditLog } @@ -62,7 +62,7 @@ type AppRepository interface { FindAppAndProjectByAppId(appId int) (*App, error) FindAppAndProjectByAppName(appName string) (*App, error) GetConnection() *pg.DB - FindAllMatchesByAppName(appName string, isJob bool) ([]*App, error) + FindAllMatchesByAppName(appName string, appType helper.AppType) ([]*App, error) FindIdsByTeamIdsAndTeamNames(teamIds []int, teamNames []string) ([]int, error) FindIdsByNames(appNames []string) ([]int, error) FetchAllActiveInstalledAppsWithAppIdAndName() ([]*App, error) @@ -128,7 +128,7 @@ func (repo AppRepositoryImpl) FindJobByDisplayName(appName string) (*App, error) Model(pipelineGroup). Where("display_name = ?", appName). Where("active = ?", true). - Where("app_store = ?", 2). + Where("app_type = ?", 2). Order("id DESC").Limit(1). Select() // there is only single active app will be present in db with a same name. @@ -177,7 +177,7 @@ func (repo AppRepositoryImpl) FindAppsByTeamIds(teamId []int, appType string) ([ } var apps []App err := repo.dbConnection.Model(&apps).Column("app.*", "Team").Where("team_id in (?)", pg.In(teamId)). - Where("app.active=?", true).Where("app.app_store=?", onlyDevtronCharts).Select() + Where("app.active=?", true).Where("app.app_type=?", onlyDevtronCharts).Select() return apps, err } @@ -192,7 +192,7 @@ func (repo AppRepositoryImpl) FindAppsByTeamName(teamName string) ([]App, error) func (repo AppRepositoryImpl) FindAll() ([]*App, error) { var apps []*App - err := repo.dbConnection.Model(&apps).Where("active = ?", true).Where("app_store = ?", 0).Select() + err := repo.dbConnection.Model(&apps).Where("active = ?", true).Where("app_type = ?", 0).Select() return apps, err } @@ -207,7 +207,7 @@ func (repo AppRepositoryImpl) FindAppsByEnvironmentId(environmentId int) ([]App, func (repo AppRepositoryImpl) FindAllActiveAppsWithTeam() ([]*App, error) { var apps []*App err := repo.dbConnection.Model(&apps).Column("Team"). - Where("app.active = ?", true).Where("app.app_store = ?", 0). + Where("app.active = ?", true).Where("app.app_type = ?", 0). Select() return apps, err } @@ -225,25 +225,25 @@ func (repo AppRepositoryImpl) FetchAppsByFilterV2(appNameIncludes string, appNam err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Join("inner join pipeline p on p.app_id=app.id"). Where("app.app_name like ?", ""+appNameIncludes+"%").Where("app.app_name not like ?", ""+appNameExcludes+"%"). - Where("app.active=?", true).Where("app_store=?", 0). + Where("app.active=?", true).Where("app_type=?", 0). Where("p.environment_id = ?", environmentId).Where("p.deleted = ?", false). Select() } else if environmentId > 0 && appNameExcludes == "" { err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Join("inner join pipeline p on p.app_id=app.id"). Where("app.app_name like ?", ""+appNameIncludes+"%"). - Where("app.active=?", true).Where("app_store=?", 0). + Where("app.active=?", true).Where("app_type=?", 0). Where("p.environment_id = ?", environmentId).Where("p.deleted = ?", false). Select() } else if environmentId == 0 && len(appNameExcludes) > 0 { err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Where("app.app_name like ?", ""+appNameIncludes+"%").Where("app.app_name not like ?", ""+appNameExcludes+"%"). - Where("app.active=?", true).Where("app_store=?", 0). + Where("app.active=?", true).Where("app_type=?", 0). Select() } else if environmentId == 0 && appNameExcludes == "" { err = repo.dbConnection.Model(&apps).ColumnExpr("DISTINCT app.*"). Where("app.app_name like ?", ""+appNameIncludes+"%"). - Where("app.active=?", true).Where("app_store=?", 0). + Where("app.active=?", true).Where("app_type=?", 0). Select() } return apps, err @@ -267,13 +267,9 @@ func (repo AppRepositoryImpl) FindAppAndProjectByAppName(appName string) (*App, return app, err } -func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string, isJob bool) ([]*App, error) { +func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string, appType helper.AppType) ([]*App, error) { var apps []*App - appStore := 0 - if isJob { - appStore = 2 - } - err := repo.dbConnection.Model(&apps).Where("display_name LIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_store = ?", appStore).Select() + err := repo.dbConnection.Model(&apps).Where("display_name LIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_type = ?", appType).Select() return apps, err } @@ -335,7 +331,7 @@ func (repo AppRepositoryImpl) FetchAllActiveDevtronAppsWithAppIdAndName() ([]*Ap err := repo.dbConnection.Model(&apps). Column("id", "app_name"). - Where("app_store = ?", 0). + Where("app_type = ?", 0). Where("active", true). Select() if err != nil && err != pg.ErrNoRows { @@ -360,7 +356,7 @@ func (repo AppRepositoryImpl) FetchAppIdsWithFilter(jobListingFilter helper.AppL Id int `json:"id"` } var jobIds []AppId - whereCondition := " where active = true and app_store = 2 " + whereCondition := " where active = true and app_type = 2 " if len(jobListingFilter.Teams) > 0 { whereCondition += " and team_id in (" + helper.GetCommaSepratedString(jobListingFilter.Teams) + ")" } diff --git a/internal/sql/repository/appStatus/AppStatusRepository.go b/internal/sql/repository/appStatus/AppStatusRepository.go index 0a90f0da49..22f84f793f 100644 --- a/internal/sql/repository/appStatus/AppStatusRepository.go +++ b/internal/sql/repository/appStatus/AppStatusRepository.go @@ -13,7 +13,7 @@ type AppStatusContainer struct { InstalledAppId int `json:"installed_app_id"` EnvId int `json:"env_id"` Status string `json:"status"` - AppStore bool `json:"app_store"` + AppType int `json:"app_type"` UpdatedOn time.Time `json:"updated_on"` } diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 2f0eb19c28..71c339932c 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -25,6 +25,14 @@ import ( "strings" ) +type AppType int + +const ( + CustomApp AppType = 0 + ChartStoreApp AppType = 1 + Job AppType = 2 +) + type AppListingRepositoryQueryBuilder struct { logger *zap.SugaredLogger } @@ -67,7 +75,7 @@ func (impl AppListingRepositoryQueryBuilder) BuildJobListingQuery(appIDs []int, " from ci_workflow cw inner join (select ci_pipeline_id, MAX(started_on) max_started_on from ci_workflow group by ci_pipeline_id ) " + "cws on cw.ci_pipeline_id = cws.ci_pipeline_id " + "and cw.started_on = cws.max_started_on order by cw.ci_pipeline_id) cwr on cwr.ci_pipeline_id = ci_pipeline.id" + - " where app.active = true and app.app_store = 2 " + " where app.active = true and app.app_type = 2 " if len(appIDs) > 0 { query += "and app.id IN (" + GetCommaSepratedString(appIDs) + ") " } @@ -142,7 +150,7 @@ func (impl AppListingRepositoryQueryBuilder) buildJobListingSortBy(appListingFil } func (impl AppListingRepositoryQueryBuilder) buildJobListingWhereCondition(jobListingFilter AppListingFilter) string { - whereCondition := "WHERE a.active = true and a.app_store = 2 " + whereCondition := "WHERE a.active = true and a.app_type = 2 " if len(jobListingFilter.Teams) > 0 { teamIds := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(jobListingFilter.Teams)), ","), "[]") @@ -162,7 +170,7 @@ func (impl AppListingRepositoryQueryBuilder) buildJobListingWhereCondition(jobLi } func (impl AppListingRepositoryQueryBuilder) buildAppListingWhereCondition(appListingFilter AppListingFilter) string { - whereCondition := "WHERE a.active = true and a.app_store = 0 " + whereCondition := "WHERE a.active = true and a.app_type = 0 " if len(appListingFilter.Environments) > 0 { envIds := strings.Trim(strings.Join(strings.Fields(fmt.Sprint(appListingFilter.Environments)), ","), "[]") whereCondition = whereCondition + "and env.id IN (" + envIds + ") " diff --git a/internal/sql/repository/security/CveStoreRepository.go b/internal/sql/repository/security/CveStoreRepository.go index 333e959242..e990346604 100644 --- a/internal/sql/repository/security/CveStoreRepository.go +++ b/internal/sql/repository/security/CveStoreRepository.go @@ -117,7 +117,7 @@ func (impl CveStoreRepositoryImpl) Update(team *CveStore) error { func (impl CveStoreRepositoryImpl) VulnerabilityExposure(request *VulnerabilityRequest) ([]*VulnerabilityExposure, error) { var items []*VulnerabilityExposure - query := "SELECT a.id as app_id, a.app_name, a.app_store, p.environment_id as pipeline_env_id, ia.environment_id as chart_env_id " + + query := "SELECT a.id as app_id, a.app_name, a.app_type, p.environment_id as pipeline_env_id, ia.environment_id as chart_env_id " + " FROM app a" + " LEFT JOIN pipeline p ON p.app_id=a.id" + " LEFT JOIN installed_apps ia ON ia.app_id=a.id" + diff --git a/pkg/app/AppCrudOperationService.go b/pkg/app/AppCrudOperationService.go index c7da823f73..4b7bddc694 100644 --- a/pkg/app/AppCrudOperationService.go +++ b/pkg/app/AppCrudOperationService.go @@ -20,7 +20,8 @@ package app import ( "encoding/json" "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository/app" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/bean" @@ -49,13 +50,13 @@ type AppCrudOperationService interface { type AppCrudOperationServiceImpl struct { logger *zap.SugaredLogger appLabelRepository pipelineConfig.AppLabelRepository - appRepository app.AppRepository + appRepository appRepository.AppRepository userRepository repository.UserRepository installedAppRepository repository2.InstalledAppRepository } func NewAppCrudOperationServiceImpl(appLabelRepository pipelineConfig.AppLabelRepository, - logger *zap.SugaredLogger, appRepository app.AppRepository, userRepository repository.UserRepository, installedAppRepository repository2.InstalledAppRepository) *AppCrudOperationServiceImpl { + logger *zap.SugaredLogger, appRepository appRepository.AppRepository, userRepository repository.UserRepository, installedAppRepository repository2.InstalledAppRepository) *AppCrudOperationServiceImpl { return &AppCrudOperationServiceImpl{ appLabelRepository: appLabelRepository, logger: logger, @@ -315,7 +316,7 @@ func (impl AppCrudOperationServiceImpl) GetAppMetaInfo(appId int) (*bean.AppMeta } } appName := app.AppName - if app.AppStore == 2 { + if app.AppType == helper.Job { appName = app.DisplayName } info := &bean.AppMetaInfoDto{ @@ -337,7 +338,7 @@ func (impl AppCrudOperationServiceImpl) GetHelmAppMetaInfo(appId string) (*bean. // adding separate function for helm apps because for CLI helm apps, apps can be of form "1|clusterName|releaseName" // In this case app details can be fetched using app name / release Name. appIdSplitted := strings.Split(appId, "|") - app := &app.App{} + app := &appRepository.App{} var err error impl.logger.Info("request payload, appId", appId) if len(appIdSplitted) > 1 { diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 9d9b7e1184..834ffe3b26 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -1533,7 +1533,7 @@ func (impl AppListingServiceImpl) FetchAppTriggerView(appId int) ([]bean.Trigger } func (impl AppListingServiceImpl) FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) { - appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId, isJob) + appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId) if isJob { for i := range appStageStatuses { if appStageStatuses[i].StageName == "TEMPLATE" || appStageStatuses[i].StageName == "CHART" || appStageStatuses[i].StageName == "CHART_ENV_CONFIG" { diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index e477965806..7ece7bc9f0 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -22,6 +22,7 @@ import ( bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/constants" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/chart" "github.com/go-pg/pg" @@ -81,12 +82,12 @@ func NewAppCloneServiceImpl(logger *zap.SugaredLogger, } type CloneRequest struct { - RefAppId int `json:"refAppId"` - Name string `json:"name"` - ProjectId int `json:"projectId"` - AppLabels []*bean.Label `json:"labels,omitempty" validate:"dive"` - Description string `json:"description"` - isJob bool `json:"isJob"` + RefAppId int `json:"refAppId"` + Name string `json:"name"` + ProjectId int `json:"projectId"` + AppLabels []*bean.Label `json:"labels,omitempty" validate:"dive"` + Description string `json:"description"` + AppType helper.AppType `json:"appType"` } func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context context.Context) (*bean.CreateAppDTO, error) { @@ -95,7 +96,10 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context if err != nil && err != pg.ErrNoRows { return nil, err } - if (templateApp == nil && templateApp.Id == 0) || (createReq.IsJob == false && templateApp.AppStore != 0) || (createReq.IsJob == true && templateApp.AppStore != 2) { + //If the template does not exist then don't clone + //If the template app-type is chart-store app then don't clone + //If the template app-type and create request app-type is not same then don't clone + if (templateApp == nil && templateApp.Id == 0) || (templateApp.AppType == helper.ChartStoreApp) || (templateApp.AppType != createReq.AppType) { impl.logger.Warnw("template app does not exist", "id", createReq.TemplateId) err = &util.ApiError{ Code: constants.AppDoesNotExist.Code, @@ -110,11 +114,11 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context Name: createReq.AppName, ProjectId: createReq.TeamId, AppLabels: createReq.AppLabels, - isJob: createReq.IsJob, + AppType: createReq.AppType, Description: createReq.Description, } userId := createReq.UserId - appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId, cloneReq.isJob) + appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId, cloneReq.AppType == helper.Job) if err != nil { return nil, err } @@ -136,7 +140,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context } //TODO check stage of current app - if !createReq.IsJob { + if createReq.AppType != helper.Job { if !refAppStatus["APP"] { impl.logger.Warnw("status not", "APP", cloneReq.RefAppId) return nil, nil @@ -163,7 +167,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context impl.logger.Errorw("error in cloning docker template", "ref", cloneReq.RefAppId, "new", newAppId, "err", err) return nil, err } - if !createReq.IsJob { + if createReq.AppType != helper.Job { if !refAppStatus["TEMPLATE"] { impl.logger.Errorw("status not", "TEMPLATE", cloneReq.RefAppId) return app, nil @@ -221,7 +225,7 @@ func (impl *AppCloneServiceImpl) CreateApp(cloneReq *CloneRequest, userId int32) UserId: userId, TeamId: cloneReq.ProjectId, AppLabels: cloneReq.AppLabels, - IsJob: cloneReq.isJob, + AppType: cloneReq.AppType, Description: cloneReq.Description, } createRes, err := impl.pipelineBuilder.CreateApp(createAppReq) diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index ab0fcdac28..6f776d53c5 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/internal/constants" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDeploymentCommon "github.com/devtron-labs/devtron/pkg/appStore/deployment/common" @@ -410,7 +411,7 @@ func (impl AppStoreDeploymentServiceImpl) createAppForAppStore(createRequest *be Active: true, AppName: createRequest.AppName, TeamId: createRequest.TeamId, - AppStore: 1, + AppType: helper.ChartStoreApp, AppOfferingMode: appInstallationMode, AuditLog: sql.AuditLog{UpdatedBy: createRequest.UserId, CreatedBy: createRequest.UserId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } diff --git a/pkg/bean/app.go b/pkg/bean/app.go index a61398521d..c99ca0db8b 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -19,6 +19,7 @@ package bean import ( "encoding/json" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -46,7 +47,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` - IsJob bool `json:"isJob"` + AppType helper.AppType `json:"isJob"` } type CreateMaterialDTO struct { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 15c525c58a..8399845065 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -27,6 +27,7 @@ import ( "fmt" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -813,7 +814,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp } // Rollback tx on error. defer tx.Rollback() - app, err := impl.createAppGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.IsJob, createRequest.Description, tx) + app, err := impl.createAppGroup(createRequest.AppName, createRequest.UserId, createRequest.TeamId, createRequest.AppType, createRequest.Description, tx) if err != nil { return nil, err } @@ -840,7 +841,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateApp(createRequest *bean.CreateApp return nil, err } createRequest.Id = app.Id - if createRequest.IsJob { + if createRequest.AppType == helper.Job { createRequest.AppName = app.DisplayName } return createRequest, nil @@ -998,12 +999,12 @@ func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*b } // FIXME: not thread safe -func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int32, teamId int, isJob bool, description string, tx *pg.Tx) (*app2.App, error) { +func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int32, teamId int, appType helper.AppType, description string, tx *pg.Tx) (*app2.App, error) { app, err := impl.appRepository.FindActiveByName(name) if err != nil && err != pg.ErrNoRows { return nil, err } - if !isJob { + if appType != helper.Job { if app != nil && app.Id > 0 { impl.logger.Warnw("app already exists", "name", name) err = &util.ApiError{ @@ -1013,8 +1014,7 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 } return nil, err } - } - if isJob { + } else { job, err := impl.appRepository.FindJobByDisplayName(name) if err != nil && err != pg.ErrNoRows { return nil, err @@ -1030,11 +1030,9 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 } } - appStore := 0 displayName := name appName := name - if isJob { - appStore = 2 + if appType == helper.Job { appName = name + "/" + util2.Generate(8) + "J" } pg := &app2.App{ @@ -1042,7 +1040,7 @@ func (impl CiCdPipelineOrchestratorImpl) createAppGroup(name string, userId int3 AppName: appName, DisplayName: displayName, TeamId: teamId, - AppStore: appStore, + AppType: appType, Description: description, AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index e24b54d9f1..108a1439bb 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -25,6 +25,7 @@ import ( client "github.com/devtron-labs/devtron/api/helm-app" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/security" "github.com/devtron-labs/devtron/pkg/chart" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -125,7 +126,7 @@ type PipelineBuilder interface { GetCiPipelineById(pipelineId int) (ciPipeline *bean.CiPipeline, err error) GetMaterialsForAppId(appId int) []*bean.GitMaterial - FindAllMatchesByAppName(appName string, isJob bool) ([]*AppBean, error) + FindAllMatchesByAppName(appName string, appType helper.AppType) ([]*AppBean, error) GetEnvironmentByCdPipelineId(pipelineId int) (int, error) PatchRegexCiPipeline(request *bean.CiRegexPatchRequest) (err error) @@ -402,7 +403,7 @@ func (impl PipelineBuilderImpl) GetApp(appId int) (application *bean.CreateAppDT } gitMaterials := impl.GetMaterialsForAppId(appId) - if app.AppStore == 2 { + if app.AppType == helper.Job { app.AppName = app.DisplayName } application = &bean.CreateAppDTO{ @@ -410,7 +411,7 @@ func (impl PipelineBuilderImpl) GetApp(appId int) (application *bean.CreateAppDT AppName: app.AppName, Material: gitMaterials, TeamId: app.TeamId, - IsJob: app.AppStore == 2, + AppType: app.AppType, } return application, nil } @@ -3305,14 +3306,14 @@ func (impl PipelineBuilderImpl) GetCiPipelineById(pipelineId int) (ciPipeline *b return ciPipeline, err } -func (impl PipelineBuilderImpl) FindAllMatchesByAppName(appName string, isJob bool) ([]*AppBean, error) { +func (impl PipelineBuilderImpl) FindAllMatchesByAppName(appName string, appType helper.AppType) ([]*AppBean, error) { var appsRes []*AppBean var apps []*app2.App var err error if len(appName) == 0 { apps, err = impl.appRepo.FindAll() } else { - apps, err = impl.appRepo.FindAllMatchesByAppName(appName, isJob) + apps, err = impl.appRepo.FindAllMatchesByAppName(appName, appType) } if err != nil { impl.logger.Errorw("error while fetching app", "err", err) @@ -3320,7 +3321,7 @@ func (impl PipelineBuilderImpl) FindAllMatchesByAppName(appName string, isJob bo } for _, app := range apps { name := app.AppName - if isJob { + if appType == helper.Job { name = app.DisplayName } appsRes = append(appsRes, &AppBean{Id: app.Id, Name: name}) diff --git a/pkg/security/ImageScanService.go b/pkg/security/ImageScanService.go index eecabf6cfd..014497b774 100644 --- a/pkg/security/ImageScanService.go +++ b/pkg/security/ImageScanService.go @@ -18,7 +18,8 @@ package security import ( - "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository1 "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" repository2 "github.com/devtron-labs/devtron/pkg/team" "time" @@ -48,7 +49,7 @@ type ImageScanServiceImpl struct { imageScanDeployInfoRepository security.ImageScanDeployInfoRepository userService user.UserService teamRepository repository2.TeamRepository - appRepository app.AppRepository + appRepository repository1.AppRepository envService cluster.EnvironmentService ciArtifactRepository repository.CiArtifactRepository policyService PolicyService @@ -123,7 +124,7 @@ func NewImageScanServiceImpl(Logger *zap.SugaredLogger, scanHistoryRepository se scanResultRepository security.ImageScanResultRepository, scanObjectMetaRepository security.ImageScanObjectMetaRepository, cveStoreRepository security.CveStoreRepository, imageScanDeployInfoRepository security.ImageScanDeployInfoRepository, userService user.UserService, teamRepository repository2.TeamRepository, - appRepository app.AppRepository, + appRepository repository1.AppRepository, envService cluster.EnvironmentService, ciArtifactRepository repository.CiArtifactRepository, policyService PolicyService, pipelineRepository pipelineConfig.PipelineRepository, ciPipelineRepository pipelineConfig.CiPipelineRepository) *ImageScanServiceImpl { return &ImageScanServiceImpl{Logger: Logger, scanHistoryRepository: scanHistoryRepository, scanResultRepository: scanResultRepository, @@ -409,7 +410,7 @@ func (impl ImageScanServiceImpl) FetchExecutionDetailResult(request *ImageScanRe imageScanResponse.EnvId = request.EnvId imageScanResponse.EnvName = env.Environment - blockCveList, err := impl.policyService.GetBlockedCVEList(cveStores, env.ClusterId, env.Id, request.AppId, app.AppStore == 1) + blockCveList, err := impl.policyService.GetBlockedCVEList(cveStores, env.ClusterId, env.Id, request.AppId, app.AppType == helper.ChartStoreApp) if err != nil { impl.Logger.Errorw("error while fetching env", "err", err) //return nil, err diff --git a/pkg/security/policyService.go b/pkg/security/policyService.go index 8d36c6f4a1..8d2e3ace45 100644 --- a/pkg/security/policyService.go +++ b/pkg/security/policyService.go @@ -21,7 +21,8 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository1 "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/sql" "net/http" "strings" @@ -52,7 +53,7 @@ type PolicyService interface { type PolicyServiceImpl struct { environmentService cluster.EnvironmentService logger *zap.SugaredLogger - apRepository app.AppRepository + apRepository repository1.AppRepository pipelineOverride chartConfig.PipelineOverrideRepository cvePolicyRepository security.CvePolicyRepository clusterService cluster.ClusterService @@ -70,7 +71,7 @@ type PolicyServiceImpl struct { func NewPolicyServiceImpl(environmentService cluster.EnvironmentService, logger *zap.SugaredLogger, - apRepository app.AppRepository, + apRepository repository1.AppRepository, pipelineOverride chartConfig.PipelineOverrideRepository, cvePolicyRepository security.CvePolicyRepository, clusterService cluster.ClusterService, @@ -190,7 +191,7 @@ func (impl *PolicyServiceImpl) VerifyImage(verifyImageRequest *VerifyImageReques return nil, err } else if app != nil { appId = app.Id - isAppStore = app.AppStore == 1 + isAppStore = app.AppType == helper.ChartStoreApp } else { //np app do nothing } diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/115_jobs.down.sql index 53ddfedd54..9328ae840f 100644 --- a/scripts/sql/115_jobs.down.sql +++ b/scripts/sql/115_jobs.down.sql @@ -1,40 +1,42 @@ -DELETE FROM ci_pipeline_material WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); +DELETE FROM ci_pipeline_material WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); -DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2))); +DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2))); -DELETE FROM ci_pipeline_history WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); +DELETE FROM ci_pipeline_history WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); -DELETE FROM ci_artifact WHERE ci_workflow_id IN (SELECT id FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2))))); +DELETE FROM ci_artifact WHERE ci_workflow_id IN (SELECT id FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2))))); -DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); +DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); -DELETE FROM pipeline_stage WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); +DELETE FROM pipeline_stage WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); -DELETE FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2))); +DELETE FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2))); -DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)))); +DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); -DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); +DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); -DELETE FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); +DELETE FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); -DELETE FROM git_material_history WHERE git_material_id IN(SELECT id FROM git_material WHERE app_id IN(SELECT id FROM app WHERE app_store = 2)); +DELETE FROM git_material_history WHERE git_material_id IN(SELECT id FROM git_material WHERE app_id IN(SELECT id FROM app WHERE app_type = 2)); -DELETE FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_store = 2); +DELETE FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM app_label WHERE app_id IN (SELECT id FROM app WHERE app_store = 2); +DELETE FROM app_label WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM app_workflow_mapping WHERE app_workflow_id IN(SELECT id FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_store = 2)); +DELETE FROM app_workflow_mapping WHERE app_workflow_id IN(SELECT id FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); -DELETE FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_store = 2); +DELETE FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM app WHERE app_store = 2; +DELETE FROM app WHERE app_type = 2; -ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; +ALTER TABLE app ALTER COLUMN app_type DROP DEFAULT; -ALTER TABLE app ALTER app_store TYPE boolean USING CASE WHEN app_store=1 THEN true ELSE false end; +ALTER TABLE app ALTER app_type TYPE boolean USING CASE WHEN app_type=1 THEN true ELSE false end; -ALTER TABLE app ALTER COLUMN app_store SET DEFAULT FALSE; +ALTER TABLE app ALTER COLUMN app_type SET DEFAULT FALSE; + +ALTER TABLE app RENAME COLUMN app_type TO app_store; ALTER TABLE app DROP COLUMN display_name; diff --git a/scripts/sql/115_jobs.up.sql b/scripts/sql/115_jobs.up.sql index 635caf44e8..1fbc1cfafe 100644 --- a/scripts/sql/115_jobs.up.sql +++ b/scripts/sql/115_jobs.up.sql @@ -1,7 +1,9 @@ ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=true THEN 1 ELSE 0 end; +ALTER TABLE app RENAME COLUMN app_store TO app_type; ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; ALTER TABLE app ADD COLUMN display_name varchar(250); ALTER TABLE app ADD COLUMN description text; ALTER TABLE ci_artifact ADD COLUMN is_artifact_uploaded BOOLEAN NOT NULL DEFAULT FALSE; + diff --git a/specs/jobs.yaml b/specs/jobs.yaml index ce47951eb8..635136ad5a 100644 --- a/specs/jobs.yaml +++ b/specs/jobs.yaml @@ -46,42 +46,6 @@ paths: required: true schema: type: integer - /orchestrator/job/ci-pipeline/patch: - post: - description: Create/Update ci pipeline. - operationId: PatchCiPipeline - requestBody: - description: A JSON object containing the pipeline configuration - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/CiPatchRequest" - responses: - "200": - description: Successfully return a message stating the operation is successful. - content: - application/json: - schema: - type: string - "400": - description: Bad Request. Validation error/wrong request body. - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - "500": - description: Internal Server Error - content: - application/json: - schema: - $ref: "#/components/schemas/Error" - "403": - description: Unauthorized User - content: - application/json: - schema: - $ref: "#/components/schemas/Error" #components components: schemas: @@ -220,113 +184,9 @@ components: type: time.Time jobCount: type: integer - Error: - required: - - code - - message - properties: - code: - type: integer - description: Error code - message: - type: string - description: Error message - CiPatchRequest: - type: object - properties: - action: - type: integer - appId: - type: integer - isJob: - type: boolean - example: true - appWorkflowId: - type: integer - ciPipeline: - $ref: "#/components/schemas/CiPipelineDetails" - CiPipelineDetails: - type: object - properties: - isDockerConfigOverridden: - type: boolean - dockerConfigOverride: - $ref: "#/components/schemas/DockerBuildConfig" - name: - type: string - isManual: - type: boolean - scanEnabled: - type: boolean - isExternal: - type: boolean - parentAppId: - type: integer - parentCiPipeline: - type: integer - linkedCount: - type: integer - ciMaterials: - type: array - items: - $ref: "#/components/schemas/CiPipelineMaterialConfig" - dockerArgs: - type: array - items: - type: object - properties: - Key: - type: string - Value: - type: string - description: map of docker arguments, i.e. key-value pairs - beforeDockerBuildScripts: - type: array - items: - $ref: "#/components/schemas/BuildScript" - afterDockerbuildScripts: - type: array - items: - $ref: "#/components/schemas/BuildScript" - DockerBuildConfig: - type: object - properties: - gitMaterialId: - type: integer - dockerfileRelativePath: - type: string - targetPlatform: - type: string - args: - type: array - items: - type: object - properties: - Key: - type: string - Value: - type: string - description: map of docker arguments, i.e. key-value pairs - CiPipelineMaterialConfig: - type: object - properties: - type: - type: string - value: - type: string - checkoutPath: - type: string - BuildScript: - type: object - properties: - index: - type: integer - name: - type: string - script: - type: string - reportDirectoryPath: - type: string + + + From b3031f29d5fbd7eaf805b6f6c6c3574a50f4a3b6 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 10 Mar 2023 23:22:53 +0530 Subject: [PATCH 088/118] Changed from app_store to app_type --- .../app/BuildPipelineRestHandler.go | 20 +++++++++---------- pkg/bean/app.go | 2 -- scripts/sql/115_jobs.up.sql | 1 + 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 4c8c09beb9..f0200b30c6 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -403,16 +403,16 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) //RBAC CHECK CD PIPELINE - FOR USER - if ciTriggerRequest.IsJob { - isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - if !isSuperAdmin || err != nil { - if err != nil { - handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - } - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - return - } - } + //if ciTriggerRequest.IsJob { + // isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) + // if !isSuperAdmin || err != nil { + // if err != nil { + // handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) + // } + // common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + // return + // } + //} pipelines, err := handler.pipelineRepository.FindByCiPipelineId(ciTriggerRequest.PipelineId) if err != nil { handler.Logger.Errorw("error in finding ccd pipelines by ciPipelineId", "err", err) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index c99ca0db8b..92721c98e7 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -208,7 +208,6 @@ type CiPatchRequest struct { Action PatchAction `json:"action"` AppWorkflowId int `json:"appWorkflowId,omitempty"` UserId int32 `json:"-"` - IsJob bool `json:"isJob"` } type CiRegexPatchRequest struct { @@ -260,7 +259,6 @@ type CiTriggerRequest struct { CiPipelineMaterial []CiPipelineMaterial `json:"ciPipelineMaterials" validate:"required"` TriggeredBy int32 `json:"triggeredBy"` InvalidateCache bool `json:"invalidateCache"` - IsJob bool `json:"isJob"` } type CiTrigger struct { diff --git a/scripts/sql/115_jobs.up.sql b/scripts/sql/115_jobs.up.sql index 1fbc1cfafe..cc09d0bba5 100644 --- a/scripts/sql/115_jobs.up.sql +++ b/scripts/sql/115_jobs.up.sql @@ -5,5 +5,6 @@ ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; ALTER TABLE app ADD COLUMN display_name varchar(250); ALTER TABLE app ADD COLUMN description text; ALTER TABLE ci_artifact ADD COLUMN is_artifact_uploaded BOOLEAN NOT NULL DEFAULT FALSE; +UPDATE ci_artifact SET is_artifact_uploaded = true; From 2a399f4369b23f1583fb1765aba2d5ff7e53e213 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 11 Mar 2023 18:15:01 +0530 Subject: [PATCH 089/118] changes as mentioned in the PR --- api/restHandler/app/AutoCompleteRestHandler.go | 13 ++++++++----- internal/sql/repository/app/AppRepository.go | 10 ++++++++-- .../helper/AppListingRepositoryQueryBuilder.go | 6 +++--- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/api/restHandler/app/AutoCompleteRestHandler.go b/api/restHandler/app/AutoCompleteRestHandler.go index a6dfb216f7..03ae78aa71 100644 --- a/api/restHandler/app/AutoCompleteRestHandler.go +++ b/api/restHandler/app/AutoCompleteRestHandler.go @@ -34,11 +34,14 @@ func (handler PipelineConfigRestHandlerImpl) GetAppListForAutocomplete(w http.Re teamId := v.Get("teamId") appName := v.Get("appName") appTypeParam := v.Get("appType") - appType, err := strconv.Atoi(appTypeParam) - if err != nil { - handler.Logger.Errorw("service err, GetAppListForAutocomplete", "err", err, "teamId", teamId, "appTypeParam", appTypeParam) - common.WriteJsonResp(w, err, "Failed to parse appType param", http.StatusInternalServerError) - return + var appType int + if appTypeParam != "" { + appType, err = strconv.Atoi(appTypeParam) + if err != nil { + handler.Logger.Errorw("service err, GetAppListForAutocomplete", "err", err, "teamId", teamId, "appTypeParam", appTypeParam) + common.WriteJsonResp(w, err, "Failed to parse appType param", http.StatusInternalServerError) + return + } } handler.Logger.Infow("request payload, GetAppListForAutocomplete", "teamId", teamId) var apps []*pipeline.AppBean diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 2ab72c7652..3bf254eb5d 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -128,7 +128,7 @@ func (repo AppRepositoryImpl) FindJobByDisplayName(appName string) (*App, error) Model(pipelineGroup). Where("display_name = ?", appName). Where("active = ?", true). - Where("app_type = ?", 2). + Where("app_type = ?", helper.Job). Order("id DESC").Limit(1). Select() // there is only single active app will be present in db with a same name. @@ -269,7 +269,13 @@ func (repo AppRepositoryImpl) FindAppAndProjectByAppName(appName string) (*App, func (repo AppRepositoryImpl) FindAllMatchesByAppName(appName string, appType helper.AppType) ([]*App, error) { var apps []*App - err := repo.dbConnection.Model(&apps).Where("display_name LIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_type = ?", appType).Select() + var err error + if appType == helper.Job { + err = repo.dbConnection.Model(&apps).Where("display_name LIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_type = ?", appType).Select() + } else { + err = repo.dbConnection.Model(&apps).Where("app_name LIKE ?", "%"+appName+"%").Where("active = ?", true).Where("app_type = ?", appType).Select() + } + return apps, err } diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 71c339932c..b167f79565 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -28,9 +28,9 @@ import ( type AppType int const ( - CustomApp AppType = 0 - ChartStoreApp AppType = 1 - Job AppType = 2 + CustomApp AppType = 0 // cicd app + ChartStoreApp AppType = 1 // helm app + Job AppType = 2 // jobs ) type AppListingRepositoryQueryBuilder struct { From c0e3ed7cf13b7e0abf84b580d4ac7e75979f5480 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 11 Mar 2023 18:46:30 +0530 Subject: [PATCH 090/118] Checked the ci_artifacts --- internal/sql/repository/pipelineConfig/CiWorkflowRepository.go | 3 ++- pkg/pipeline/CiHandler.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index a6a3df2177..662964223d 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -87,6 +87,7 @@ type WorkflowWithArtifact struct { CiArtifactId int `json:"ci_artifact_d"` BlobStorageEnabled bool `json:"blobStorageEnabled"` CiBuildType string `json:"ci_build_type"` + IsArtifactUploaded bool `json:"is_artifact_uploaded"` } type GitCommit struct { @@ -158,7 +159,7 @@ func (impl *CiWorkflowRepositoryImpl) FindByStatusesIn(activeStatuses []string) func (impl *CiWorkflowRepositoryImpl) FindByPipelineId(pipelineId int, offset int, limit int) ([]WorkflowWithArtifact, error) { var wfs []WorkflowWithArtifact - queryTemp := "select cia.id as ci_artifact_id, cia.image, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id where wf.ci_pipeline_id = ? order by wf.started_on desc offset ? limit ?;" + queryTemp := "select cia.id as ci_artifact_id, cia.image, cia.is_artifact_uploaded, wf.*, u.email_id from ci_workflow wf left join users u on u.id = wf.triggered_by left join ci_artifact cia on wf.id = cia.ci_workflow_id where wf.ci_pipeline_id = ? order by wf.started_on desc offset ? limit ?;" _, err := impl.dbConnection.Query(&wfs, queryTemp, pipelineId, offset, limit) if err != nil { return nil, err diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 8770f5a928..346ee6e6be 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -398,6 +398,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, offset int, size int) TriggeredByEmail: w.EmailId, ArtifactId: w.CiArtifactId, BlobStorageEnabled: w.BlobStorageEnabled, + IsArtifactUploaded: w.IsArtifactUploaded, } ciWorkLowResponses = append(ciWorkLowResponses, wfResponse) } From b9ff2a5eae9565904fa8066cfa178e0363dbff26 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 11 Mar 2023 19:16:54 +0530 Subject: [PATCH 091/118] Changes done as mentioned in the PR --- api/restHandler/AppListingRestHandler.go | 7 +------ pkg/app/AppListingService.go | 13 ++----------- pkg/appClone/AppCloneService.go | 2 +- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 88f35be6da..8fb97774e5 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -30,7 +30,6 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/internal/constants" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" @@ -641,10 +640,6 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - v := r.URL.Query() - appTypeParam := v.Get("appType") - appType, err := strconv.Atoi(appTypeParam) - handler.logger.Infow("request payload, FetchAppStageStatus", "appId", appId) token := r.Header.Get("token") app, err := handler.pipeline.GetApp(appId) @@ -662,7 +657,7 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit } //RBAC enforcer Ends - triggerView, err := handler.appListingService.FetchAppStageStatus(appId, appType == int(helper.Job)) + triggerView, err := handler.appListingService.FetchAppStageStatus(appId) if err != nil { handler.logger.Errorw("service err, FetchAppStageStatus", "err", err, "appId", appId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 834ffe3b26..60c612ab88 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -80,7 +80,7 @@ type AppListingService interface { GraphAPI(appId int, envId int) error FetchAppTriggerView(appId int) ([]bean.TriggerView, error) - FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) + FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) FetchOtherEnvironment(ctx context.Context, appId int) ([]*bean.Environment, error) RedirectToLinkouts(Id int, appId int, envId int, podName string, containerName string) (string, error) @@ -1532,17 +1532,8 @@ func (impl AppListingServiceImpl) FetchAppTriggerView(appId int) ([]bean.Trigger return impl.appListingRepository.FetchAppTriggerView(appId) } -func (impl AppListingServiceImpl) FetchAppStageStatus(appId int, isJob bool) ([]bean.AppStageStatus, error) { +func (impl AppListingServiceImpl) FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) { appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId) - if isJob { - for i := range appStageStatuses { - if appStageStatuses[i].StageName == "TEMPLATE" || appStageStatuses[i].StageName == "CHART" || appStageStatuses[i].StageName == "CHART_ENV_CONFIG" { - appStageStatuses[i].Status = true - } else if appStageStatuses[i].StageName == "APP" || appStageStatuses[i].StageName == "CD_PIPELINE" { - appStageStatuses[i].Status = false - } - } - } return appStageStatuses, err } diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index 7ece7bc9f0..dc0568773f 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -118,7 +118,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context Description: createReq.Description, } userId := createReq.UserId - appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId, cloneReq.AppType == helper.Job) + appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) if err != nil { return nil, err } From b8daf29c46b388317a85507e5ca6cf4ab0caec5c Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Sat, 11 Mar 2023 19:19:12 +0530 Subject: [PATCH 092/118] Changes done as mentioned in the PR --- api/restHandler/app/BuildPipelineRestHandler.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index f0200b30c6..edc9f1fae8 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -403,16 +403,6 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) //RBAC CHECK CD PIPELINE - FOR USER - //if ciTriggerRequest.IsJob { - // isSuperAdmin, err := handler.userAuthService.IsSuperAdmin(int(userId)) - // if !isSuperAdmin || err != nil { - // if err != nil { - // handler.Logger.Errorw("request err, CheckSuperAdmin", "err", isSuperAdmin, "isSuperAdmin", isSuperAdmin) - // } - // common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) - // return - // } - //} pipelines, err := handler.pipelineRepository.FindByCiPipelineId(ciTriggerRequest.PipelineId) if err != nil { handler.Logger.Errorw("error in finding ccd pipelines by ciPipelineId", "err", err) From 7714bc1dc8ba68c2a6864523c3d0fea1bcd26dba Mon Sep 17 00:00:00 2001 From: Kripansh Date: Mon, 13 Mar 2023 13:05:07 +0530 Subject: [PATCH 093/118] data optimization added --- internal/sql/repository/AppListingRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 54d68244db..9396f04b62 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -432,7 +432,7 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppS " LEFT JOIN charts ch on ch.app_id=app.id" + " LEFT JOIN pipeline p on p.app_id=app.id" + " LEFT JOIN chart_env_config_override ceco on ceco.chart_id=ch.id" + - " WHERE app.id=? and app.app_type = ?;" + " WHERE app.id=? and app.app_type = ? limit 1;" impl.Logger.Debugw("last app stages status query:", "query", query) appType := helper.CustomApp From d3e8872b80ec36cac4eee1ef4a57d245d85d7c7e Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 14:13:23 +0530 Subject: [PATCH 094/118] Fixed minor bug --- pkg/bean/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 92721c98e7..f9951ca62c 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -47,7 +47,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` - AppType helper.AppType `json:"isJob"` + AppType helper.AppType `json:"appType"` } type CreateMaterialDTO struct { From 089fa82e47dee4dd34dff81296ccafd4571a6715 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 14:24:52 +0530 Subject: [PATCH 095/118] Fixed minor bug --- internal/sql/repository/app/AppRepository.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 3bf254eb5d..d7ecf51f8e 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -159,7 +159,8 @@ func (repo AppRepositoryImpl) CheckAppExists(appNames []string) ([]*App, error) func (repo AppRepositoryImpl) FindById(id int) (*App, error) { pipelineGroup := &App{} - err := repo.dbConnection.Model(pipelineGroup).Where("id = ?", id).Select() + err := repo.dbConnection.Model(pipelineGroup).Where("id = ?", id). + Where("active = ?", true).Select() return pipelineGroup, err } From a2c82e42390986635e5d6d232ce9fca4f243011d Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 16:12:59 +0530 Subject: [PATCH 096/118] Jobs sql up number update --- scripts/sql/{115_jobs.down.sql => 122_jobs.down.sql} | 0 scripts/sql/{115_jobs.up.sql => 122_jobs.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{115_jobs.down.sql => 122_jobs.down.sql} (100%) rename scripts/sql/{115_jobs.up.sql => 122_jobs.up.sql} (100%) diff --git a/scripts/sql/115_jobs.down.sql b/scripts/sql/122_jobs.down.sql similarity index 100% rename from scripts/sql/115_jobs.down.sql rename to scripts/sql/122_jobs.down.sql diff --git a/scripts/sql/115_jobs.up.sql b/scripts/sql/122_jobs.up.sql similarity index 100% rename from scripts/sql/115_jobs.up.sql rename to scripts/sql/122_jobs.up.sql From 782d9e60b5a28de5a40a4729dd528acb8c18e389 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 16:46:53 +0530 Subject: [PATCH 097/118] Bug fix --- api/restHandler/app/BuildPipelineRestHandler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index edc9f1fae8..e4f3372dc0 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -409,6 +409,11 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } + if pipelines[0] != nil && (pipelines[0].App.AppType == helper.Job && ciTriggerRequest.InvalidateCache == true) { + handler.Logger.Errorw("Invalidate cache option is not valid for job ci-pipeline", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } var authorizedPipelines []pipelineConfig.Pipeline var unauthorizedPipelines []pipelineConfig.Pipeline token := r.Header.Get("token") From 8d17528b0128499243341edddb396f935d28186b Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 17:00:26 +0530 Subject: [PATCH 098/118] Bug fix --- api/restHandler/app/BuildPipelineRestHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index e4f3372dc0..7e75d90b4a 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -409,7 +409,7 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - if pipelines[0] != nil && (pipelines[0].App.AppType == helper.Job && ciTriggerRequest.InvalidateCache == true) { + if pipelines != nil && (pipelines[0].App.AppType == helper.Job && ciTriggerRequest.InvalidateCache == true) { handler.Logger.Errorw("Invalidate cache option is not valid for job ci-pipeline", "err", err) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return From 86a7d31e0c30af99b6439886aaec20dfc147d78d Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 17:09:41 +0530 Subject: [PATCH 099/118] Bug fix --- api/restHandler/AppListingRestHandler.go | 6 ++++-- api/restHandler/app/BuildPipelineRestHandler.go | 5 ----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 8fb97774e5..603870235a 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/internal/constants" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" @@ -228,8 +229,9 @@ func (handler AppListingRestHandlerImpl) FetchJobOverviewCiPipelines(w http.Resp common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - _, err = handler.pipeline.GetApp(jobId) - if err != nil { + job, err := handler.pipeline.GetApp(jobId) + if job.AppType != helper.Job || err != nil { + handler.logger.Errorw("Job with the given Id does not exist") common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 7e75d90b4a..edc9f1fae8 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -409,11 +409,6 @@ func (handler PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWr common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - if pipelines != nil && (pipelines[0].App.AppType == helper.Job && ciTriggerRequest.InvalidateCache == true) { - handler.Logger.Errorw("Invalidate cache option is not valid for job ci-pipeline", "err", err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } var authorizedPipelines []pipelineConfig.Pipeline var unauthorizedPipelines []pipelineConfig.Pipeline token := r.Header.Get("token") From 0296ede032625f4590e2ecfba60344337f5e9152 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 17:32:16 +0530 Subject: [PATCH 100/118] Bug fix --- api/restHandler/AppListingRestHandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 603870235a..79708cba3c 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -231,8 +231,8 @@ func (handler AppListingRestHandlerImpl) FetchJobOverviewCiPipelines(w http.Resp } job, err := handler.pipeline.GetApp(jobId) if job.AppType != helper.Job || err != nil { - handler.logger.Errorw("Job with the given Id does not exist") - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + handler.logger.Errorw("Job with the given Id does not exist", "err", err, "jobId", jobId) + common.WriteJsonResp(w, err, "Job with the given Id does not exist", http.StatusBadRequest) return } From d511439519e1a0371b39e6db1dc294999021950b Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 18:04:20 +0530 Subject: [PATCH 101/118] Bug fix --- scripts/sql/{122_jobs.down.sql => 121_jobs.down.sql} | 0 scripts/sql/{122_jobs.up.sql => 121_jobs.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{122_jobs.down.sql => 121_jobs.down.sql} (100%) rename scripts/sql/{122_jobs.up.sql => 121_jobs.up.sql} (100%) diff --git a/scripts/sql/122_jobs.down.sql b/scripts/sql/121_jobs.down.sql similarity index 100% rename from scripts/sql/122_jobs.down.sql rename to scripts/sql/121_jobs.down.sql diff --git a/scripts/sql/122_jobs.up.sql b/scripts/sql/121_jobs.up.sql similarity index 100% rename from scripts/sql/122_jobs.up.sql rename to scripts/sql/121_jobs.up.sql From 3ffdfe2568f55defdc6671d7179281b0fc18acba Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 13 Mar 2023 19:30:32 +0530 Subject: [PATCH 102/118] Bug fix --- api/restHandler/AppListingRestHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 79708cba3c..893827978a 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -230,7 +230,7 @@ func (handler AppListingRestHandlerImpl) FetchJobOverviewCiPipelines(w http.Resp return } job, err := handler.pipeline.GetApp(jobId) - if job.AppType != helper.Job || err != nil { + if err != nil || job == nil || job.AppType != helper.Job { handler.logger.Errorw("Job with the given Id does not exist", "err", err, "jobId", jobId) common.WriteJsonResp(w, err, "Job with the given Id does not exist", http.StatusBadRequest) return From ebb9cb4864560b74f7af6111470b081161af0a34 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 14 Mar 2023 16:36:07 +0530 Subject: [PATCH 103/118] Bug fix --- api/restHandler/AppListingRestHandler.go | 2 +- internal/sql/repository/AppListingRepository.go | 9 +++------ pkg/app/AppListingService.go | 6 +++--- pkg/appClone/AppCloneService.go | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/api/restHandler/AppListingRestHandler.go b/api/restHandler/AppListingRestHandler.go index 893827978a..d53c4b3d73 100644 --- a/api/restHandler/AppListingRestHandler.go +++ b/api/restHandler/AppListingRestHandler.go @@ -659,7 +659,7 @@ func (handler AppListingRestHandlerImpl) FetchAppStageStatus(w http.ResponseWrit } //RBAC enforcer Ends - triggerView, err := handler.appListingService.FetchAppStageStatus(appId) + triggerView, err := handler.appListingService.FetchAppStageStatus(appId, int(app.AppType)) if err != nil { handler.logger.Errorw("service err, FetchAppStageStatus", "err", err, "appId", appId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 9396f04b62..793eaa543b 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -44,7 +44,7 @@ type AppListingRepository interface { FetchAppDetail(ctx context.Context, appId int, envId int) (bean.AppDetailContainer, error) FetchAppTriggerView(appId int) ([]bean.TriggerView, error) - FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) + FetchAppStageStatus(appId int, appType int) ([]bean.AppStageStatus, error) //Not in used PrometheusApiByEnvId(id int) (*string, error) @@ -409,7 +409,7 @@ func (impl AppListingRepositoryImpl) FetchAppTriggerView(appId int) ([]bean.Trig return triggerViewResponse, nil } -func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) { +func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, appType int) ([]bean.AppStageStatus, error) { impl.Logger.Debug("reached at AppListingRepository:") var appStageStatus []bean.AppStageStatus @@ -435,10 +435,7 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int) ([]bean.AppS " WHERE app.id=? and app.app_type = ? limit 1;" impl.Logger.Debugw("last app stages status query:", "query", query) - appType := helper.CustomApp - //if isJob { - // appType = helper.Job - //} + _, err := impl.dbConnection.Query(&stages, query, appId, appType) if err != nil { impl.Logger.Errorw("error:", err) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 60c612ab88..e8750434b9 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -80,7 +80,7 @@ type AppListingService interface { GraphAPI(appId int, envId int) error FetchAppTriggerView(appId int) ([]bean.TriggerView, error) - FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) + FetchAppStageStatus(appId int, appType int) ([]bean.AppStageStatus, error) FetchOtherEnvironment(ctx context.Context, appId int) ([]*bean.Environment, error) RedirectToLinkouts(Id int, appId int, envId int, podName string, containerName string) (string, error) @@ -1532,8 +1532,8 @@ func (impl AppListingServiceImpl) FetchAppTriggerView(appId int) ([]bean.Trigger return impl.appListingRepository.FetchAppTriggerView(appId) } -func (impl AppListingServiceImpl) FetchAppStageStatus(appId int) ([]bean.AppStageStatus, error) { - appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId) +func (impl AppListingServiceImpl) FetchAppStageStatus(appId int, appType int) ([]bean.AppStageStatus, error) { + appStageStatuses, err := impl.appListingRepository.FetchAppStageStatus(appId, appType) return appStageStatuses, err } diff --git a/pkg/appClone/AppCloneService.go b/pkg/appClone/AppCloneService.go index dc0568773f..bde508f85d 100644 --- a/pkg/appClone/AppCloneService.go +++ b/pkg/appClone/AppCloneService.go @@ -118,7 +118,7 @@ func (impl *AppCloneServiceImpl) CloneApp(createReq *bean.CreateAppDTO, context Description: createReq.Description, } userId := createReq.UserId - appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId) + appStatus, err := impl.appListingService.FetchAppStageStatus(cloneReq.RefAppId, int(cloneReq.AppType)) if err != nil { return nil, err } From 56b87b1a09950c3af1274ce4f5e5594472942384 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Tue, 14 Mar 2023 21:30:27 +0530 Subject: [PATCH 104/118] Artifact logger added --- api/router/pubsub/CiEventHandler.go | 24 ++++++++++++++---------- pkg/pipeline/WebhookService.go | 22 ++++++++++++---------- scripts/sql/121_jobs.down.sql | 2 +- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index 4bf0358ef7..4268ae222b 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -64,7 +64,7 @@ type CiCompleteEvent struct { MaterialType string `json:"materialType"` Metrics util.CIMetrics `json:"metrics"` AppName string `json:"appName"` - isArtifactUploaded bool `json:"isArtifactUploaded"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` } func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSubClientServiceImpl, webhookService pipeline.WebhookService, ciEventConfig *CiEventConfig) *CiEventHandlerImpl { @@ -84,7 +84,7 @@ func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSu func (impl *CiEventHandlerImpl) Subscribe() error { callback := func(msg *pubsub.PubSubMsg) { - impl.logger.Debug("ci complete event received") + impl.logger.Debugw("ci complete event received", "data", msg.Data) //defer msg.Ack() ciCompleteEvent := CiCompleteEvent{} err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent) @@ -167,6 +167,8 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* event.TriggeredBy = 1 // system triggered event } + impl.logger.Infow("event data", "event", event) + request := &pipeline.CiArtifactWebhookRequest{ Image: event.DockerImage, ImageDigest: event.Digest, @@ -175,8 +177,9 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* MaterialInfo: rawMaterialInfo, UserId: event.TriggeredBy, WorkflowId: event.WorkflowId, - IsArtifactUploaded: event.isArtifactUploaded, + IsArtifactUploaded: event.IsArtifactUploaded, } + impl.logger.Infow("req data", "req", request) return request, nil } @@ -238,13 +241,14 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequestForWebhook(event CiComplet } request := &pipeline.CiArtifactWebhookRequest{ - Image: event.DockerImage, - ImageDigest: event.Digest, - DataSource: event.DataSource, - PipelineName: event.PipelineName, - MaterialInfo: rawMaterialInfo, - UserId: event.TriggeredBy, - WorkflowId: event.WorkflowId, + Image: event.DockerImage, + ImageDigest: event.Digest, + DataSource: event.DataSource, + PipelineName: event.PipelineName, + MaterialInfo: rawMaterialInfo, + UserId: event.TriggeredBy, + WorkflowId: event.WorkflowId, + IsArtifactUploaded: event.IsArtifactUploaded, } return request, nil } diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index c8c1ae59e1..3e3218bb05 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -169,7 +169,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C if pipeline.ScanEnabled { artifact.Scanned = true } - + impl.logger.Infow("saving artifact data", "artifact", artifact) if err = impl.ciArtifactRepository.Save(artifact); err != nil { impl.logger.Errorw("error in saving material", "err", err) return 0, err @@ -184,15 +184,16 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C var ciArtifactArr []*repository.CiArtifact for _, ci := range childrenCi { ciArtifact := &repository.CiArtifact{ - Image: request.Image, - ImageDigest: request.ImageDigest, - MaterialInfo: string(materialJson), - DataSource: request.DataSource, - PipelineId: ci.Id, - ParentCiArtifact: artifact.Id, - ScanEnabled: ci.ScanEnabled, - Scanned: false, - AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, + Image: request.Image, + ImageDigest: request.ImageDigest, + MaterialInfo: string(materialJson), + DataSource: request.DataSource, + PipelineId: ci.Id, + ParentCiArtifact: artifact.Id, + ScanEnabled: ci.ScanEnabled, + Scanned: false, + IsArtifactUploaded: request.IsArtifactUploaded, + AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if ci.ScanEnabled { ciArtifact.Scanned = true @@ -264,6 +265,7 @@ func (impl WebhookServiceImpl) HandleExternalCiWebhook(externalCiId int, request ExternalCiPipelineId: externalCiId, ScanEnabled: false, Scanned: false, + IsArtifactUploaded: request.IsArtifactUploaded, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if err = impl.ciArtifactRepository.Save(artifact); err != nil { diff --git a/scripts/sql/121_jobs.down.sql b/scripts/sql/121_jobs.down.sql index 9328ae840f..e1d421c4a8 100644 --- a/scripts/sql/121_jobs.down.sql +++ b/scripts/sql/121_jobs.down.sql @@ -20,7 +20,7 @@ DELETE FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WH DELETE FROM git_material_history WHERE git_material_id IN(SELECT id FROM git_material WHERE app_id IN(SELECT id FROM app WHERE app_type = 2)); -DELETE FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); +DELETE FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2) ; DELETE FROM app_label WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); From a5c4ecd64f0d174a7060853046ef530435c36112 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 15 Mar 2023 11:07:42 +0530 Subject: [PATCH 105/118] Additional loggers removed --- api/router/pubsub/CiEventHandler.go | 5 +---- pkg/pipeline/WebhookService.go | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/api/router/pubsub/CiEventHandler.go b/api/router/pubsub/CiEventHandler.go index 4268ae222b..dba228538f 100644 --- a/api/router/pubsub/CiEventHandler.go +++ b/api/router/pubsub/CiEventHandler.go @@ -84,7 +84,7 @@ func NewCiEventHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pubsub.PubSu func (impl *CiEventHandlerImpl) Subscribe() error { callback := func(msg *pubsub.PubSubMsg) { - impl.logger.Debugw("ci complete event received", "data", msg.Data) + impl.logger.Debugw("ci complete event received") //defer msg.Ack() ciCompleteEvent := CiCompleteEvent{} err := json.Unmarshal([]byte(string(msg.Data)), &ciCompleteEvent) @@ -167,8 +167,6 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* event.TriggeredBy = 1 // system triggered event } - impl.logger.Infow("event data", "event", event) - request := &pipeline.CiArtifactWebhookRequest{ Image: event.DockerImage, ImageDigest: event.Digest, @@ -179,7 +177,6 @@ func (impl *CiEventHandlerImpl) BuildCiArtifactRequest(event CiCompleteEvent) (* WorkflowId: event.WorkflowId, IsArtifactUploaded: event.IsArtifactUploaded, } - impl.logger.Infow("req data", "req", request) return request, nil } diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index 3e3218bb05..cd918fc49d 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -169,7 +169,6 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C if pipeline.ScanEnabled { artifact.Scanned = true } - impl.logger.Infow("saving artifact data", "artifact", artifact) if err = impl.ciArtifactRepository.Save(artifact); err != nil { impl.logger.Errorw("error in saving material", "err", err) return 0, err From 74a1484dee915ec849cc8f733753d7a5febf45e9 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 15 Mar 2023 17:43:46 +0530 Subject: [PATCH 106/118] Logger No tasks configured for this job added --- pkg/pipeline/CiService.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 51d4d4b5a1..6d69bf0dcd 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/app" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -143,6 +144,9 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { return 0, err } + if len(workflowRequest.BeforeDockerBuildScripts) == 0 && pipeline.App != nil && pipeline.App.AppType == helper.Job { + return 0, errors.New("No tasks configured for this job") + } err = impl.updateCiWorkflow(workflowRequest, savedCiWf) appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) From 61e59a70ca7b4e8e870f71d8cb965ac3e421e137 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 15 Mar 2023 20:27:05 +0530 Subject: [PATCH 107/118] Removed No tasks configured for this job added --- pkg/pipeline/CiService.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 6d69bf0dcd..51d4d4b5a1 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/app" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -144,9 +143,6 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger Trigger) (int, error) { return 0, err } - if len(workflowRequest.BeforeDockerBuildScripts) == 0 && pipeline.App != nil && pipeline.App.AppType == helper.Job { - return 0, errors.New("No tasks configured for this job") - } err = impl.updateCiWorkflow(workflowRequest, savedCiWf) appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) From c5275908da82fda69ba2b9b53fbce9a1d5c7acf8 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Mar 2023 13:36:30 +0530 Subject: [PATCH 108/118] Added a validation for AppType --- pkg/bean/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index f9951ca62c..272c8f3e03 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -47,7 +47,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` - AppType helper.AppType `json:"appType"` + AppType helper.AppType `json:"appType" validate:"gt=-1,lt=3"` } type CreateMaterialDTO struct { From 4c32037236cf9acb91dc6ac6e239a6e1e1fbded6 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 16 Mar 2023 13:40:14 +0530 Subject: [PATCH 109/118] Added comments for validation for AppType --- pkg/bean/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 272c8f3e03..a5ed70be5c 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -47,7 +47,7 @@ type CreateAppDTO struct { TemplateId int `json:"templateId"` AppLabels []*Label `json:"labels,omitempty" validate:"dive"` Description string `json:"description"` - AppType helper.AppType `json:"appType" validate:"gt=-1,lt=3"` + AppType helper.AppType `json:"appType" validate:"gt=-1,lt=3"` //TODO: Change Validation if new AppType is introduced } type CreateMaterialDTO struct { From e07997e3795b354e977be5f7fe541fa5f3fd2b34 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Mar 2023 15:27:47 +0530 Subject: [PATCH 110/118] Added a default value to IsArtifactUploaded --- internal/sql/repository/CiArtifactRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index d58416fb38..480a976983 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -43,7 +43,7 @@ type CiArtifact struct { ScanEnabled bool `sql:"scan_enabled,notnull"` Scanned bool `sql:"scanned,notnull"` ExternalCiPipelineId int `sql:"external_ci_pipeline_id"` - IsArtifactUploaded bool `sql:"is_artifact_uploaded"` + IsArtifactUploaded bool `sql:"is_artifact_uploaded,default:false"` DeployedTime time.Time `sql:"-"` Deployed bool `sql:"-"` Latest bool `sql:"-"` From a41d0ae63a52664880fe6c483e7437ac7a636a23 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Mar 2023 15:39:22 +0530 Subject: [PATCH 111/118] Removed default value to IsArtifactUploaded --- internal/sql/repository/CiArtifactRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/CiArtifactRepository.go b/internal/sql/repository/CiArtifactRepository.go index 480a976983..d58416fb38 100644 --- a/internal/sql/repository/CiArtifactRepository.go +++ b/internal/sql/repository/CiArtifactRepository.go @@ -43,7 +43,7 @@ type CiArtifact struct { ScanEnabled bool `sql:"scan_enabled,notnull"` Scanned bool `sql:"scanned,notnull"` ExternalCiPipelineId int `sql:"external_ci_pipeline_id"` - IsArtifactUploaded bool `sql:"is_artifact_uploaded,default:false"` + IsArtifactUploaded bool `sql:"is_artifact_uploaded"` DeployedTime time.Time `sql:"-"` Deployed bool `sql:"-"` Latest bool `sql:"-"` From 46066317766ac3a421b8a76c96c96ee15eb36c05 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Mar 2023 16:25:40 +0530 Subject: [PATCH 112/118] Added false to isArtifactUploaded --- pkg/pipeline/WebhookService.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index cd918fc49d..2f1249b196 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -154,6 +154,10 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } materialJson = dst.Bytes() + var IsArtifactUploadedDB bool + if request.IsArtifactUploaded != true { + IsArtifactUploadedDB = false + } artifact := &repository.CiArtifact{ Image: request.Image, ImageDigest: request.ImageDigest, @@ -163,7 +167,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C WorkflowId: request.WorkflowId, ScanEnabled: pipeline.ScanEnabled, Scanned: false, - IsArtifactUploaded: request.IsArtifactUploaded, + IsArtifactUploaded: IsArtifactUploadedDB, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if pipeline.ScanEnabled { From 14fdad71b2f4c88cd97213760d66cfbdd1806017 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Mar 2023 16:36:43 +0530 Subject: [PATCH 113/118] Removed false to isArtifactUploaded --- pkg/pipeline/WebhookService.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index 2f1249b196..cd918fc49d 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -154,10 +154,6 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C return 0, err } materialJson = dst.Bytes() - var IsArtifactUploadedDB bool - if request.IsArtifactUploaded != true { - IsArtifactUploadedDB = false - } artifact := &repository.CiArtifact{ Image: request.Image, ImageDigest: request.ImageDigest, @@ -167,7 +163,7 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C WorkflowId: request.WorkflowId, ScanEnabled: pipeline.ScanEnabled, Scanned: false, - IsArtifactUploaded: IsArtifactUploadedDB, + IsArtifactUploaded: request.IsArtifactUploaded, AuditLog: sql.AuditLog{CreatedBy: request.UserId, UpdatedBy: request.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now()}, } if pipeline.ScanEnabled { From 39a1d028a1cc229cd2c38a7e8b40aa5fb7224c81 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Mar 2023 18:00:08 +0530 Subject: [PATCH 114/118] Optimized sql up and down. --- scripts/sql/121_jobs.down.sql | 46 ++++++++--------------------------- scripts/sql/121_jobs.up.sql | 12 ++++----- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/scripts/sql/121_jobs.down.sql b/scripts/sql/121_jobs.down.sql index e1d421c4a8..93804569c7 100644 --- a/scripts/sql/121_jobs.down.sql +++ b/scripts/sql/121_jobs.down.sql @@ -1,45 +1,19 @@ -DELETE FROM ci_pipeline_material WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); +UPDATE ci_pipeline_material SET active = false FROM git_material JOIN app ON app.id = git_material.app_id WHERE app.app_type = 2 AND ci_pipeline_material.git_material_id = git_material.id; -DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2))); +UPDATE ci_template_history SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM ci_pipeline_history WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); +UPDATE ci_pipeline SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM ci_artifact WHERE ci_workflow_id IN (SELECT id FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2))))); +UPDATE ci_template_history SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); +UPDATE ci_template SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM pipeline_stage WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); +UPDATE git_material_history SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2))); +UPDATE git_material SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM ci_workflow WHERE ci_pipeline_id IN (SELECT id FROM ci_pipeline WHERE ci_template_id IN (SELECT id FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)))); +UPDATE app_workflow_mapping SET active = false FROM app_workflow JOIN app ON app_workflow.app_id = app.id AND app.app_type = 2WHERE app_workflow_mapping.app_workflow_id = app_workflow.id; -DELETE FROM ci_template_history WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); +UPDATE app_workflow SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -DELETE FROM ci_template WHERE git_material_id IN (SELECT id FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); - -DELETE FROM git_material_history WHERE git_material_id IN(SELECT id FROM git_material WHERE app_id IN(SELECT id FROM app WHERE app_type = 2)); - -DELETE FROM git_material WHERE app_id IN (SELECT id FROM app WHERE app_type = 2) ; - -DELETE FROM app_label WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); - -DELETE FROM app_workflow_mapping WHERE app_workflow_id IN(SELECT id FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_type = 2)); - -DELETE FROM app_workflow WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); - -DELETE FROM app WHERE app_type = 2; - -ALTER TABLE app ALTER COLUMN app_type DROP DEFAULT; - -ALTER TABLE app ALTER app_type TYPE boolean USING CASE WHEN app_type=1 THEN true ELSE false end; - -ALTER TABLE app ALTER COLUMN app_type SET DEFAULT FALSE; - -ALTER TABLE app RENAME COLUMN app_type TO app_store; - -ALTER TABLE app DROP COLUMN display_name; - -ALTER TABLE app DROP COLUMN description; - -ALTER TABLE ci_artifact DROP COLUMN is_artifact_uploaded; \ No newline at end of file +UPDATE app SET active = false WHERE app_type = 2; \ No newline at end of file diff --git a/scripts/sql/121_jobs.up.sql b/scripts/sql/121_jobs.up.sql index cc09d0bba5..32a68ee7eb 100644 --- a/scripts/sql/121_jobs.up.sql +++ b/scripts/sql/121_jobs.up.sql @@ -1,10 +1,8 @@ -ALTER TABLE app ALTER COLUMN app_store DROP DEFAULT; -ALTER TABLE app ALTER app_store TYPE integer USING CASE WHEN app_store=true THEN 1 ELSE 0 end; -ALTER TABLE app RENAME COLUMN app_store TO app_type; -ALTER TABLE app ALTER COLUMN app_store SET DEFAULT 0; -ALTER TABLE app ADD COLUMN display_name varchar(250); -ALTER TABLE app ADD COLUMN description text; -ALTER TABLE ci_artifact ADD COLUMN is_artifact_uploaded BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE app ADD COLUMN IF NOT EXISTS app_type integer not null DEFAULT 0; +UPDATE app SET app_type = CASE WHEN app_store = false THEN 0 WHEN app_store = true THEN 1 ELSE app_type END; +ALTER TABLE app ADD COLUMN IF NOT EXISTS description text; +ALTER TABLE app ADD COLUMN IF NOT EXISTS display_name varchar(250); +ALTER TABLE ci_artifact ADD COLUMN IF NOT EXISTS is_artifact_uploaded BOOLEAN DEFAULT FALSE; UPDATE ci_artifact SET is_artifact_uploaded = true; From f9da98e64d8e431ea837f26c719a70a004ec47b6 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 17 Mar 2023 18:05:29 +0530 Subject: [PATCH 115/118] changed the sql number --- scripts/sql/{121_jobs.down.sql => 122_jobs.down.sql} | 0 scripts/sql/{121_jobs.up.sql => 122_jobs.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename scripts/sql/{121_jobs.down.sql => 122_jobs.down.sql} (100%) rename scripts/sql/{121_jobs.up.sql => 122_jobs.up.sql} (100%) diff --git a/scripts/sql/121_jobs.down.sql b/scripts/sql/122_jobs.down.sql similarity index 100% rename from scripts/sql/121_jobs.down.sql rename to scripts/sql/122_jobs.down.sql diff --git a/scripts/sql/121_jobs.up.sql b/scripts/sql/122_jobs.up.sql similarity index 100% rename from scripts/sql/121_jobs.up.sql rename to scripts/sql/122_jobs.up.sql From aaedddf4c6fcc92a496dc7d2b150ffd6dda89c85 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 20 Mar 2023 10:25:32 +0530 Subject: [PATCH 116/118] Optimized the sql query. --- scripts/sql/122_jobs.down.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/sql/122_jobs.down.sql b/scripts/sql/122_jobs.down.sql index 93804569c7..ecc0ce1c19 100644 --- a/scripts/sql/122_jobs.down.sql +++ b/scripts/sql/122_jobs.down.sql @@ -16,4 +16,7 @@ UPDATE app_workflow_mapping SET active = false FROM app_workflow JOIN app ON app UPDATE app_workflow SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -UPDATE app SET active = false WHERE app_type = 2; \ No newline at end of file +UPDATE app SET active = false WHERE app_type = 2; + + + From ed3bceb20416d9fd8d50e36d0282e7f2b6bf51bd Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 20 Mar 2023 11:17:23 +0530 Subject: [PATCH 117/118] Removed the history tables from sql down. --- scripts/sql/122_jobs.down.sql | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/sql/122_jobs.down.sql b/scripts/sql/122_jobs.down.sql index ecc0ce1c19..fbb9773cb7 100644 --- a/scripts/sql/122_jobs.down.sql +++ b/scripts/sql/122_jobs.down.sql @@ -1,15 +1,9 @@ UPDATE ci_pipeline_material SET active = false FROM git_material JOIN app ON app.id = git_material.app_id WHERE app.app_type = 2 AND ci_pipeline_material.git_material_id = git_material.id; -UPDATE ci_template_history SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); - UPDATE ci_pipeline SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -UPDATE ci_template_history SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); - UPDATE ci_template SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -UPDATE git_material_history SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); - UPDATE git_material SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); UPDATE app_workflow_mapping SET active = false FROM app_workflow JOIN app ON app_workflow.app_id = app.id AND app.app_type = 2WHERE app_workflow_mapping.app_workflow_id = app_workflow.id; From 0a9a8b93db71449c7f8218e4ff4c6bb918ec2f61 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 20 Mar 2023 12:42:11 +0530 Subject: [PATCH 118/118] Removed the history tables from sql down. --- scripts/sql/122_jobs.down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sql/122_jobs.down.sql b/scripts/sql/122_jobs.down.sql index fbb9773cb7..2b4a77e8e0 100644 --- a/scripts/sql/122_jobs.down.sql +++ b/scripts/sql/122_jobs.down.sql @@ -6,7 +6,7 @@ UPDATE ci_template SET active = false WHERE app_id IN (SELECT id FROM app WHERE UPDATE git_material SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2); -UPDATE app_workflow_mapping SET active = false FROM app_workflow JOIN app ON app_workflow.app_id = app.id AND app.app_type = 2WHERE app_workflow_mapping.app_workflow_id = app_workflow.id; +UPDATE app_workflow_mapping SET active = false FROM app_workflow JOIN app ON app_workflow.app_id = app.id WHERE app.app_type = 2 AND app_workflow_mapping.app_workflow_id = app_workflow.id; UPDATE app_workflow SET active = false WHERE app_id IN (SELECT id FROM app WHERE app_type = 2);