diff --git a/Wire.go b/Wire.go index dce534c9ff..70d0fefb6e 100644 --- a/Wire.go +++ b/Wire.go @@ -41,6 +41,7 @@ import ( "github.com/devtron-labs/devtron/api/deployment" "github.com/devtron-labs/devtron/api/externalLink" client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/infraConfig" "github.com/devtron-labs/devtron/api/k8s" "github.com/devtron-labs/devtron/api/module" "github.com/devtron-labs/devtron/api/restHandler" @@ -125,6 +126,8 @@ import ( "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + infraConfigService "github.com/devtron-labs/devtron/pkg/infraConfig" + "github.com/devtron-labs/devtron/pkg/infraConfig/units" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" repository7 "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs/repository" "github.com/devtron-labs/devtron/pkg/notifier" @@ -132,6 +135,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" history3 "github.com/devtron-labs/devtron/pkg/pipeline/history" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -181,20 +185,20 @@ func InitializeApp() (*App, error) { deployment2.DeploymentWireSet, // -------wireset end ---------- - //------- + // ------- gitSensor.GetConfig, gitSensor.NewGitSensorClient, wire.Bind(new(gitSensor.Client), new(*gitSensor.ClientImpl)), - //------- + // ------- helper.NewAppListingRepositoryQueryBuilder, - //sql.GetConfig, + // sql.GetConfig, eClient.GetEventClientConfig, util2.GetGlobalEnvVariables, - //sql.NewDbConnection, - //app.GetACDAuthConfig, + // sql.NewDbConnection, + // app.GetACDAuthConfig, util3.GetACDAuthConfig, connection.SettingsManager, - //auth.GetConfig, + // auth.GetConfig, connection.GetConfig, wire.Bind(new(session2.ServiceClient), new(*middleware.LoginService)), @@ -203,13 +207,13 @@ func InitializeApp() (*App, error) { trigger2.NewPipelineTriggerRouter, wire.Bind(new(trigger2.PipelineTriggerRouter), new(*trigger2.PipelineTriggerRouterImpl)), - //---- pprof start ---- + // ---- pprof start ---- restHandler.NewPProfRestHandler, wire.Bind(new(restHandler.PProfRestHandler), new(*restHandler.PProfRestHandlerImpl)), router.NewPProfRouter, wire.Bind(new(router.PProfRouter), new(*router.PProfRouterImpl)), - //---- pprof end ---- + // ---- pprof end ---- trigger.NewPipelineRestHandler, wire.Bind(new(trigger.PipelineTriggerRestHandler), new(*trigger.PipelineTriggerRestHandlerImpl)), @@ -238,6 +242,20 @@ func InitializeApp() (*App, error) { wire.Bind(new(dashboardEvent.DashboardTelemetryRouter), new(*dashboardEvent.DashboardTelemetryRouterImpl)), + infraConfigService.NewInfraProfileRepositoryImpl, + wire.Bind(new(infraConfigService.InfraConfigRepository), new(*infraConfigService.InfraConfigRepositoryImpl)), + + units.NewUnits, + infraConfigService.NewInfraConfigServiceImpl, + wire.Bind(new(infraConfigService.InfraConfigService), new(*infraConfigService.InfraConfigServiceImpl)), + infraProviders.NewInfraProviderImpl, + wire.Bind(new(infraProviders.InfraProvider), new(*infraProviders.InfraProviderImpl)), + infraConfig.NewInfraConfigRestHandlerImpl, + wire.Bind(new(infraConfig.InfraConfigRestHandler), new(*infraConfig.InfraConfigRestHandlerImpl)), + + infraConfig.NewInfraProfileRouterImpl, + wire.Bind(new(infraConfig.InfraConfigRouter), new(*infraConfig.InfraConfigRouterImpl)), + router.NewMuxRouter, app2.NewAppRepositoryImpl, @@ -293,7 +311,7 @@ func InitializeApp() (*App, error) { util.NewChartTemplateServiceImpl, wire.Bind(new(util.ChartTemplateService), new(*util.ChartTemplateServiceImpl)), - //scoped variables start + // scoped variables start variables.NewScopedVariableServiceImpl, wire.Bind(new(variables.ScopedVariableService), new(*variables.ScopedVariableServiceImpl)), @@ -315,7 +333,7 @@ func InitializeApp() (*App, error) { variables.NewScopedVariableCMCSManagerImpl, wire.Bind(new(variables.ScopedVariableCMCSManager), new(*variables.ScopedVariableCMCSManagerImpl)), - //end + // end chart.NewChartServiceImpl, wire.Bind(new(chart.ChartService), new(*chart.ChartServiceImpl)), @@ -395,10 +413,10 @@ func InitializeApp() (*App, error) { //app.GetConfig, pipeline.GetEcrConfig, - //otel.NewOtelTracingServiceImpl, - //wire.Bind(new(otel.OtelTracingService), new(*otel.OtelTracingServiceImpl)), + // otel.NewOtelTracingServiceImpl, + // wire.Bind(new(otel.OtelTracingService), new(*otel.OtelTracingServiceImpl)), NewApp, - //session.NewK8sClient, + // session.NewK8sClient, repository8.NewImageTaggingRepositoryImpl, wire.Bind(new(repository8.ImageTaggingRepository), new(*repository8.ImageTaggingRepositoryImpl)), pipeline.NewImageTaggingServiceImpl, @@ -581,7 +599,7 @@ func InitializeApp() (*App, error) { restHandler.NewPubSubClientRestHandlerImpl, wire.Bind(new(restHandler.PubSubClientRestHandler), new(*restHandler.PubSubClientRestHandlerImpl)), - //Batch actions + // Batch actions batch.NewWorkflowActionImpl, wire.Bind(new(batch.WorkflowAction), new(*batch.WorkflowActionImpl)), batch.NewDeploymentActionImpl, @@ -786,9 +804,9 @@ func InitializeApp() (*App, error) { history3.NewDeployedConfigurationHistoryServiceImpl, wire.Bind(new(history3.DeployedConfigurationHistoryService), new(*history3.DeployedConfigurationHistoryServiceImpl)), - //history ends + // history ends - //plugin starts + // plugin starts repository6.NewGlobalPluginRepository, wire.Bind(new(repository6.GlobalPluginRepository), new(*repository6.GlobalPluginRepositoryImpl)), @@ -806,7 +824,7 @@ func InitializeApp() (*App, error) { pipeline.NewPipelineStageService, wire.Bind(new(pipeline.PipelineStageService), new(*pipeline.PipelineStageServiceImpl)), - //plugin ends + // plugin ends connection.NewArgoCDConnectionManagerImpl, wire.Bind(new(connection.ArgoCDConnectionManager), new(*connection.ArgoCDConnectionManagerImpl)), @@ -818,12 +836,12 @@ func InitializeApp() (*App, error) { cron.NewCdApplicationStatusUpdateHandlerImpl, wire.Bind(new(cron.CdApplicationStatusUpdateHandler), new(*cron.CdApplicationStatusUpdateHandlerImpl)), - //app_status + // app_status appStatusRepo.NewAppStatusRepositoryImpl, wire.Bind(new(appStatusRepo.AppStatusRepository), new(*appStatusRepo.AppStatusRepositoryImpl)), appStatus.NewAppStatusServiceImpl, wire.Bind(new(appStatus.AppStatusService), new(*appStatus.AppStatusServiceImpl)), - //app_status ends + // app_status ends cron.GetCiWorkflowStatusUpdateConfig, cron.NewCiStatusUpdateCronImpl, @@ -868,8 +886,8 @@ func InitializeApp() (*App, error) { repository.NewGlobalCMCSRepositoryImpl, wire.Bind(new(repository.GlobalCMCSRepository), new(*repository.GlobalCMCSRepositoryImpl)), - //chartRepoRepository.NewGlobalStrategyMetadataRepositoryImpl, - //wire.Bind(new(chartRepoRepository.GlobalStrategyMetadataRepository), new(*chartRepoRepository.GlobalStrategyMetadataRepositoryImpl)), + // chartRepoRepository.NewGlobalStrategyMetadataRepositoryImpl, + // wire.Bind(new(chartRepoRepository.GlobalStrategyMetadataRepository), new(*chartRepoRepository.GlobalStrategyMetadataRepositoryImpl)), chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl, wire.Bind(new(chartRepoRepository.GlobalStrategyMetadataChartRefMappingRepository), new(*chartRepoRepository.GlobalStrategyMetadataChartRefMappingRepositoryImpl)), diff --git a/api/infraConfig/restHandler.go b/api/infraConfig/restHandler.go new file mode 100644 index 0000000000..7fe69b741c --- /dev/null +++ b/api/infraConfig/restHandler.go @@ -0,0 +1,120 @@ +package infraConfig + +import ( + "encoding/json" + "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/infraConfig" + "github.com/devtron-labs/devtron/util/rbac" + "github.com/gorilla/mux" + "github.com/pkg/errors" + "go.uber.org/zap" + "gopkg.in/go-playground/validator.v9" + "net/http" + "strings" +) + +type InfraConfigRestHandler interface { + UpdateInfraProfile(w http.ResponseWriter, r *http.Request) + GetProfile(w http.ResponseWriter, r *http.Request) +} +type InfraConfigRestHandlerImpl struct { + logger *zap.SugaredLogger + infraProfileService infraConfig.InfraConfigService + userService user.UserService + enforcer casbin.Enforcer + enforcerUtil rbac.EnforcerUtil + validator *validator.Validate +} + +func NewInfraConfigRestHandlerImpl(logger *zap.SugaredLogger, infraProfileService infraConfig.InfraConfigService, userService user.UserService, enforcer casbin.Enforcer, enforcerUtil rbac.EnforcerUtil, validator *validator.Validate) *InfraConfigRestHandlerImpl { + return &InfraConfigRestHandlerImpl{ + logger: logger, + infraProfileService: infraProfileService, + userService: userService, + enforcer: enforcer, + enforcerUtil: enforcerUtil, + validator: validator, + } +} + +func (handler *InfraConfigRestHandlerImpl) UpdateInfraProfile(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + token := r.Header.Get("token") + if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*"); !ok { + common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) + return + } + + vars := mux.Vars(r) + profileName := strings.ToLower(vars["name"]) + if profileName == "" { + common.WriteJsonResp(w, errors.New(infraConfig.InvalidProfileName), nil, http.StatusBadRequest) + return + } + payload := &infraConfig.ProfileBean{} + decoder := json.NewDecoder(r.Body) + err = decoder.Decode(payload) + if err != nil { + handler.logger.Errorw("error in decoding the request payload", "err", err, "requestBody", r.Body) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + payload.Name = strings.ToLower(payload.Name) + err = handler.validator.Struct(payload) + if err != nil { + err = errors.Wrap(err, infraConfig.PayloadValidationError) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + } + if profileName == "" || (profileName == infraConfig.DEFAULT_PROFILE_NAME && payload.Name != infraConfig.DEFAULT_PROFILE_NAME) { + common.WriteJsonResp(w, errors.New(infraConfig.InvalidProfileName), nil, http.StatusBadRequest) + } + err = handler.infraProfileService.UpdateProfile(userId, profileName, payload) + if err != nil { + handler.logger.Errorw("error in updating profile and configurations", "profileName", profileName, "payLoad", payload, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + common.WriteJsonResp(w, nil, nil, http.StatusOK) +} + +func (handler *InfraConfigRestHandlerImpl) GetProfile(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + token := r.Header.Get("token") + if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) + return + } + + vars := mux.Vars(r) + profileName := strings.ToLower(vars["name"]) + if profileName == "" { + common.WriteJsonResp(w, errors.New(infraConfig.InvalidProfileName), nil, http.StatusBadRequest) + return + } + + var profile *infraConfig.ProfileBean + defaultProfile, err := handler.infraProfileService.GetProfileByName(profileName) + if err != nil { + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + if profileName == infraConfig.DEFAULT_PROFILE_NAME { + profile = defaultProfile + } + resp := infraConfig.ProfileResponse{ + Profile: *profile, + } + resp.ConfigurationUnits = handler.infraProfileService.GetConfigurationUnits() + resp.DefaultConfigurations = defaultProfile.Configurations + common.WriteJsonResp(w, nil, resp, http.StatusOK) +} diff --git a/api/infraConfig/router.go b/api/infraConfig/router.go new file mode 100644 index 0000000000..e79014cff7 --- /dev/null +++ b/api/infraConfig/router.go @@ -0,0 +1,27 @@ +package infraConfig + +import "github.com/gorilla/mux" + +type InfraConfigRouter interface { + InitInfraConfigRouter(configRouter *mux.Router) +} + +type InfraConfigRouterImpl struct { + infraConfigRestHandler InfraConfigRestHandler +} + +func NewInfraProfileRouterImpl(infraConfigRestHandler InfraConfigRestHandler) *InfraConfigRouterImpl { + return &InfraConfigRouterImpl{ + infraConfigRestHandler: infraConfigRestHandler, + } +} + +func (impl *InfraConfigRouterImpl) InitInfraConfigRouter(configRouter *mux.Router) { + configRouter.Path("/profile/{name}"). + HandlerFunc(impl.infraConfigRestHandler.GetProfile). + Methods("GET") + + configRouter.Path("/profile/{name}"). + HandlerFunc(impl.infraConfigRestHandler.UpdateInfraProfile). + Methods("PUT") +} diff --git a/api/router/router.go b/api/router/router.go index 57440bf749..b8fe28140a 100644 --- a/api/router/router.go +++ b/api/router/router.go @@ -32,6 +32,7 @@ import ( "github.com/devtron-labs/devtron/api/deployment" "github.com/devtron-labs/devtron/api/externalLink" client "github.com/devtron-labs/devtron/api/helm-app" + "github.com/devtron-labs/devtron/api/infraConfig" "github.com/devtron-labs/devtron/api/k8s/application" "github.com/devtron-labs/devtron/api/k8s/capacity" "github.com/devtron-labs/devtron/api/module" @@ -118,6 +119,7 @@ type MuxRouter struct { rbacRoleRouter user.RbacRoleRouter scopedVariableRouter ScopedVariableRouter ciTriggerCron cron.CiTriggerCron + infraConfigRouter infraConfig.InfraConfigRouter } func NewMuxRouter(logger *zap.SugaredLogger, @@ -148,7 +150,8 @@ func NewMuxRouter(logger *zap.SugaredLogger, rbacRoleRouter user.RbacRoleRouter, scopedVariableRouter ScopedVariableRouter, ciTriggerCron cron.CiTriggerCron, - proxyRouter proxy.ProxyRouter) *MuxRouter { + proxyRouter proxy.ProxyRouter, + infraConfigRouter infraConfig.InfraConfigRouter) *MuxRouter { r := &MuxRouter{ Router: mux.NewRouter(), EnvironmentClusterMappingsRouter: EnvironmentClusterMappingsRouter, @@ -213,6 +216,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, rbacRoleRouter: rbacRoleRouter, scopedVariableRouter: scopedVariableRouter, ciTriggerCron: ciTriggerCron, + infraConfigRouter: infraConfigRouter, } return r } @@ -222,8 +226,8 @@ func (r MuxRouter) Init() { r.Router.StrictSlash(true) r.Router.Handle("/metrics", promhttp.Handler()) - //prometheus.MustRegister(pipeline.CiTriggerCounter) - //prometheus.MustRegister(app.CdTriggerCounter) + // prometheus.MustRegister(pipeline.CiTriggerCounter) + // prometheus.MustRegister(app.CdTriggerCounter) r.Router.Path("/health").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.Header().Set("Content-Type", "application/json") writer.WriteHeader(200) @@ -375,10 +379,10 @@ func (r MuxRouter) Init() { r.dashboardTelemetryRouter.Init(dashboardTelemetryRouter) // dashboard event router ends - //GitOps,Acd + HelmCLi both apps deployment related api's + // GitOps,Acd + HelmCLi both apps deployment related api's applicationSubRouter := r.Router.PathPrefix("/orchestrator/application").Subrouter() r.commonDeploymentRouter.Init(applicationSubRouter) - //this router must placed after commonDeploymentRouter + // this router must placed after commonDeploymentRouter r.helmAppRouter.InitAppListRouter(applicationSubRouter) externalLinkRouter := r.Router.PathPrefix("/orchestrator/external-links").Subrouter() @@ -411,4 +415,7 @@ func (r MuxRouter) Init() { rbacRoleRouter := r.Router.PathPrefix("/orchestrator/rbac/role").Subrouter() r.rbacRoleRouter.InitRbacRoleRouter(rbacRoleRouter) + + infraConfigRouter := r.Router.PathPrefix("/orchestrator/infra-config").Subrouter() + r.infraConfigRouter.InitInfraConfigRouter(infraConfigRouter) } diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index 538eae2654..a4bdd269f6 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -30,7 +30,7 @@ import ( type App struct { tableName struct{} `sql:"app" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` - AppName string `sql:"app_name,notnull"` //same as app name + AppName string `sql:"app_name,notnull"` // same as app name DisplayName string `sql:"display_name"` Active bool `sql:"active, notnull"` TeamId int `sql:"team_id"` @@ -77,6 +77,7 @@ type AppRepository interface { FindAllActiveAppsWithTeamByAppNameMatch(appNameMatch string, appType helper.AppType) ([]*App, error) FindAppAndProjectByIdsIn(ids []int) ([]*App, error) FetchAppIdsByDisplayNamesForJobs(names []string) (map[int]string, []int, error) + GetActiveCiCdAppsCount() (int, error) } const DevtronApp = "DevtronApp" @@ -464,3 +465,10 @@ func (repo AppRepositoryImpl) FetchAppIdsByDisplayNamesForJobs(names []string) ( } return appResp, jobIds, err } + +func (repo AppRepositoryImpl) GetActiveCiCdAppsCount() (int, error) { + return repo.dbConnection.Model(&App{}). + Where("active=?", true). + Where("app_type=?", helper.CustomApp). + Count() +} diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index f100d4023e..652e9b23ca 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -74,14 +74,14 @@ import ( type AppServiceConfig struct { CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *"` CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *"` - CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20"` //in minutes - PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10"` //in minutes - GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12"` //in hours - HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120"` //in seconds + CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20"` // in minutes + PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10"` // in minutes + GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12"` // in hours + HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120"` // in seconds ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` EnableAsyncInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false"` DevtronChartInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6"` - ArgocdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3"` //in minutes + ArgocdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3"` // in minutes } func GetAppServiceConfig() (*AppServiceConfig, error) { @@ -127,28 +127,29 @@ type AppServiceImpl struct { } type AppService interface { - //HandleCDTriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) - //TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, builtChartPath string, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) + // HandleCDTriggerRelease(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) + // TriggerRelease(overrideRequest *bean.ValuesOverrideRequest, valuesOverrideResponse *ValuesOverrideResponse, builtChartPath string, ctx context.Context, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifest []byte, err error) UpdateReleaseStatus(request *bean.ReleaseStatusUpdateRequest) (bool, error) UpdateDeploymentStatusAndCheckIsSucceeded(app *v1alpha1.Application, statusTime time.Time, isAppStore bool) (bool, *chartConfig.PipelineOverride, error) - //TriggerCD(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error + // TriggerCD(artifact *repository.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error GetConfigMapAndSecretJson(appId int, envId int, pipelineId int) ([]byte, error) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Application, cdWfrId int, updateTimedOutStatus bool) error GetCmSecretNew(appId int, envId int, isJob bool, scope resourceQualifiers.Scope) (*bean.ConfigMapJson, *bean.ConfigSecretJson, error) - //MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error + // MarkImageScanDeployed(appId int, envId int, imageDigest string, clusterId int, isScanEnabled bool) error UpdateDeploymentStatusForGitOpsPipelines(app *v1alpha1.Application, statusTime time.Time, isAppStore bool) (bool, bool, *chartConfig.PipelineOverride, error) WriteCDSuccessEvent(appId int, envId int, override *chartConfig.PipelineOverride) GetGitOpsRepoPrefix() string - //GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*ValuesOverrideResponse, error) - //GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) - //GetAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) - //GetDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) + // GetValuesOverrideForTrigger(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*ValuesOverrideResponse, error) + // GetEnvOverrideByTriggerType(overrideRequest *bean.ValuesOverrideRequest, triggeredAt time.Time, ctx context.Context) (*chartConfig.EnvConfigOverride, error) + // GetAppMetricsByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (bool, error) + // GetDeploymentStrategyByTriggerType(overrideRequest *bean.ValuesOverrideRequest, ctx context.Context) (*chartConfig.PipelineStrategy, error) CreateGitopsRepo(app *app.App, userId int32) (gitopsRepoName string, chartGitAttr *commonBean.ChartGitAttribute, err error) GetDeployedManifestByPipelineIdAndCDWorkflowId(appId int, envId int, cdWorkflowId int, ctx context.Context) ([]byte, error) - //SetPipelineFieldsInOverrideRequest(overrideRequest *bean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) + // SetPipelineFieldsInOverrideRequest(overrideRequest *bean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) BuildChartAndGetPath(appName string, envOverride *chartConfig.EnvConfigOverride, ctx context.Context) (string, error) IsDevtronAsyncInstallModeEnabled(deploymentAppType string) bool + GetActiveCiCdAppsCount() (int, error) } func NewAppService( @@ -327,13 +328,13 @@ func (impl *AppServiceImpl) UpdateDeploymentStatusForGitOpsPipelines(app *v1alph reconciledAt = app.Status.ReconciledAt } var kubectlSyncedTimeline *pipelineConfig.PipelineStatusTimeline - //updating cd pipeline status timeline + // updating cd pipeline status timeline isTimelineUpdated, isTimelineTimedOut, kubectlSyncedTimeline, err = impl.UpdatePipelineStatusTimelineForApplicationChanges(app, cdWfr.Id, statusTime, cdWfr.StartedOn, timeoutDuration, latestTimelineBeforeThisEvent, reconciledAt, false) if err != nil { impl.logger.Errorw("error in updating pipeline status timeline", "err", err) } if isTimelineTimedOut { - //not checking further and directly updating timedOutStatus + // not checking further and directly updating timedOutStatus err := impl.UpdateCdWorkflowRunnerByACDObject(app, cdWfr.Id, true) if err != nil { impl.logger.Errorw("error on update cd workflow runner", "CdWorkflowId", pipelineOverride.CdWorkflowId, "status", pipelineConfig.WorkflowTimedOut, "err", err) @@ -388,19 +389,19 @@ func (impl *AppServiceImpl) UpdateDeploymentStatusForGitOpsPipelines(app *v1alph if err != nil { impl.logger.Errorw("error occurred while updating app status in app_status table", "error", err, "appId", appId, "envId", envId) } - //reconcile time is how often your applications will sync from Argo CD to the Git repository + // reconcile time is how often your applications will sync from Argo CD to the Git repository reconciledAt := &metav1.Time{} if app != nil { reconciledAt = app.Status.ReconciledAt } var kubectlSyncedTimeline *pipelineConfig.PipelineStatusTimeline - //updating versionHistory pipeline status timeline + // updating versionHistory pipeline status timeline isTimelineUpdated, isTimelineTimedOut, kubectlSyncedTimeline, err = impl.UpdatePipelineStatusTimelineForApplicationChanges(app, installedAppVersionHistory.Id, statusTime, installedAppVersionHistory.StartedOn, timeoutDuration, latestTimelineBeforeThisEvent, reconciledAt, true) if err != nil { impl.logger.Errorw("error in updating pipeline status timeline", "err", err) } if isTimelineTimedOut { - //not checking further and directly updating timedOutStatus + // not checking further and directly updating timedOutStatus err := impl.UpdateInstalledAppVersionHistoryByACDObject(app, installedAppVersionHistory.Id, true) if err != nil { impl.logger.Errorw("error on update installedAppVersionHistory", "installedAppVersionHistory", installedAppVersionHistory.Id, "status", pipelineConfig.WorkflowTimedOut, "err", err) @@ -432,7 +433,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA installedAppVersionHistory := &repository4.InstalledAppVersionHistory{} installedAppId := 0 gitOpsAppNameAndInstalledAppMapping := make(map[string]*int) - //checking if the gitOpsAppName is present in installed_apps table, if yes the find installed_app_version_history else return + // checking if the gitOpsAppName is present in installed_apps table, if yes the find installed_app_version_history else return gitOpsAppNameAndInstalledAppId, err := impl.installedAppRepository.GetAllGitOpsAppNameAndInstalledAppMapping() if err != nil { impl.logger.Errorw("error in getting all installed apps in GetAllGitOpsAppNameAndInstalledAppMapping", "err", err, "gitOpsAppName", gitOpsAppName) @@ -469,13 +470,13 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA return isValid, installedAppVersionHistory, appId, envId, err } if installedAppVersionHistoryByHash.StartedOn.Before(installedAppVersionHistory.StartedOn) { - //we have received trigger hash which is committed before this apps actual gitHash stored by us + // we have received trigger hash which is committed before this apps actual gitHash stored by us // this means that the hash stored by us will be synced later, so we will drop this event return isValid, installedAppVersionHistory, appId, envId, nil } } if util2.IsTerminalStatus(installedAppVersionHistory.Status) { - //drop event + // drop event return isValid, installedAppVersionHistory, appId, envId, nil } if !impl.acdConfig.ArgoCDAutoSyncEnabled { @@ -491,7 +492,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHash string) (bool, pipelineConfig.Pipeline, pipelineConfig.CdWorkflowRunner, *chartConfig.PipelineOverride, error) { isValid := false var err error - //var deploymentStatus repository.DeploymentStatus + // var deploymentStatus repository.DeploymentStatus var pipeline pipelineConfig.Pipeline var pipelineOverride *chartConfig.PipelineOverride var cdWfr pipelineConfig.CdWorkflowRunner @@ -500,7 +501,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa impl.logger.Errorw("error in getting cd pipeline by argoAppName", "err", err, "argoAppName", argoAppName) return isValid, pipeline, cdWfr, pipelineOverride, err } - //getting latest pipelineOverride for app (by appId and envId) + // getting latest pipelineOverride for app (by appId and envId) pipelineOverride, err = impl.pipelineOverrideRepository.FindLatestByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId, bean2.ArgoCd) if err != nil { impl.logger.Errorw("error in getting latest pipelineOverride by appId and envId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) @@ -513,7 +514,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa return isValid, pipeline, cdWfr, pipelineOverride, err } if pipelineOverrideByHash.CommitTime.Before(pipelineOverride.CommitTime) { - //we have received trigger hash which is committed before this apps actual gitHash stored by us + // we have received trigger hash which is committed before this apps actual gitHash stored by us // this means that the hash stored by us will be synced later, so we will drop this event return isValid, pipeline, cdWfr, pipelineOverride, nil } @@ -524,7 +525,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa return isValid, pipeline, cdWfr, pipelineOverride, err } if util2.IsTerminalStatus(cdWfr.Status) { - //drop event + // drop event return isValid, pipeline, cdWfr, pipelineOverride, nil } if !impl.acdConfig.ArgoCDAutoSyncEnabled { @@ -571,7 +572,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap latestTimelineBeforeUpdate *pipelineConfig.PipelineStatusTimeline, reconciledAt *metav1.Time, isAppStore bool) (isTimelineUpdated bool, isTimelineTimedOut bool, kubectlApplySyncedTimeline *pipelineConfig.PipelineStatusTimeline, err error) { - //pipelineId can be wfrId or installedAppVersionHistoryId + // pipelineId can be wfrId or installedAppVersionHistoryId impl.logger.Debugw("updating pipeline status timeline", "app", app, "pipelineOverride", pipelineId, "APP_TO_UPDATE", app.Name) isTimelineUpdated = false isTimelineTimedOut = false @@ -604,13 +605,13 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap if app != nil && app.Status.OperationState != nil { timeline.StatusDetail = app.Status.OperationState.Message } - //checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times + // checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times _, err, isTimelineUpdated = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(pipelineId, timeline.Status, timeline, false) if err != nil { impl.logger.Errorw("error in saving pipeline status timeline", "err", err) return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, err } - //saving timeline resource details + // saving timeline resource details err = impl.pipelineStatusTimelineResourcesService.SaveOrUpdatePipelineTimelineResources(pipelineId, app, nil, 1, false) if err != nil { impl.logger.Errorw("error in saving/updating timeline resources", "err", err, "cdWfrId", pipelineId) @@ -625,7 +626,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap timeline.Id = 0 timeline.Status = pipelineConfig.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED timeline.StatusDetail = app.Status.OperationState.Message - //checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times + // checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) if err != nil { impl.logger.Errorw("error in saving pipeline status timeline", "err", err) @@ -645,7 +646,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap timeline.StatusDetail = "App status is Healthy." } if haveNewTimeline { - //not checking if this status is already present or not because already checked for terminal status existence earlier + // not checking if this status is already present or not because already checked for terminal status existence earlier err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, false) if err != nil { impl.logger.Errorw("error in creating timeline status", "err", err, "timeline", timeline) @@ -657,7 +658,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap } if !isTimelineUpdated { - //no timeline updated since before, in this case we will check for timeout cases + // no timeline updated since before, in this case we will check for timeout cases var lastTimeToCheckForTimeout time.Time if latestTimelineBeforeUpdate == nil { lastTimeToCheckForTimeout = triggeredAt @@ -665,7 +666,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap lastTimeToCheckForTimeout = latestTimelineBeforeUpdate.StatusTime } if time.Since(lastTimeToCheckForTimeout) >= time.Duration(statusTimeoutDuration)*time.Minute { - //mark as timed out if not already marked + // mark as timed out if not already marked timeline.Status = pipelineConfig.TIMELINE_STATUS_FETCH_TIMED_OUT timeline.StatusDetail = "Deployment timed out." _, err, isTimelineUpdated = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(pipelineId, timeline.Status, timeline, false) @@ -707,13 +708,13 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap if app != nil && app.Status.OperationState != nil { timeline.StatusDetail = app.Status.OperationState.Message } - //checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times + // checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times _, err, isTimelineUpdated = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(pipelineId, timeline.Status, timeline, true) if err != nil { impl.logger.Errorw("error in saving pipeline status timeline", "err", err) return isTimelineUpdated, isTimelineTimedOut, kubectlApplySyncedTimeline, err } - //saving timeline resource details + // saving timeline resource details err = impl.pipelineStatusTimelineResourcesService.SaveOrUpdatePipelineTimelineResources(pipelineId, app, nil, 1, true) if err != nil { impl.logger.Errorw("error in saving/updating timeline resources", "err", err, "installedAppVersionId", pipelineId) @@ -728,7 +729,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap timeline.Id = 0 timeline.Status = pipelineConfig.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED timeline.StatusDetail = app.Status.OperationState.Message - //checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times + // checking and saving if this timeline is present or not because kubewatch may stream same objects multiple times err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, true) if err != nil { impl.logger.Errorw("error in saving pipeline status timeline", "err", err) @@ -748,7 +749,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap timeline.StatusDetail = "App status is Healthy." } if haveNewTimeline { - //not checking if this status is already present or not because already checked for terminal status existence earlier + // not checking if this status is already present or not because already checked for terminal status existence earlier err = impl.pipelineStatusTimelineService.SaveTimeline(timeline, nil, true) if err != nil { impl.logger.Errorw("error in creating timeline status", "err", err, "timeline", timeline) @@ -760,7 +761,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap } if !isTimelineUpdated { - //no timeline updated since before, in this case we will check for timeout cases + // no timeline updated since before, in this case we will check for timeout cases var lastTimeToCheckForTimeout time.Time if latestTimelineBeforeUpdate == nil { lastTimeToCheckForTimeout = triggeredAt @@ -768,7 +769,7 @@ func (impl *AppServiceImpl) UpdatePipelineStatusTimelineForApplicationChanges(ap lastTimeToCheckForTimeout = latestTimelineBeforeUpdate.StatusTime } if time.Since(lastTimeToCheckForTimeout) >= time.Duration(statusTimeoutDuration)*time.Minute { - //mark as timed out if not already marked + // mark as timed out if not already marked timeline.Status = pipelineConfig.TIMELINE_STATUS_FETCH_TIMED_OUT timeline.StatusDetail = "Deployment timed out." _, err, isTimelineUpdated = impl.pipelineStatusTimelineService.SavePipelineStatusTimelineIfNotAlreadyPresent(pipelineId, timeline.Status, timeline, true) @@ -814,7 +815,7 @@ type ValuesOverrideResponse struct { } func (impl *AppServiceImpl) buildACDContext() (acdContext context.Context, err error) { - //this method should only call in case of argo-integration and gitops configured + // this method should only call in case of argo-integration and gitops configured acdToken, err := impl.argoUserService.GetLatestDevtronArgoCdUserToken() if err != nil { impl.logger.Errorw("error in getting acd token", "err", err) @@ -999,8 +1000,8 @@ func (impl *AppServiceImpl) GetCmSecretNew(appId int, envId int, isJob bool, sco var secretDataJsonApp string var configMapJsonEnv string var secretDataJsonEnv string - //var configMapJsonPipeline string - //var secretDataJsonPipeline string + // var configMapJsonPipeline string + // var secretDataJsonPipeline string configMapA, err := impl.configMapRepository.GetByAppIdAppLevel(appId) if err != nil && pg.ErrNoRows != err { @@ -1260,3 +1261,7 @@ func (impl *AppServiceImpl) IsDevtronAsyncInstallModeEnabled(deploymentAppType s return impl.appStatusConfig.EnableAsyncInstallDevtronChart && deploymentAppType == bean2.Helm } + +func (impl *AppServiceImpl) GetActiveCiCdAppsCount() (int, error) { + return impl.appRepository.GetActiveCiCdAppsCount() +} diff --git a/pkg/infraConfig/bean.go b/pkg/infraConfig/bean.go new file mode 100644 index 0000000000..263f24924a --- /dev/null +++ b/pkg/infraConfig/bean.go @@ -0,0 +1,276 @@ +package infraConfig + +import ( + "github.com/devtron-labs/devtron/pkg/infraConfig/units" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/util" + "math" + "time" +) + +// repo structs + +type InfraProfileEntity struct { + tableName struct{} `sql:"infra_profile" pg:",discard_unknown_columns"` + Id int `sql:"id"` + Name string `sql:"name"` + Description string `sql:"description"` + Active bool `sql:"active"` + sql.AuditLog +} + +func (infraProfile *InfraProfileEntity) ConvertToProfileBean() ProfileBean { + profileType := DEFAULT + if infraProfile.Name != DEFAULT_PROFILE_NAME { + profileType = NORMAL + } + return ProfileBean{ + Id: infraProfile.Id, + Name: infraProfile.Name, + Type: profileType, + Description: infraProfile.Description, + Active: infraProfile.Active, + CreatedBy: infraProfile.CreatedBy, + CreatedOn: infraProfile.CreatedOn, + UpdatedBy: infraProfile.UpdatedBy, + UpdatedOn: infraProfile.UpdatedOn, + } +} + +type InfraProfileConfigurationEntity struct { + tableName struct{} `sql:"infra_profile_configuration" pg:",discard_unknown_columns"` + Id int `sql:"id"` + Key ConfigKey `sql:"key"` + Value float64 `sql:"value"` + Unit units.UnitSuffix `sql:"unit"` + ProfileId int `sql:"profile_id"` + Active bool `sql:"active"` + sql.AuditLog +} + +func (infraProfileConfiguration *InfraProfileConfigurationEntity) ConvertToConfigurationBean() ConfigurationBean { + return ConfigurationBean{ + Id: infraProfileConfiguration.Id, + Key: GetConfigKeyStr(infraProfileConfiguration.Key), + Value: infraProfileConfiguration.Value, + Unit: GetUnitSuffixStr(infraProfileConfiguration.Key, infraProfileConfiguration.Unit), + ProfileId: infraProfileConfiguration.ProfileId, + Active: infraProfileConfiguration.Active, + } +} + +// service layer structs + +type ProfileBean struct { + Id int `json:"id"` + Name string `json:"name" validate:"required,min=1,max=50"` + Description string `json:"description" validate:"max=300"` + Active bool `json:"active"` + Configurations []ConfigurationBean `json:"configurations" validate:"dive"` + Type ProfileType `json:"type"` + AppCount int `json:"appCount"` + CreatedBy int32 `json:"createdBy"` + CreatedOn time.Time `json:"createdOn"` + UpdatedBy int32 `json:"updatedBy"` + UpdatedOn time.Time `json:"updatedOn"` +} + +func (profileBean *ProfileBean) ConvertToInfraProfileEntity() *InfraProfileEntity { + return &InfraProfileEntity{ + Id: profileBean.Id, + Name: profileBean.Name, + Description: profileBean.Description, + } +} + +type ConfigurationBean struct { + Id int `json:"id"` + Key ConfigKeyStr `json:"key"` + Value float64 `json:"value" validate:"required,gt=0"` + Unit string `json:"unit" validate:"required,gt=0"` + ProfileName string `json:"profileName"` + ProfileId int `json:"profileId"` + Active bool `json:"active"` +} + +func (configurationBean *ConfigurationBean) ConvertToInfraProfileConfigurationEntity() *InfraProfileConfigurationEntity { + value := util.TruncateFloat(configurationBean.Value, 2) + if configurationBean.Key == TIME_OUT { + value = math.Min(math.Floor(value), math.MaxInt64) + } + return &InfraProfileConfigurationEntity{ + Id: configurationBean.Id, + Key: GetConfigKey(configurationBean.Key), + Value: value, + Unit: GetUnitSuffix(configurationBean.Key, configurationBean.Unit), + ProfileId: configurationBean.ProfileId, + Active: configurationBean.Active, + } +} + +type InfraConfigMetaData struct { + DefaultConfigurations []ConfigurationBean `json:"defaultConfigurations"` + ConfigurationUnits map[ConfigKeyStr]map[string]units.Unit `json:"configurationUnits"` +} +type ProfileResponse struct { + Profile ProfileBean `json:"profile"` + InfraConfigMetaData +} + +type ProfilesResponse struct { + Profiles []ProfileBean `json:"profiles"` + InfraConfigMetaData +} + +type Scope struct { + AppId int +} + +// InfraConfig is used for read only purpose outside this package +type InfraConfig struct { + // currently only for ci + CiLimitCpu string `env:"LIMIT_CI_CPU" envDefault:"0.5"` + CiLimitMem string `env:"LIMIT_CI_MEM" envDefault:"3G"` + CiReqCpu string `env:"REQ_CI_CPU" envDefault:"0.5"` + CiReqMem string `env:"REQ_CI_MEM" envDefault:"3G"` + CiDefaultTimeout int64 `env:"DEFAULT_TIMEOUT" envDefault:"3600"` +} + +func (infraConfig InfraConfig) GetCiLimitCpu() string { + return infraConfig.CiLimitCpu +} + +func (infraConfig *InfraConfig) setCiLimitCpu(cpu string) { + infraConfig.CiLimitCpu = cpu +} + +func (infraConfig InfraConfig) GetCiLimitMem() string { + return infraConfig.CiLimitMem +} + +func (infraConfig *InfraConfig) setCiLimitMem(mem string) { + infraConfig.CiLimitMem = mem +} + +func (infraConfig InfraConfig) GetCiReqCpu() string { + return infraConfig.CiReqCpu +} + +func (infraConfig *InfraConfig) setCiReqCpu(cpu string) { + infraConfig.CiReqCpu = cpu +} + +func (infraConfig InfraConfig) GetCiReqMem() string { + return infraConfig.CiReqMem +} + +func (infraConfig *InfraConfig) setCiReqMem(mem string) { + infraConfig.CiReqMem = mem +} + +func (infraConfig InfraConfig) GetCiDefaultTimeout() int64 { + return infraConfig.CiDefaultTimeout +} + +func (infraConfig *InfraConfig) setCiDefaultTimeout(timeout int64) { + infraConfig.CiDefaultTimeout = timeout +} + +func (infraConfig InfraConfig) LoadCiLimitCpu() (*InfraProfileConfigurationEntity, error) { + val, suffix, err := units.ParseValAndUnit(infraConfig.CiLimitCpu) + if err != nil { + return nil, err + } + return &InfraProfileConfigurationEntity{ + Key: CPULimit, + Value: val, + Unit: units.CPUUnitStr(suffix).GetCPUUnit(), + }, nil + +} + +func (infraConfig InfraConfig) LoadCiLimitMem() (*InfraProfileConfigurationEntity, error) { + val, suffix, err := units.ParseValAndUnit(infraConfig.CiLimitMem) + if err != nil { + return nil, err + } + return &InfraProfileConfigurationEntity{ + Key: MemoryLimit, + Value: val, + Unit: units.MemoryUnitStr(suffix).GetMemoryUnit(), + }, nil + +} + +func (infraConfig InfraConfig) LoadCiReqCpu() (*InfraProfileConfigurationEntity, error) { + val, suffix, err := units.ParseValAndUnit(infraConfig.CiReqCpu) + if err != nil { + return nil, err + } + return &InfraProfileConfigurationEntity{ + Key: CPURequest, + Value: val, + Unit: units.CPUUnitStr(suffix).GetCPUUnit(), + }, nil +} + +func (infraConfig InfraConfig) LoadCiReqMem() (*InfraProfileConfigurationEntity, error) { + val, suffix, err := units.ParseValAndUnit(infraConfig.CiReqMem) + if err != nil { + return nil, err + } + + return &InfraProfileConfigurationEntity{ + Key: MemoryRequest, + Value: val, + Unit: units.MemoryUnitStr(suffix).GetMemoryUnit(), + }, nil +} + +func (infraConfig InfraConfig) LoadDefaultTimeout() (*InfraProfileConfigurationEntity, error) { + return &InfraProfileConfigurationEntity{ + Key: TimeOut, + Value: float64(infraConfig.CiDefaultTimeout), + Unit: units.SecondStr.GetTimeUnit(), + }, nil +} + +func (infraConfig InfraConfig) LoadInfraConfigInEntities() ([]*InfraProfileConfigurationEntity, error) { + cpuLimit, err := infraConfig.LoadCiLimitCpu() + if err != nil { + return nil, err + } + memLimit, err := infraConfig.LoadCiLimitMem() + if err != nil { + return nil, err + } + cpuReq, err := infraConfig.LoadCiReqCpu() + if err != nil { + return nil, err + } + memReq, err := infraConfig.LoadCiReqMem() + if err != nil { + return nil, err + } + timeout, err := infraConfig.LoadDefaultTimeout() + if err != nil { + return nil, err + } + + defaultConfigurations := []*InfraProfileConfigurationEntity{cpuLimit, memLimit, cpuReq, memReq, timeout} + return defaultConfigurations, nil +} + +func UpdateProfileMissingConfigurationsWithDefault(profile ProfileBean, defaultConfigurations []ConfigurationBean) ProfileBean { + extraConfigurations := make([]ConfigurationBean, 0) + for _, defaultConfiguration := range defaultConfigurations { + // if profile doesn't have the default configuration, add it to the profile + if !util.Contains(profile.Configurations, func(config ConfigurationBean) bool { + return config.Key == defaultConfiguration.Key + }) { + extraConfigurations = append(extraConfigurations, defaultConfiguration) + } + } + profile.Configurations = append(profile.Configurations, extraConfigurations...) + return profile +} diff --git a/pkg/infraConfig/constants.go b/pkg/infraConfig/constants.go new file mode 100644 index 0000000000..1d218cc669 --- /dev/null +++ b/pkg/infraConfig/constants.go @@ -0,0 +1,31 @@ +package infraConfig + +type ConfigKey int +type ConfigKeyStr string +type ProfileType string + +const NORMAL ProfileType = "NORMAL" +const InvalidUnit = "invalid %s unit found in %s " +const DEFAULT_PROFILE_NAME = "default" +const DEFAULT_PROFILE_EXISTS = "default profile exists" +const NO_PROPERTIES_FOUND = "no properties found" +const DEFAULT ProfileType = "DEFAULT" +const InvalidProfileName = "profile name is invalid" +const PayloadValidationError = "payload validation failed" +const CPULimReqErrorCompErr = "cpu limit should not be less than cpu request" +const MEMLimReqErrorCompErr = "memory limit should not be less than memory request" + +const CPULimit ConfigKey = 1 +const CPURequest ConfigKey = 2 +const MemoryLimit ConfigKey = 3 +const MemoryRequest ConfigKey = 4 +const TimeOut ConfigKey = 5 + +// whenever new constant gets added here , +// we need to add it in GetDefaultConfigKeysMap method as well + +const CPU_LIMIT ConfigKeyStr = "cpu_limit" +const CPU_REQUEST ConfigKeyStr = "cpu_request" +const MEMORY_LIMIT ConfigKeyStr = "memory_limit" +const MEMORY_REQUEST ConfigKeyStr = "memory_request" +const TIME_OUT ConfigKeyStr = "timeout" diff --git a/pkg/infraConfig/infraConfigRepository.go b/pkg/infraConfig/infraConfigRepository.go new file mode 100644 index 0000000000..7bf97817bb --- /dev/null +++ b/pkg/infraConfig/infraConfigRepository.go @@ -0,0 +1,102 @@ +package infraConfig + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" + "github.com/pkg/errors" +) + +type InfraConfigRepository interface { + GetProfileByName(name string) (*InfraProfileEntity, error) + GetConfigurationsByProfileName(profileName string) ([]*InfraProfileConfigurationEntity, error) + GetConfigurationsByProfileId(profileId int) ([]*InfraProfileConfigurationEntity, error) + + CreateProfile(tx *pg.Tx, infraProfile *InfraProfileEntity) error + CreateConfigurations(tx *pg.Tx, configurations []*InfraProfileConfigurationEntity) error + + UpdateConfigurations(tx *pg.Tx, configurations []*InfraProfileConfigurationEntity) error + UpdateProfile(tx *pg.Tx, profileName string, profile *InfraProfileEntity) error + sql.TransactionWrapper +} + +type InfraConfigRepositoryImpl struct { + dbConnection *pg.DB + *sql.TransactionUtilImpl +} + +func NewInfraProfileRepositoryImpl(dbConnection *pg.DB) *InfraConfigRepositoryImpl { + return &InfraConfigRepositoryImpl{ + dbConnection: dbConnection, + TransactionUtilImpl: sql.NewTransactionUtilImpl(dbConnection), + } +} + +// CreateProfile saves the default profile in the database only once in a lifetime. +// If the default profile already exists, it will not be saved again. +func (impl *InfraConfigRepositoryImpl) CreateProfile(tx *pg.Tx, infraProfile *InfraProfileEntity) error { + err := tx.Insert(infraProfile) + return err +} + +func (impl *InfraConfigRepositoryImpl) GetProfileByName(name string) (*InfraProfileEntity, error) { + infraProfile := &InfraProfileEntity{} + err := impl.dbConnection.Model(infraProfile). + Where("name = ?", name). + Where("active = ?", true). + Select() + return infraProfile, err +} + +func (impl *InfraConfigRepositoryImpl) CreateConfigurations(tx *pg.Tx, configurations []*InfraProfileConfigurationEntity) error { + err := tx.Insert(&configurations) + return err +} + +func (impl *InfraConfigRepositoryImpl) UpdateConfigurations(tx *pg.Tx, configurations []*InfraProfileConfigurationEntity) error { + var err error + for _, configuration := range configurations { + _, err = tx.Model(configuration). + Set("value = ?", configuration.Value). + Set("unit = ?", configuration.Unit). + Set("updated_by = ?", configuration.UpdatedBy). + Set("updated_on = ?", configuration.UpdatedOn). + Where("id = ?", configuration.Id). + Update() + } + return err +} + +func (impl *InfraConfigRepositoryImpl) GetConfigurationsByProfileName(profileName string) ([]*InfraProfileConfigurationEntity, error) { + var configurations []*InfraProfileConfigurationEntity + err := impl.dbConnection.Model(&configurations). + Where("profile_id IN (SELECT id FROM infra_profile WHERE name = ? AND active = true)", profileName). + Where("active = ?", true). + Select() + if errors.Is(err, pg.ErrNoRows) { + return nil, errors.New(NO_PROPERTIES_FOUND) + } + return configurations, err +} + +func (impl *InfraConfigRepositoryImpl) GetConfigurationsByProfileId(profileId int) ([]*InfraProfileConfigurationEntity, error) { + var configurations []*InfraProfileConfigurationEntity + err := impl.dbConnection.Model(&configurations). + Where("profile_id = ?", profileId). + Where("active = ?", true). + Select() + if errors.Is(err, pg.ErrNoRows) { + return nil, errors.New(NO_PROPERTIES_FOUND) + } + return configurations, err +} + +func (impl *InfraConfigRepositoryImpl) UpdateProfile(tx *pg.Tx, profileName string, profile *InfraProfileEntity) error { + _, err := tx.Model(profile). + Set("description=?", profile.Description). + Set("updated_by=?", profile.UpdatedBy). + Set("updated_on=?", profile.UpdatedOn). + Where("name = ?", profileName). + Where("active = ?", true). + Update() + return err +} diff --git a/pkg/infraConfig/infraConfigService.go b/pkg/infraConfig/infraConfigService.go new file mode 100644 index 0000000000..c7fb89f591 --- /dev/null +++ b/pkg/infraConfig/infraConfigService.go @@ -0,0 +1,393 @@ +package infraConfig + +import ( + "fmt" + "github.com/caarlos0/env" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/infraConfig/units" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/util" + "github.com/go-pg/pg" + "github.com/pkg/errors" + "go.uber.org/zap" + "time" +) + +type InfraConfigService interface { + + // GetConfigurationUnits fetches all the units for the configurations. + GetConfigurationUnits() map[ConfigKeyStr]map[string]units.Unit + // GetProfileByName fetches the profile and its configurations matching the given profileName. + GetProfileByName(name string) (*ProfileBean, error) + // UpdateProfile updates the profile and its configurations matching the given profileName. + // If profileName is empty, it will return an error. + UpdateProfile(userId int32, profileName string, profileBean *ProfileBean) error + + GetInfraConfigurationsByScope(scope Scope) (*InfraConfig, error) +} + +type InfraConfigServiceImpl struct { + logger *zap.SugaredLogger + infraProfileRepo InfraConfigRepository + appService app.AppService + units *units.Units + infraConfig *InfraConfig +} + +func NewInfraConfigServiceImpl(logger *zap.SugaredLogger, + infraProfileRepo InfraConfigRepository, + appService app.AppService, + units *units.Units) (*InfraConfigServiceImpl, error) { + infraConfiguration := &InfraConfig{} + err := env.Parse(infraConfiguration) + if err != nil { + return nil, err + } + infraProfileService := &InfraConfigServiceImpl{ + logger: logger, + infraProfileRepo: infraProfileRepo, + appService: appService, + units: units, + infraConfig: infraConfiguration, + } + err = infraProfileService.loadDefaultProfile() + return infraProfileService, err +} + +func (impl *InfraConfigServiceImpl) GetProfileByName(name string) (*ProfileBean, error) { + infraProfile, err := impl.infraProfileRepo.GetProfileByName(name) + if err != nil { + impl.logger.Errorw("error in fetching default profile", "error", err) + return nil, err + } + + profileBean := infraProfile.ConvertToProfileBean() + infraConfigurations, err := impl.infraProfileRepo.GetConfigurationsByProfileId(infraProfile.Id) + if err != nil { + impl.logger.Errorw("error in fetching default configurations", "error", err) + return nil, err + } + + configurationBeans := util.Transform(infraConfigurations, func(config *InfraProfileConfigurationEntity) ConfigurationBean { + configBean := config.ConvertToConfigurationBean() + configBean.ProfileName = profileBean.Name + return configBean + }) + + profileBean.Configurations = configurationBeans + appCount, err := impl.appService.GetActiveCiCdAppsCount() + if err != nil { + impl.logger.Errorw("error in fetching app count for default profile", "error", err) + return nil, err + } + profileBean.AppCount = appCount + return &profileBean, nil +} + +func (impl *InfraConfigServiceImpl) UpdateProfile(userId int32, profileName string, profileToUpdate *ProfileBean) error { + // validation + defaultProfile, err := impl.GetProfileByName(profileName) + if err != nil { + impl.logger.Errorw("error in fetching default profile", "profileName", profileName, "profileCreateRequest", profileToUpdate, "error", err) + return err + } + if err = impl.Validate(profileToUpdate, defaultProfile); err != nil { + impl.logger.Errorw("error occurred in validation the profile create request", "profileName", profileName, "profileCreateRequest", profileToUpdate, "error", err) + return err + } + // validations end + + infraProfileEntity := profileToUpdate.ConvertToInfraProfileEntity() + // user couldn't delete the profile, always set this to active + infraProfileEntity.Active = true + infraConfigurations := util.Transform(profileToUpdate.Configurations, func(config ConfigurationBean) *InfraProfileConfigurationEntity { + config.ProfileId = defaultProfile.Id + // user couldn't delete the configuration for default profile, always set this to active + if profileName == DEFAULT_PROFILE_NAME { + config.Active = true + } + configuration := config.ConvertToInfraProfileConfigurationEntity() + configuration.UpdatedOn = time.Now() + configuration.UpdatedBy = userId + return configuration + }) + + tx, err := impl.infraProfileRepo.StartTx() + if err != nil { + impl.logger.Errorw("error in starting transaction to update profile", "profileBean", profileToUpdate, "error", err) + return err + } + defer impl.infraProfileRepo.RollbackTx(tx) + infraProfileEntity.UpdatedOn = time.Now() + infraProfileEntity.UpdatedBy = userId + err = impl.infraProfileRepo.UpdateProfile(tx, profileName, infraProfileEntity) + if err != nil { + impl.logger.Errorw("error in updating profile", "error", "profileName", profileName, "profileCreateRequest", profileToUpdate, err) + return err + } + + err = impl.infraProfileRepo.UpdateConfigurations(tx, infraConfigurations) + if err != nil { + impl.logger.Errorw("error in creating configurations", "error", "profileName", profileName, "profileCreateRequest", profileToUpdate, err) + return err + } + err = impl.infraProfileRepo.CommitTx(tx) + if err != nil { + impl.logger.Errorw("error in committing transaction to update profile", "profileName", profileName, "profileCreateRequest", profileToUpdate, "error", err) + } + return err +} + +// loadDefaultProfile loads default configurations from environment and save them in db. +// this will only create the default profile only once if not exists in db.(container restarts won't create new default profile everytime) +// this will load the default configurations provided in InfraConfig. if db is in out of sync with InfraConfig then it will create new entries for those missing configurations in db. +func (impl *InfraConfigServiceImpl) loadDefaultProfile() error { + + profile, err := impl.infraProfileRepo.GetProfileByName(DEFAULT_PROFILE_NAME) + // make sure about no rows error + if err != nil && !errors.Is(err, pg.ErrNoRows) { + return err + } + profileCreationRequired := errors.Is(err, pg.ErrNoRows) + tx, err := impl.infraProfileRepo.StartTx() + if err != nil { + impl.logger.Errorw("error in starting transaction to save default configurations", "error", err) + return err + } + defer impl.infraProfileRepo.RollbackTx(tx) + if profileCreationRequired { + // if default profiles not found then create default profile + defaultProfile := &InfraProfileEntity{ + Name: DEFAULT_PROFILE_NAME, + Description: "", + Active: true, + AuditLog: sql.NewDefaultAuditLog(1), + } + + err = impl.infraProfileRepo.CreateProfile(tx, defaultProfile) + if err != nil { + impl.logger.Errorw("error in saving default profile", "error", err) + return err + } + profile = defaultProfile + } + + defaultConfigurationsFromEnv, err := impl.infraConfig.LoadInfraConfigInEntities() + if err != nil { + impl.logger.Errorw("error in loading default configurations from environment", "error", err) + return err + } + + // get db configurations and create new entries if db is out of sync + defaultConfigurationsFromDB, err := impl.infraProfileRepo.GetConfigurationsByProfileName(DEFAULT_PROFILE_NAME) + // todo: check the error logic here + if err != nil { + impl.logger.Errorw("error in fetching default configurations", "error", err) + return err + } + defaultConfigurationsFromDBMap := make(map[ConfigKey]bool) + for _, defaultConfigurationFromDB := range defaultConfigurationsFromDB { + defaultConfigurationsFromDBMap[defaultConfigurationFromDB.Key] = true + } + + creatableConfigurations := make([]*InfraProfileConfigurationEntity, 0, len(defaultConfigurationsFromEnv)) + for _, configurationFromEnv := range defaultConfigurationsFromEnv { + if !defaultConfigurationsFromDBMap[configurationFromEnv.Key] { + configurationFromEnv.ProfileId = profile.Id + configurationFromEnv.Active = true + configurationFromEnv.AuditLog = sql.NewDefaultAuditLog(1) + creatableConfigurations = append(creatableConfigurations, configurationFromEnv) + } + } + + if len(creatableConfigurations) > 0 { + err = impl.infraProfileRepo.CreateConfigurations(tx, creatableConfigurations) + if err != nil { + impl.logger.Errorw("error in saving default configurations", "configurations", creatableConfigurations, "error", err) + return err + } + } + + err = impl.infraProfileRepo.CommitTx(tx) + if err != nil { + impl.logger.Errorw("error in committing transaction to save default configurations", "error", err) + } + return err +} + +func (impl *InfraConfigServiceImpl) GetInfraConfigurationsByScope(scope Scope) (*InfraConfig, error) { + infraConfiguration := &InfraConfig{} + overrideInfraConfigFunc := func(config ConfigurationBean) { + switch config.Key { + case CPU_LIMIT: + infraConfiguration.setCiLimitCpu(impl.getResolvedValue(config).(string)) + case CPU_REQUEST: + infraConfiguration.setCiReqCpu(impl.getResolvedValue(config).(string)) + case MEMORY_LIMIT: + infraConfiguration.setCiLimitMem(impl.getResolvedValue(config).(string)) + case MEMORY_REQUEST: + infraConfiguration.setCiReqMem(impl.getResolvedValue(config).(string)) + case TIME_OUT: + infraConfiguration.setCiDefaultTimeout(impl.getResolvedValue(config).(int64)) + } + } + defaultConfigurations, err := impl.infraProfileRepo.GetConfigurationsByProfileName(DEFAULT_PROFILE_NAME) + if err != nil { + impl.logger.Errorw("error in fetching default configurations", "scope", scope, "error", err) + return nil, err + } + + for _, defaultConfiguration := range defaultConfigurations { + defaultConfigurationBean := defaultConfiguration.ConvertToConfigurationBean() + overrideInfraConfigFunc(defaultConfigurationBean) + } + return infraConfiguration, nil +} + +func (impl *InfraConfigServiceImpl) getResolvedValue(configurationBean ConfigurationBean) interface{} { + // for timeout we need to get the value in seconds + if configurationBean.Key == GetConfigKeyStr(TimeOut) { + // if user ever gives the timeout in float, after conversion to int64 it will be rounded off + timeUnit := units.TimeUnitStr(configurationBean.Unit) + return int64(configurationBean.Value * impl.units.GetTimeUnits()[timeUnit].ConversionFactor) + } + if configurationBean.Unit == string(units.CORE) || configurationBean.Unit == string(units.BYTE) { + return fmt.Sprintf("%v", configurationBean.Value) + } + return fmt.Sprintf("%v%v", configurationBean.Value, configurationBean.Unit) +} + +func (impl *InfraConfigServiceImpl) GetConfigurationUnits() map[ConfigKeyStr]map[string]units.Unit { + configurationUnits := make(map[ConfigKeyStr]map[string]units.Unit) + cpuUnits := make(map[string]units.Unit) + memUnits := make(map[string]units.Unit) + timeUnits := make(map[string]units.Unit) + for key, val := range impl.units.GetCpuUnits() { + cpuUnits[string(key)] = val + } + for key, val := range impl.units.GetMemoryUnits() { + memUnits[string(key)] = val + } + for key, val := range impl.units.GetTimeUnits() { + timeUnits[string(key)] = val + } + + configurationUnits[CPU_REQUEST] = cpuUnits + configurationUnits[CPU_LIMIT] = cpuUnits + + configurationUnits[MEMORY_REQUEST] = memUnits + configurationUnits[MEMORY_LIMIT] = memUnits + + configurationUnits[TIME_OUT] = timeUnits + + return configurationUnits +} + +func (impl *InfraConfigServiceImpl) Validate(profileToUpdate *ProfileBean, defaultProfile *ProfileBean) error { + var err error = nil + defaultConfigurationsKeyMap := GetDefaultConfigKeysMap() + // validate configurations only contain default configurations types.(cpu_limit,cpu_request,mem_limit,mem_request,timeout) + for _, propertyConfig := range profileToUpdate.Configurations { + if _, ok := defaultConfigurationsKeyMap[propertyConfig.Key]; !ok { + errorMsg := fmt.Sprintf("invalid configuration property \"%s\"", propertyConfig.Key) + if err == nil { + err = errors.New(errorMsg) + } + err = errors.Wrap(err, errorMsg) + } + } + + if err != nil { + err = errors.Wrap(err, PayloadValidationError) + return err + } + + err = impl.validateCpuMem(profileToUpdate, defaultProfile) + if err != nil { + err = errors.Wrap(err, PayloadValidationError) + return err + } + return nil +} + +func (impl *InfraConfigServiceImpl) validateCpuMem(profileBean *ProfileBean, defaultProfile *ProfileBean) error { + + // currently validating cpu and memory limits and reqs only + var ( + cpuLimit *ConfigurationBean + cpuReq *ConfigurationBean + memLimit *ConfigurationBean + memReq *ConfigurationBean + ) + + for i, _ := range profileBean.Configurations { + // get cpu limit and req + switch profileBean.Configurations[i].Key { + case CPU_LIMIT: + cpuLimit = &profileBean.Configurations[i] + case CPU_REQUEST: + cpuReq = &profileBean.Configurations[i] + case MEMORY_LIMIT: + memLimit = &profileBean.Configurations[i] + case MEMORY_REQUEST: + memReq = &profileBean.Configurations[i] + } + } + + // validate cpu + err := impl.validateCPU(cpuLimit, cpuReq) + if err != nil { + return err + } + // validate mem + err = impl.validateMEM(memLimit, memReq) + if err != nil { + return err + } + return nil +} + +func (impl *InfraConfigServiceImpl) validateCPU(cpuLimit, cpuReq *ConfigurationBean) error { + cpuLimitUnitSuffix := units.CPUUnitStr(cpuLimit.Unit) + cpuReqUnitSuffix := units.CPUUnitStr(cpuReq.Unit) + cpuUnits := impl.units.GetCpuUnits() + cpuLimitUnit, ok := cpuUnits[cpuLimitUnitSuffix] + if !ok { + return errors.New(fmt.Sprintf(InvalidUnit, cpuLimit.Unit, cpuLimit.Key)) + } + cpuReqUnit, ok := cpuUnits[cpuReqUnitSuffix] + if !ok { + return errors.New(fmt.Sprintf(InvalidUnit, cpuReq.Unit, cpuReq.Key)) + } + + if !validLimReq(cpuLimit.Value, cpuLimitUnit.ConversionFactor, cpuReq.Value, cpuReqUnit.ConversionFactor) { + return errors.New(CPULimReqErrorCompErr) + } + return nil +} + +func (impl *InfraConfigServiceImpl) validateMEM(memLimit, memReq *ConfigurationBean) error { + memLimitUnitSuffix := units.MemoryUnitStr(memLimit.Unit) + memReqUnitSuffix := units.MemoryUnitStr(memReq.Unit) + memUnits := impl.units.GetMemoryUnits() + memLimitUnit, ok := memUnits[memLimitUnitSuffix] + if !ok { + return errors.New(fmt.Sprintf(InvalidUnit, memLimit.Unit, memLimit.Key)) + } + memReqUnit, ok := memUnits[memReqUnitSuffix] + if !ok { + return errors.New(fmt.Sprintf(InvalidUnit, memReq.Unit, memReq.Key)) + } + + if !validLimReq(memLimit.Value, memLimitUnit.ConversionFactor, memReq.Value, memReqUnit.ConversionFactor) { + return errors.New(MEMLimReqErrorCompErr) + } + return nil +} + +func validLimReq(lim, limFactor, req, reqFactor float64) bool { + // this condition should be true for valid case => (lim/req)*(lf/rf) >= 1 + limitToReqRatio := lim / req + convFactor := limFactor / reqFactor + return limitToReqRatio*convFactor >= 1 +} diff --git a/pkg/infraConfig/units/units.go b/pkg/infraConfig/units/units.go new file mode 100644 index 0000000000..7a0d82bc31 --- /dev/null +++ b/pkg/infraConfig/units/units.go @@ -0,0 +1,442 @@ +package units + +import ( + "github.com/devtron-labs/devtron/util" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/api/resource" + "strconv" + "strings" +) + +// memory units +// Ei, Pi, Ti, Gi, Mi, Ki +// E, P, T, G, M, k + +type UnitSuffix int + +const ( + Byte UnitSuffix = 1 + KiByte UnitSuffix = 2 // 1024 + MiByte UnitSuffix = 3 + GiByte UnitSuffix = 4 + TiByte UnitSuffix = 5 + PiByte UnitSuffix = 6 + EiByte UnitSuffix = 7 + K UnitSuffix = 8 // 1000 + M UnitSuffix = 9 + G UnitSuffix = 10 + T UnitSuffix = 11 + P UnitSuffix = 12 + E UnitSuffix = 13 + Core UnitSuffix = 14 // CPU cores + Milli UnitSuffix = 15 + Second UnitSuffix = 16 + Minute UnitSuffix = 17 + Hour UnitSuffix = 18 +) + +type UnitStr interface { + CPUUnitStr | MemoryUnitStr | TimeUnitStr +} + +type CPUUnitStr string + +func (cpuUnit UnitSuffix) GetCPUUnitStr() CPUUnitStr { + switch cpuUnit { + case Core: + return CORE + case Milli: + return MILLI + default: + return CORE + } +} + +func (cpuUnitStr CPUUnitStr) GetCPUUnit() UnitSuffix { + switch cpuUnitStr { + case CORE: + return Core + case MILLI: + return Milli + default: + return Core + } +} + +const ( + CORE CPUUnitStr = "Core" + MILLI CPUUnitStr = "m" +) + +type MemoryUnitStr string + +const ( + BYTE MemoryUnitStr = "byte" + KIBYTE MemoryUnitStr = "Ki" + MIBYTE MemoryUnitStr = "Mi" + GIBYTE MemoryUnitStr = "Gi" + TIBYTE MemoryUnitStr = "Ti" + PIBYTE MemoryUnitStr = "Pi" + EIBYTE MemoryUnitStr = "Ei" + KBYTE MemoryUnitStr = "k" + MBYTE MemoryUnitStr = "M" + GBYTE MemoryUnitStr = "G" + TBYTE MemoryUnitStr = "T" + PBYTE MemoryUnitStr = "P" + EBYTE MemoryUnitStr = "E" +) + +func (memoryUnit UnitSuffix) GetMemoryUnitStr() MemoryUnitStr { + switch memoryUnit { + case Byte: + return BYTE + case KiByte: + return KIBYTE + case MiByte: + return MIBYTE + case GiByte: + return GIBYTE + case TiByte: + return TIBYTE + case PiByte: + return PIBYTE + case EiByte: + return EIBYTE + case K: + return KBYTE + case M: + return MBYTE + case G: + return GBYTE + case T: + return TBYTE + case P: + return PBYTE + case E: + return EBYTE + default: + return BYTE + } +} + +func (memoryUnitStr MemoryUnitStr) GetMemoryUnit() UnitSuffix { + switch memoryUnitStr { + case BYTE: + return Byte + case KIBYTE: + return KiByte + case MIBYTE: + return MiByte + case GIBYTE: + return GiByte + case TIBYTE: + return TiByte + case PIBYTE: + return PiByte + case EIBYTE: + return EiByte + case KBYTE: + return K + case MBYTE: + return M + case GBYTE: + return G + case TBYTE: + return T + case PBYTE: + return P + case EBYTE: + return E + default: + return Byte + } +} + +type TimeUnitStr string + +const ( + SecondStr TimeUnitStr = "Seconds" + MinuteStr TimeUnitStr = "Minutes" + HourStr TimeUnitStr = "Hours" +) + +func (timeUnit UnitSuffix) GetTimeUnitStr() TimeUnitStr { + switch timeUnit { + case Second: + return SecondStr + case Minute: + return MinuteStr + case Hour: + return HourStr + default: + return SecondStr + } +} + +func (timeUnitStr TimeUnitStr) GetTimeUnit() UnitSuffix { + switch timeUnitStr { + case SecondStr: + return Second + case MinuteStr: + return Minute + case HourStr: + return Hour + default: + return Second + } +} + +type Units struct { + cpuUnits map[CPUUnitStr]Unit + memoryUnits map[MemoryUnitStr]Unit + timeUnits map[TimeUnitStr]Unit +} + +func NewUnits() *Units { + cpuUnits := map[CPUUnitStr]Unit{ + MILLI: { + Name: string(MILLI), + ConversionFactor: 1e-3, + }, + CORE: { + Name: string(CORE), + ConversionFactor: 1, + }, + } + + memoryUnits := map[MemoryUnitStr]Unit{ + BYTE: { + Name: string(BYTE), + ConversionFactor: 1, + }, + KBYTE: { + Name: string(KBYTE), + ConversionFactor: 1000, + }, + MBYTE: { + Name: string(MBYTE), + ConversionFactor: 1000000, + }, + GBYTE: { + Name: string(GBYTE), + ConversionFactor: 1000000000, + }, + TBYTE: { + Name: string(TBYTE), + ConversionFactor: 1000000000000, + }, + PBYTE: { + Name: string(PBYTE), + ConversionFactor: 1000000000000000, + }, + EBYTE: { + Name: string(EBYTE), + ConversionFactor: 1000000000000000000, + }, + KIBYTE: { + Name: string(KIBYTE), + ConversionFactor: 1024, + }, + MIBYTE: { + Name: string(MIBYTE), + ConversionFactor: 1024 * 1024, + }, + GIBYTE: { + Name: string(GIBYTE), + ConversionFactor: 1024 * 1024 * 1024, + }, + TIBYTE: { + Name: string(TIBYTE), + ConversionFactor: 1024 * 1024 * 1024 * 1024, + }, + PIBYTE: { + Name: string(PIBYTE), + ConversionFactor: 1024 * 1024 * 1024 * 1024 * 1024, + }, + EIBYTE: { + Name: string(EIBYTE), + ConversionFactor: 1024 * 1024 * 1024 * 1024 * 1024 * 1024, + }, + } + + timeUnits := map[TimeUnitStr]Unit{ + SecondStr: { + Name: string(SecondStr), + ConversionFactor: 1, + }, + MinuteStr: { + Name: string(MinuteStr), + ConversionFactor: 60, + }, + HourStr: { + Name: string(HourStr), + ConversionFactor: 3600, + }, + } + return &Units{ + cpuUnits: cpuUnits, + memoryUnits: memoryUnits, + timeUnits: timeUnits, + } +} + +func (u *Units) GetCpuUnits() map[CPUUnitStr]Unit { + return u.cpuUnits +} + +func (u *Units) GetMemoryUnits() map[MemoryUnitStr]Unit { + return u.memoryUnits +} + +func (u *Units) GetTimeUnits() map[TimeUnitStr]Unit { + return u.timeUnits +} + +// Unit represents unit of a configuration +type Unit struct { + // Name is unit name + Name string `json:"name"` + // ConversionFactor is used to convert this unit to the base unit + // if ConversionFactor is 1, then this is the base unit + ConversionFactor float64 `json:"conversionFactor"` +} + +// ParseValAndUnit parses the quantity which have number values string and returns the value and unit +// returns error if parsing fails +func ParseValAndUnit(quantity string) (float64, string, error) { + positive, _, num, denom, suffix, err := ParseQuantityString(quantity) + if err != nil { + return 0, "", err + } + if !positive { + return 0, "", errors.New("negative value not allowed for cpu limits") + } + valStr := num + if denom != "" { + valStr = num + "." + denom + } + + val, err := strconv.ParseFloat(valStr, 64) + + // currently we are not supporting exponential values upto 2 decimals + val = util.TruncateFloat(val, 2) + return val, suffix, err +} + +// ParseQuantityString is a fast scanner for quantity values. +// this parsing is only for cpu and mem resources +func ParseQuantityString(str string) (positive bool, value, num, denom, suffix string, err error) { + positive = true + pos := 0 + end := len(str) + + // handle leading sign + if pos < end { + switch str[0] { + case '-': + positive = false + pos++ + case '+': + pos++ + } + } + + // strip leading zeros +Zeroes: + for i := pos; ; i++ { + if i >= end { + num = "0" + value = num + return + } + switch str[i] { + case '0': + pos++ + default: + break Zeroes + } + } + + // extract the numerator +Num: + for i := pos; ; i++ { + if i >= end { + num = str[pos:end] + value = str[0:end] + return + } + switch str[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + default: + num = str[pos:i] + pos = i + break Num + } + } + + // if we stripped all numerator positions, always return 0 + if len(num) == 0 { + num = "0" + } + + // handle a denominator + if pos < end && str[pos] == '.' { + pos++ + Denom: + for i := pos; ; i++ { + if i >= end { + denom = str[pos:end] + value = str[0:end] + return + } + switch str[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + default: + denom = str[pos:i] + pos = i + break Denom + } + } + // TODO: we currently allow 1.G, but we may not want to in the future. + // if len(denom) == 0 { + // err = ErrFormatWrong + // return + // } + } + value = str[0:pos] + + // grab the elements of the suffix + suffixStart := pos + for i := pos; ; i++ { + if i >= end { + suffix = str[suffixStart:end] + return + } + if !strings.ContainsAny(str[i:i+1], "eEinumkKMGTP") { + pos = i + break + } + } + if pos < end { + switch str[pos] { + case '-', '+': + pos++ + } + } +Suffix: + for i := pos; ; i++ { + if i >= end { + suffix = str[suffixStart:end] + return + } + switch str[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + default: + break Suffix + } + } + // we encountered a non decimal in the Suffix loop, but the last character + // was not a valid exponent + err = resource.ErrFormatWrong + return +} diff --git a/pkg/infraConfig/units/units_test.go b/pkg/infraConfig/units/units_test.go new file mode 100644 index 0000000000..987c8236f8 --- /dev/null +++ b/pkg/infraConfig/units/units_test.go @@ -0,0 +1,19 @@ +package units + +import ( + "fmt" + "testing" +) + +// todo: add more test cases +func TestParseQuantityString(t *testing.T) { + memLimit := "01.400Gi" + pos, val, num, denom, suf, err := ParseQuantityString(memLimit) + fmt.Println("pos: ", pos) + fmt.Println("val: ", val) + fmt.Println("num: ", num) + fmt.Println("denom: ", denom) + fmt.Println("suf: ", suf) + fmt.Println("err: ", err) + +} diff --git a/pkg/infraConfig/utils.go b/pkg/infraConfig/utils.go new file mode 100644 index 0000000000..ccd6001f53 --- /dev/null +++ b/pkg/infraConfig/utils.go @@ -0,0 +1,68 @@ +package infraConfig + +import "github.com/devtron-labs/devtron/pkg/infraConfig/units" + +// GetUnitSuffix loosely typed method to get the unit suffix using the unitKey type +func GetUnitSuffix(unitKey ConfigKeyStr, unitStr string) units.UnitSuffix { + switch unitKey { + case CPU_LIMIT, CPU_REQUEST: + return units.CPUUnitStr(unitStr).GetCPUUnit() + case MEMORY_LIMIT, MEMORY_REQUEST: + return units.MemoryUnitStr(unitStr).GetMemoryUnit() + } + return units.TimeUnitStr(unitStr).GetTimeUnit() +} + +// GetUnitSuffixStr loosely typed method to get the unit suffix using the unitKey type +func GetUnitSuffixStr(unitKey ConfigKey, unit units.UnitSuffix) string { + switch unitKey { + case CPULimit, CPURequest: + return string(unit.GetCPUUnitStr()) + case MemoryLimit, MemoryRequest: + return string(unit.GetMemoryUnitStr()) + } + return string(unit.GetTimeUnitStr()) +} + +// GetDefaultConfigKeysMap returns a map of default config keys +func GetDefaultConfigKeysMap() map[ConfigKeyStr]bool { + return map[ConfigKeyStr]bool{ + CPU_LIMIT: true, + CPU_REQUEST: true, + MEMORY_LIMIT: true, + MEMORY_REQUEST: true, + TIME_OUT: true, + } +} + +func GetConfigKeyStr(configKey ConfigKey) ConfigKeyStr { + switch configKey { + case CPULimit: + return CPU_LIMIT + case CPURequest: + return CPU_REQUEST + case MemoryLimit: + return MEMORY_LIMIT + case MemoryRequest: + return MEMORY_REQUEST + case TimeOut: + return TIME_OUT + } + return "" +} + +func GetConfigKey(configKeyStr ConfigKeyStr) ConfigKey { + switch configKeyStr { + case CPU_LIMIT: + return CPULimit + case CPU_REQUEST: + return CPURequest + case MEMORY_LIMIT: + return MemoryLimit + case MEMORY_REQUEST: + return MemoryRequest + case TIME_OUT: + return TimeOut + } + return 0 +} diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 4512ef81e3..d051a2ae26 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -21,6 +21,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/devtron-labs/devtron/pkg/infraConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "path/filepath" "strconv" "strings" @@ -82,6 +84,7 @@ type CiServiceImpl struct { scopedVariableManager variables.ScopedVariableManager pluginInputVariableParser PluginInputVariableParser globalPluginService plugin.GlobalPluginService + infraProvider infraProviders.InfraProvider } func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, @@ -96,6 +99,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService customTagService CustomTagService, pluginInputVariableParser PluginInputVariableParser, globalPluginService plugin.GlobalPluginService, + infraProvider infraProviders.InfraProvider, ) *CiServiceImpl { cis := &CiServiceImpl{ Logger: Logger, @@ -117,6 +121,7 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService customTagService: customTagService, pluginInputVariableParser: pluginInputVariableParser, globalPluginService: globalPluginService, + infraProvider: infraProvider, } config, err := types.GetCiConfig() if err != nil { @@ -178,7 +183,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) if isJob && env != nil { ciWorkflowConfig.Namespace = env.Namespace - //This will be populated for jobs running in selected environment + // This will be populated for jobs running in selected environment scope.EnvId = env.Id scope.ClusterId = env.ClusterId @@ -192,7 +197,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) ciWorkflowConfig.Namespace = impl.config.GetDefaultNamespace() } - //preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) + // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, bean2.CiStage, scope) if err != nil { impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) @@ -229,7 +234,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) } } - //savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" + // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) err = impl.updateCiWorkflow(workflowRequest, savedCiWf) @@ -337,7 +342,7 @@ func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflow := &pipelineConfig.CiWorkflow{ Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), - Status: pipelineConfig.WorkflowStarting, //starting CIStage + Status: pipelineConfig.WorkflowStarting, // starting CIStage Message: "", StartedOn: time.Now(), CiPipelineId: pipeline.Id, @@ -452,13 +457,34 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { - //found beforeDockerBuildScripts/afterDockerBuildScripts - //building preCiSteps & postCiSteps from them, refPluginsData not needed + // found beforeDockerBuildScripts/afterDockerBuildScripts + // building preCiSteps & postCiSteps from them, refPluginsData not needed preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) refPluginsData = []*bean2.RefPluginObject{} } + infraConfigScope := &infraConfig.Scope{ + AppId: pipeline.AppId, + } + infraGetter, err := impl.infraProvider.GetInfraProvider(bean2.CI_WORKFLOW_PIPELINE_TYPE) + if err != nil { + impl.Logger.Errorw("error in getting infra provider", "err", err, "infraProviderType", bean2.CI_WORKFLOW_PIPELINE_TYPE) + return nil, err + } + if isJob { + infraGetter, err = impl.infraProvider.GetInfraProvider(bean2.JOB_WORKFLOW_PIPELINE_TYPE) + if err != nil { + impl.Logger.Errorw("error in getting infra provider", "err", err, "infraProviderType", bean2.JOB_WORKFLOW_PIPELINE_TYPE) + return nil, err + } + } + infraConfiguration, err := infraGetter.GetInfraConfigurationsByScope(infraConfigScope) + if err != nil { + impl.Logger.Errorw("error in getting infra configuration using scope ", "ciPipelineId", pipeline.Id, "scope", infraConfigScope, "err", err) + return nil, err + } + if ciWorkflowConfig.CiCacheBucket == "" { ciWorkflowConfig.CiCacheBucket = impl.config.DefaultCacheBucket } @@ -470,8 +496,10 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. if ciWorkflowConfig.CiImage == "" { ciWorkflowConfig.CiImage = impl.config.GetDefaultImage() } + if ciWorkflowConfig.CiTimeout == 0 { - ciWorkflowConfig.CiTimeout = impl.config.GetDefaultTimeout() + // get it from infraConfig + ciWorkflowConfig.CiTimeout = infraConfiguration.GetCiDefaultTimeout() } ciTemplate := pipeline.CiTemplate @@ -543,7 +571,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. return nil, err } savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} - //imagePath = docker.io/avd0/dashboard:fd23414b + // imagePath = docker.io/avd0/dashboard:fd23414b imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") if len(imagePathSplit) >= 1 { dockerImageTag = imagePathSplit[len(imagePathSplit)-1] @@ -577,7 +605,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) } - //mergedArgs := string(merged) + // mergedArgs := string(merged) oldArgs := ciTemplate.Args ciBuildConfigBean, err = bean2.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) if err != nil { @@ -593,7 +621,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. buildContextCheckoutPath = checkoutPath } if ciBuildConfigBean.UseRootBuildContext { - //use root build context i.e '.' + // use root build context i.e '.' buildContextCheckoutPath = "." } @@ -689,7 +717,7 @@ func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig. } switch workflowRequest.CloudProvider { case types.BLOB_STORAGE_S3: - //No AccessKey is used for uploading artifacts, instead IAM based auth is used + // No AccessKey is used for uploading artifacts, instead IAM based auth is used workflowRequest.CiCacheRegion = ciWorkflowConfig.CiCacheRegion workflowRequest.CiCacheLocation = ciWorkflowConfig.CiCacheBucket workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfig, savedWf) @@ -802,8 +830,8 @@ func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, regis } func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*bean2.StepObject { - //before plugin support, few variables were set as env vars in ci-runner - //these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload + // before plugin support, few variables were set as env vars in ci-runner + // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload inputVars := []*bean2.VariableObject{ { Name: "DOCKER_IMAGE_TAG", diff --git a/pkg/pipeline/WorkflowService.go b/pkg/pipeline/WorkflowService.go index 675e3b47f2..a2cd7228d5 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/pipeline/WorkflowService.go @@ -28,9 +28,11 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/infraConfig" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/types" "go.uber.org/zap" v12 "k8s.io/api/core/v1" @@ -43,11 +45,11 @@ import ( type WorkflowService interface { SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, error) - //DeleteWorkflow(wfName string, namespace string) error + // DeleteWorkflow(wfName string, namespace string) error GetWorkflow(executorType pipelineConfig.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*unstructured.UnstructuredList, error) GetWorkflowStatus(executorType pipelineConfig.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*types.WorkflowStatus, error) - //ListAllWorkflows(namespace string) (*v1alpha1.WorkflowList, error) - //UpdateWorkflow(wf *v1alpha1.Workflow) (*v1alpha1.Workflow, error) + // ListAllWorkflows(namespace string) (*v1alpha1.WorkflowList, error) + // UpdateWorkflow(wf *v1alpha1.Workflow) (*v1alpha1.Workflow, error) TerminateWorkflow(executorType pipelineConfig.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config, isExt bool, environment *repository.Environment) error } @@ -62,6 +64,7 @@ type WorkflowServiceImpl struct { systemWorkflowExecutor executors.SystemWorkflowExecutor k8sUtil *k8s.K8sServiceImpl k8sCommonService k8s2.K8sCommonService + infraProvider infraProviders.InfraProvider } // TODO: Move to bean @@ -69,8 +72,10 @@ type WorkflowServiceImpl struct { func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository.EnvironmentRepository, ciCdConfig *types.CiCdConfig, appService app.AppService, globalCMCSService GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, k8sUtil *k8s.K8sServiceImpl, - systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService) (*WorkflowServiceImpl, error) { - commonWorkflowService := &WorkflowServiceImpl{Logger: Logger, + systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService, + infraProvider infraProviders.InfraProvider) (*WorkflowServiceImpl, error) { + commonWorkflowService := &WorkflowServiceImpl{ + Logger: Logger, ciCdConfig: ciCdConfig, appService: appService, envRepository: envRepository, @@ -79,6 +84,7 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository. k8sUtil: k8sUtil, systemWorkflowExecutor: systemWorkflowExecutor, k8sCommonService: k8sCommonService, + infraProvider: infraProvider, } restConfig, err := k8sUtil.GetK8sInClusterRestConfig() if err != nil { @@ -132,7 +138,20 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W workflowTemplate.Volumes = executors.ExtractVolumesFromCmCs(workflowConfigMaps, workflowSecrets) workflowRequest.AddNodeConstraintsFromConfig(&workflowTemplate, impl.ciCdConfig) - workflowMainContainer, err := workflowRequest.GetWorkflowMainContainer(impl.ciCdConfig, workflowJson, &workflowTemplate, workflowConfigMaps, workflowSecrets) + infraConfiguration := &infraConfig.InfraConfig{} + if workflowRequest.Type == bean3.CI_WORKFLOW_PIPELINE_TYPE || workflowRequest.Type == bean3.JOB_WORKFLOW_PIPELINE_TYPE { + infraConfigScope := &infraConfig.Scope{ + AppId: workflowRequest.AppId, + } + infraGetter, _ := impl.infraProvider.GetInfraProvider(workflowRequest.Type) + infraConfiguration, err = infraGetter.GetInfraConfigurationsByScope(infraConfigScope) + if err != nil { + impl.Logger.Errorw("error occurred while getting infra config", "infraConfigScope", infraConfigScope, "err", err) + return bean3.WorkflowTemplate{}, err + } + } + + workflowMainContainer, err := workflowRequest.GetWorkflowMainContainer(impl.ciCdConfig, infraConfiguration, workflowJson, &workflowTemplate, workflowConfigMaps, workflowSecrets) if err != nil { impl.Logger.Errorw("error occurred while getting workflow main container", "err", err) @@ -238,8 +257,8 @@ func (impl *WorkflowServiceImpl) addExistingCmCsInWorkflow(workflowRequest *type } } - //internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from - //in-cluster configured blob storage, if USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW = false and isExt = true + // internally inducing BlobStorageCmName and BlobStorageSecretName for getting logs, caches and artifacts from + // in-cluster configured blob storage, if USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW = false and isExt = true if workflowRequest.UseExternalClusterBlob { workflowConfigMaps, workflowSecrets = impl.addExtBlobStorageCmCsInResponse(workflowConfigMaps, workflowSecrets) } diff --git a/pkg/pipeline/infraProviders/InfraProvider.go b/pkg/pipeline/infraProviders/InfraProvider.go new file mode 100644 index 0000000000..86b9735996 --- /dev/null +++ b/pkg/pipeline/infraProviders/InfraProvider.go @@ -0,0 +1,40 @@ +package infraProviders + +import ( + "github.com/devtron-labs/devtron/pkg/infraConfig" + "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/ciPipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/job" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +type InfraProvider interface { + GetInfraProvider(providerType bean.WorkflowPipelineType) (infraGetters.InfraGetter, error) +} + +type InfraProviderImpl struct { + logger *zap.SugaredLogger + ciInfraGetter infraGetters.InfraGetter + jobInfraGetter infraGetters.InfraGetter +} + +func NewInfraProviderImpl(logger *zap.SugaredLogger, service infraConfig.InfraConfigService) *InfraProviderImpl { + return &InfraProviderImpl{ + logger: logger, + ciInfraGetter: ciPipeline.NewCiInfraGetter(service), + jobInfraGetter: job.NewJobInfraGetter(), + } +} + +func (infraProvider *InfraProviderImpl) GetInfraProvider(providerType bean.WorkflowPipelineType) (infraGetters.InfraGetter, error) { + switch providerType { + case bean.CI_WORKFLOW_PIPELINE_TYPE: + return infraProvider.ciInfraGetter, nil + case bean.JOB_WORKFLOW_PIPELINE_TYPE: + return infraProvider.jobInfraGetter, nil + default: + return nil, errors.New("Invalid workflow pipeline type") + } +} diff --git a/pkg/pipeline/infraProviders/infraGetters/ciPipeline/ciPipelineInfraGetter.go b/pkg/pipeline/infraProviders/infraGetters/ciPipeline/ciPipelineInfraGetter.go new file mode 100644 index 0000000000..88d49611f2 --- /dev/null +++ b/pkg/pipeline/infraProviders/infraGetters/ciPipeline/ciPipelineInfraGetter.go @@ -0,0 +1,17 @@ +package ciPipeline + +import "github.com/devtron-labs/devtron/pkg/infraConfig" + +// CiInfraGetter gets infra config for ci workflows +type CiInfraGetter struct { + infraConfigService infraConfig.InfraConfigService +} + +func NewCiInfraGetter(infraConfigService infraConfig.InfraConfigService) *CiInfraGetter { + return &CiInfraGetter{infraConfigService: infraConfigService} +} + +// GetInfraConfigurationsByScope gets infra config for ci workflows using the scope +func (ciInfraGetter CiInfraGetter) GetInfraConfigurationsByScope(scope *infraConfig.Scope) (*infraConfig.InfraConfig, error) { + return ciInfraGetter.infraConfigService.GetInfraConfigurationsByScope(*scope) +} diff --git a/pkg/pipeline/infraProviders/infraGetters/infraGetter.go b/pkg/pipeline/infraProviders/infraGetters/infraGetter.go new file mode 100644 index 0000000000..23aa82a894 --- /dev/null +++ b/pkg/pipeline/infraProviders/infraGetters/infraGetter.go @@ -0,0 +1,7 @@ +package infraGetters + +import "github.com/devtron-labs/devtron/pkg/infraConfig" + +type InfraGetter interface { + GetInfraConfigurationsByScope(scope *infraConfig.Scope) (*infraConfig.InfraConfig, error) +} diff --git a/pkg/pipeline/infraProviders/infraGetters/job/jobInfraGetter.go b/pkg/pipeline/infraProviders/infraGetters/job/jobInfraGetter.go new file mode 100644 index 0000000000..02c6bebe8b --- /dev/null +++ b/pkg/pipeline/infraProviders/infraGetters/job/jobInfraGetter.go @@ -0,0 +1,25 @@ +package job + +import ( + "github.com/caarlos0/env" + "github.com/devtron-labs/devtron/pkg/infraConfig" +) + +// JobInfraGetter gets infra config for job workflows +type JobInfraGetter struct { + jobInfra infraConfig.InfraConfig +} + +func NewJobInfraGetter() *JobInfraGetter { + infra := infraConfig.InfraConfig{} + env.Parse(&infra) + return &JobInfraGetter{ + jobInfra: infra, + } +} + +// GetInfraConfigurationsByScope gets infra config for ci workflows using the scope +func (jobInfraGetter JobInfraGetter) GetInfraConfigurationsByScope(scope *infraConfig.Scope) (*infraConfig.InfraConfig, error) { + infra := jobInfraGetter.jobInfra + return &infra, nil +} diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index b5a6ae11e1..934a82a682 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -19,6 +19,8 @@ import ( "time" ) +// build infra configurations like ciTimeout,ciCpuLimit,ciMemLimit,ciCpuReq,ciMemReq are being managed by infraConfig service + type CiCdConfig struct { // from ciConfig DefaultCacheBucket string `env:"DEFAULT_CACHE_BUCKET" envDefault:"ci-caching"` @@ -26,13 +28,8 @@ type CiCdConfig struct { CiLogsKeyPrefix string `env:"CI_LOGS_KEY_PREFIX" envDxefault:"my-artifacts"` CiDefaultImage string `env:"DEFAULT_CI_IMAGE" envDefault:"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47"` CiDefaultNamespace string `env:"DEFAULT_NAMESPACE" envDefault:"devtron-ci"` - CiDefaultTimeout int64 `env:"DEFAULT_TIMEOUT" envDefault:"3600"` CiDefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" envDefault:"devtron-pro-ci-logs"` CiDefaultCdLogsBucketRegion string `env:"DEFAULT_CD_LOGS_BUCKET_REGION" envDefault:"us-east-2"` - CiLimitCpu string `env:"LIMIT_CI_CPU" envDefault:"0.5"` - CiLimitMem string `env:"LIMIT_CI_MEM" envDefault:"3G"` - CiReqCpu string `env:"REQ_CI_CPU" envDefault:"0.5"` - CiReqMem string `env:"REQ_CI_MEM" envDefault:"3G"` CiTaintKey string `env:"CI_NODE_TAINTS_KEY" envDefault:""` CiTaintValue string `env:"CI_NODE_TAINTS_VALUE" envDefault:""` CiNodeLabelSelector []string `env:"CI_NODE_LABEL_SELECTOR"` @@ -249,8 +246,6 @@ func (impl *CiCdConfig) GetDefaultNamespace() string { } func (impl *CiCdConfig) GetDefaultTimeout() int64 { switch impl.Type { - case CiConfigType: - return impl.CiDefaultTimeout case CdConfigType: return impl.CdDefaultTimeout default: @@ -281,8 +276,6 @@ func (impl *CiCdConfig) GetDefaultCdLogsBucketRegion() string { func (impl *CiCdConfig) GetLimitCpu() string { switch impl.Type { - case CiConfigType: - return impl.CiLimitCpu case CdConfigType: return impl.CdLimitCpu default: @@ -292,8 +285,6 @@ func (impl *CiCdConfig) GetLimitCpu() string { func (impl *CiCdConfig) GetLimitMem() string { switch impl.Type { - case CiConfigType: - return impl.CiLimitMem case CdConfigType: return impl.CdLimitMem default: @@ -303,8 +294,6 @@ func (impl *CiCdConfig) GetLimitMem() string { func (impl *CiCdConfig) GetReqCpu() string { switch impl.Type { - case CiConfigType: - return impl.CiReqCpu case CdConfigType: return impl.CdReqCpu default: @@ -314,8 +303,6 @@ func (impl *CiCdConfig) GetReqCpu() string { func (impl *CiCdConfig) GetReqMem() string { switch impl.Type { - case CiConfigType: - return impl.CiReqMem case CdConfigType: return impl.CdReqMem default: diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index 9a749bd9fb..5a7c0258c1 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/infraConfig" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" @@ -406,15 +407,15 @@ func (workflowRequest *WorkflowRequest) GetNodeConstraints(config *CiCdConfig) * } } -func (workflowRequest *WorkflowRequest) GetLimitReqCpuMem(config *CiCdConfig) v1.ResourceRequirements { +func (workflowRequest *WorkflowRequest) GetLimitReqCpuMem(config *CiCdConfig, infraConfigurations *infraConfig.InfraConfig) v1.ResourceRequirements { limitReqCpuMem := &bean.LimitReqCpuMem{} switch workflowRequest.Type { case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: limitReqCpuMem = &bean.LimitReqCpuMem{ - LimitCpu: config.CiLimitCpu, - LimitMem: config.CiLimitMem, - ReqCpu: config.CiReqCpu, - ReqMem: config.CiReqMem, + LimitCpu: infraConfigurations.GetCiLimitCpu(), + LimitMem: infraConfigurations.GetCiLimitMem(), + ReqCpu: infraConfigurations.GetCiReqCpu(), + ReqMem: infraConfigurations.GetCiReqMem(), } case bean.CD_WORKFLOW_PIPELINE_TYPE: limitReqCpuMem = &bean.LimitReqCpuMem{ @@ -447,7 +448,7 @@ func (workflowRequest *WorkflowRequest) getWorkflowImage() string { } } -func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdConfig, workflowJson []byte, workflowTemplate *bean.WorkflowTemplate, workflowConfigMaps []bean3.ConfigSecretMap, workflowSecrets []bean3.ConfigSecretMap) (v1.Container, error) { +func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdConfig, infraConfigurations *infraConfig.InfraConfig, workflowJson []byte, workflowTemplate *bean.WorkflowTemplate, workflowConfigMaps []bean3.ConfigSecretMap, workflowSecrets []bean3.ConfigSecretMap) (v1.Container, error) { privileged := true pvc := workflowRequest.getPVCForWorkflowRequest() containerEnvVariables := workflowRequest.getContainerEnvVariables(config, workflowJson) @@ -458,7 +459,7 @@ func (workflowRequest *WorkflowRequest) GetWorkflowMainContainer(config *CiCdCon SecurityContext: &v1.SecurityContext{ Privileged: &privileged, }, - Resources: workflowRequest.GetLimitReqCpuMem(config), + Resources: workflowRequest.GetLimitReqCpuMem(config, infraConfigurations), } if workflowRequest.Type == bean.CI_WORKFLOW_PIPELINE_TYPE || workflowRequest.Type == bean.JOB_WORKFLOW_PIPELINE_TYPE { workflowMainContainer.Ports = []v1.ContainerPort{{ diff --git a/pkg/resourceQualifiers/ResourceQualifiersMappingDto.go b/pkg/resourceQualifiers/ResourceQualifiersMappingDto.go index efc4c2b908..81846e1f6f 100644 --- a/pkg/resourceQualifiers/ResourceQualifiersMappingDto.go +++ b/pkg/resourceQualifiers/ResourceQualifiersMappingDto.go @@ -9,6 +9,7 @@ const ( Filter = 1 ImageDigest = 2 ImageDigestResourceId = -1 // for ImageDigest resource id will is constant unlike filter and variables + InfraProfile = 3 ) type QualifierMapping struct { @@ -23,7 +24,7 @@ type QualifierMapping struct { IdentifierValueString string `sql:"identifier_value_string"` ParentIdentifier int `sql:"parent_identifier"` CompositeKey string `sql:"-"` - //Data string `sql:"-"` - //VariableData *VariableData + // Data string `sql:"-"` + // VariableData *VariableData sql.AuditLog } diff --git a/scripts/sql/223_infra_configuration.down.sql b/scripts/sql/223_infra_configuration.down.sql new file mode 100644 index 0000000000..84d94bcd00 --- /dev/null +++ b/scripts/sql/223_infra_configuration.down.sql @@ -0,0 +1,6 @@ +DROP TABLE "public"."infra_profile_configuration"; +DROP INDEX idx_unique_profile_name; +DROP TABLE "public"."infra_profile"; +DROP SEQUENCE "public"."id_seq_infra_profile"; +DROP SEQUENCE "public"."id_seq_infra_profile_configuration"; + diff --git a/scripts/sql/223_infra_configuration.up.sql b/scripts/sql/223_infra_configuration.up.sql new file mode 100644 index 0000000000..2862671722 --- /dev/null +++ b/scripts/sql/223_infra_configuration.up.sql @@ -0,0 +1,36 @@ +CREATE SEQUENCE IF NOT EXISTS id_seq_infra_profile; +CREATE TABLE IF NOT EXISTS public.infra_profile +( + "id" int NOT NULL DEFAULT nextval('id_seq_infra_profile'::regclass), + "name" VARCHAR(50) NOT NULL, + "description" VARCHAR(300), + "active" bool NOT NULL, + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz NOT NULL, + "updated_by" int4 NOT NULL, + PRIMARY KEY ("id") + ); + +CREATE UNIQUE INDEX idx_unique_profile_name + ON infra_profile (name) + WHERE active = true; + +CREATE SEQUENCE IF NOT EXISTS id_seq_infra_profile_configuration; + +CREATE TABLE IF NOT EXISTS public.infra_profile_configuration +( + "id" int NOT NULL DEFAULT nextval('id_seq_infra_profile_configuration'::regclass), + "key" int NOT NULL, + "value" float NOT NULL, + "profile_id" int NOT NULL, + "unit" int NOT NULL, + "active" bool NOT NULL, + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz NOT NULL, + "updated_by" int4 NOT NULL, + PRIMARY KEY ("id"), + CONSTRAINT "infra_profile_configuration_profile_id_fkey" FOREIGN KEY ("profile_id") REFERENCES "public"."infra_profile" ("id") + ); + diff --git a/specs/buildInfraConfig/build-infra-config.yaml b/specs/buildInfraConfig/build-infra-config.yaml new file mode 100644 index 0000000000..4698196884 --- /dev/null +++ b/specs/buildInfraConfig/build-infra-config.yaml @@ -0,0 +1,160 @@ +openapi: 3.0.3 +info: + title: Infra Config + description: API SPEC for Infra Configurations + version: 1.0.0 +servers: + - url: 'https' +paths: +# send 404 responses if resource doesn't exist + /orchestrator/infra-config/profile/{name}: + get: + description: Get Infra Profile by name + responses: + "200": + description: gets the infra config profile by its name. + content: + application/json: + schema: + $ref: "#/components/schemas/ProfileResponse" + put: + description: Update Infra Profile + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/Profile' + responses: + "200": + description: creates a infra config profile. + content: + application/json: + schema: + $ref: "#/components/schemas/Profile" + + +components: + schemas: + Unit: + type: object + properties: + name: + type: string + description: Unit Name + example: "mi" + conversionFactor: + type: number + description: Conversion Factor to convert to base unit + example: 1 + + ConfigurationUnits: + type: object + properties: + name: + type: string + description: Configuration Units + units: + type: array + description: Configuration Units + items: + $ref: '#/components/schemas/Unit' + ProfileResponse: + type: object + properties: + configurationUnits: + type: array + description: Configuration Units + items: + $ref: '#/components/schemas/ConfigurationUnits' + defaultConfigurations: + type: array + description: Default Configurations + items: + $ref: '#/components/schemas/Configuration' + profile: + $ref: '#/components/schemas/Profile' + + + + + + + Configuration: + type: object + properties: + id: + type: integer + description: Property Id + example: 1 + key: + type: string + description: Property Name + required: true, + example: "cpu_limits" + value: + required: true, + type: string + description: Property Value + example: "0.5" + profileName: + type: string + description: Profile Name + example: "java" + unit: + type: string + description: Property Unit + example: "m" + active: + type: boolean + description: Property Active + example: true + + Profile: + type: object + properties: + id: + type: integer + description: Profile Id + example: 1 + name: + type: string + description: Profile Name + example: "java" + description: + type: string + description: Profile Description + example: "all java apps should have this infra profile" + type: + type: string + description: type of profile "eg:0,1,2" + example: DEFAULT,NORMAL,CUSTOM + configurations: + type: array + description: Profile Configurations + items: + $ref: '#/components/schemas/Configuration' + appCount: + readOnly: true + type: integer + description: Number of apps using this profile + example: 1 + createdAt: + required: false + type: string + description: Profile Created At + example: "2021-06-01T06:30:00.000Z" + updatedAt: + type: string + description: Profile Updated At + example: "2021-06-01T06:30:00.000Z" + createdBy: + type: integer + description: Profile Created By + example: 1 + updatedBy: + type: integer + description: Profile Updated By + example: 1 + + diff --git a/util/TypeUtils.go b/util/TypeUtils.go index b813ac7e25..36655a2648 100644 --- a/util/TypeUtils.go +++ b/util/TypeUtils.go @@ -1,6 +1,7 @@ package util import ( + "math" "strconv" "strings" ) @@ -56,3 +57,28 @@ func GetMapValuesPtr[T any](valueMap map[string]*T) []*T { } return values } + +func Transform[T any, K any](input []T, transform func(inp T) K) []K { + + res := make([]K, len(input)) + for i, _ := range input { + res[i] = transform(input[i]) + } + return res + +} + +func Contains[T any](input []T, check func(inp T) bool) bool { + for i, _ := range input { + if check(input[i]) { + return true + } + } + return false +} + +// TruncateFloat truncates a float64 value to n decimal points using the math package. +func TruncateFloat(value float64, decimals int) float64 { + pow10 := math.Pow10(decimals) + return math.Trunc(value*pow10) / pow10 +} diff --git a/wire_gen.go b/wire_gen.go index 7ceb27cd03..f1dac237b8 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -31,6 +31,7 @@ import ( client3 "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/api/helm-app/service" + infraConfig2 "github.com/devtron-labs/devtron/api/infraConfig" application3 "github.com/devtron-labs/devtron/api/k8s/application" capacity2 "github.com/devtron-labs/devtron/api/k8s/capacity" module2 "github.com/devtron-labs/devtron/api/module" @@ -142,6 +143,8 @@ import ( git2 "github.com/devtron-labs/devtron/pkg/git" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + "github.com/devtron-labs/devtron/pkg/infraConfig" + "github.com/devtron-labs/devtron/pkg/infraConfig/units" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" "github.com/devtron-labs/devtron/pkg/k8s/capacity" @@ -156,6 +159,7 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" repository9 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" repository10 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -438,7 +442,14 @@ func InitializeApp() (*App, error) { globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) argoWorkflowExecutorImpl := executors.NewArgoWorkflowExecutorImpl(sugaredLogger) systemWorkflowExecutorImpl := executors.NewSystemWorkflowExecutorImpl(sugaredLogger, k8sServiceImpl) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, appServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl) + infraConfigRepositoryImpl := infraConfig.NewInfraProfileRepositoryImpl(db) + unitsUnits := units.NewUnits() + infraConfigServiceImpl, err := infraConfig.NewInfraConfigServiceImpl(sugaredLogger, infraConfigRepositoryImpl, appServiceImpl, unitsUnits) + if err != nil { + return nil, err + } + infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraConfigServiceImpl) + workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, appServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) if err != nil { return nil, err } @@ -463,7 +474,7 @@ func InitializeApp() (*App, error) { customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciWorkflowRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, mergeUtil, ciPipelineRepositoryImpl, prePostCiScriptHistoryServiceImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -841,7 +852,9 @@ func InitializeApp() (*App, error) { return nil, err } proxyRouterImpl := proxy.NewProxyRouterImpl(sugaredLogger, proxyConfig, enforcerImpl) - muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl) + infraConfigRestHandlerImpl := infraConfig2.NewInfraConfigRestHandlerImpl(sugaredLogger, infraConfigServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, validate) + infraConfigRouterImpl := infraConfig2.NewInfraProfileRouterImpl(infraConfigRestHandlerImpl) + muxRouter := router.NewMuxRouter(sugaredLogger, environmentRouterImpl, clusterRouterImpl, webhookRouterImpl, userAuthRouterImpl, gitProviderRouterImpl, gitHostRouterImpl, dockerRegRouterImpl, notificationRouterImpl, teamRouterImpl, gitWebhookHandlerImpl, workflowStatusUpdateHandlerImpl, applicationStatusHandlerImpl, ciEventHandlerImpl, pubSubClientServiceImpl, userRouterImpl, chartRefRouterImpl, configMapRouterImpl, appStoreRouterImpl, chartRepositoryRouterImpl, releaseMetricsRouterImpl, deploymentGroupRouterImpl, batchOperationRouterImpl, chartGroupRouterImpl, imageScanRouterImpl, policyRouterImpl, gitOpsConfigRouterImpl, dashboardRouterImpl, attributesRouterImpl, userAttributesRouterImpl, commonRouterImpl, grafanaRouterImpl, ssoLoginRouterImpl, telemetryRouterImpl, telemetryEventClientImplExtended, bulkUpdateRouterImpl, webhookListenerRouterImpl, appRouterImpl, coreAppRouterImpl, helmAppRouterImpl, k8sApplicationRouterImpl, pProfRouterImpl, deploymentConfigRouterImpl, dashboardTelemetryRouterImpl, commonDeploymentRouterImpl, externalLinkRouterImpl, globalPluginRouterImpl, moduleRouterImpl, serverRouterImpl, apiTokenRouterImpl, cdApplicationStatusUpdateHandlerImpl, k8sCapacityRouterImpl, webhookHelmRouterImpl, globalCMCSRouterImpl, userTerminalAccessRouterImpl, jobRouterImpl, ciStatusUpdateCronImpl, resourceGroupingRouterImpl, rbacRoleRouterImpl, scopedVariableRouterImpl, ciTriggerCronImpl, proxyRouterImpl, infraConfigRouterImpl) loggingMiddlewareImpl := util4.NewLoggingMiddlewareImpl(userServiceImpl) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, pubSubClientServiceImpl, sessionManager, posthogClient, loggingMiddlewareImpl) return mainApp, nil