From 4ba7514e772b2aafe0cadcf51e2d8ddbb1f02fb6 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 19 Jan 2024 02:49:14 +0530 Subject: [PATCH 01/28] multiple 5xx fixes --- go.mod | 2 +- go.sum | 4 ++-- pkg/delete/DeleteServiceExtended.go | 10 ++++---- pkg/k8s/capacity/bean/bean.go | 23 ++++++++++--------- pkg/k8s/capacity/k8sCapacityService.go | 6 ++++- pkg/pipeline/CiService.go | 5 ++-- .../pubsub-lib/PubSubClientService.go | 18 +++++++-------- .../common-lib/utils/k8s/K8sUtil.go | 12 ++++++++-- .../devtron-labs/common-lib/utils/k8s/bean.go | 1 + vendor/modules.txt | 2 +- 10 files changed, 49 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index f81542d6e4..0169dfd73e 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.32 - github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5 + github.com/devtron-labs/common-lib v0.0.10-0.20240118204514-a0fc3a647a4a github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index 2c2cfe9666..9ee4793e63 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.32 h1:JAIJ0WqTXWj2nW7b8so9wunNICQn7O1Qpkk8INpatcs= github.com/devtron-labs/authenticator v0.4.32/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5 h1:+Nh2SMzAdgBr1tgdKAlF5cN0CvTPUj1V/sI5aRUrZnE= -github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5/go.mod h1:pBThgympEjsza6GShqNNGCPBFXNDx0DGMc7ID/VHTAw= +github.com/devtron-labs/common-lib v0.0.10-0.20240118204514-a0fc3a647a4a h1:jf5NzBtTKCcbHIDdA6ghKF76jyXQren08OMp4CTkKEs= +github.com/devtron-labs/common-lib v0.0.10-0.20240118204514-a0fc3a647a4a/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/pkg/delete/DeleteServiceExtended.go b/pkg/delete/DeleteServiceExtended.go index 9485e799a3..39ac917e43 100644 --- a/pkg/delete/DeleteServiceExtended.go +++ b/pkg/delete/DeleteServiceExtended.go @@ -5,6 +5,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/app" dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" repository2 "github.com/devtron-labs/devtron/pkg/appStore/deployment/repository" "github.com/devtron-labs/devtron/pkg/chartRepo" @@ -14,6 +15,7 @@ import ( "github.com/devtron-labs/devtron/pkg/team" "github.com/go-pg/pg" "go.uber.org/zap" + "net/http" ) type DeleteServiceExtendedImpl struct { @@ -61,7 +63,7 @@ func (impl DeleteServiceExtendedImpl) DeleteCluster(deleteRequest *cluster.Clust } if len(env) > 0 { impl.logger.Errorw("err in deleting cluster, found env in this cluster", "clusterName", deleteRequest.ClusterName, "err", err) - return fmt.Errorf(" Please delete all related environments before deleting this cluster") + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related environments before deleting this cluster"} } err = impl.clusterService.DeleteFromDb(deleteRequest, userId) if err != nil { @@ -89,13 +91,13 @@ func (impl DeleteServiceExtendedImpl) DeleteEnvironment(deleteRequest *cluster.E } if len(installedApps) > 0 && len(pipelines) > 0 { impl.logger.Errorw("err in deleting env, found cd pipelines and helm apps in this env", "envName", deleteRequest.Environment, "err", err) - return fmt.Errorf(" Please delete all related cd pipelines and helm apps before deleting this environment") + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related cd pipelines and helm apps before deleting this environment"} } else if len(installedApps) > 0 { impl.logger.Errorw("err in deleting env, found helm apps in this env", "envName", deleteRequest.Environment, "err", err) - return fmt.Errorf(" Please delete all related helm apps before deleting this environment") + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related helm apps before deleting this environment"} } else if len(pipelines) > 0 { impl.logger.Errorw("err in deleting env, found cd pipelines in this env", "envName", deleteRequest.Environment, "err", err) - return fmt.Errorf(" Please delete all related cd pipelines before deleting this environment") + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related cd pipelines before deleting this environment"} } err = impl.environmentService.Delete(deleteRequest, userId) diff --git a/pkg/k8s/capacity/bean/bean.go b/pkg/k8s/capacity/bean/bean.go index 95e35c8644..3d2ae29384 100644 --- a/pkg/k8s/capacity/bean/bean.go +++ b/pkg/k8s/capacity/bean/bean.go @@ -27,17 +27,18 @@ const NamespaceAll string = "" // below const set is used for pod filters const ( - daemonSetFatal = "DaemonSet-managed Pods (use --ignore-daemonsets to ignore)" - daemonSetWarning = "ignoring DaemonSet-managed Pods" - localStorageFatal = "Pods with local storage (use --delete-emptydir-data to override)" - localStorageWarning = "deleting Pods with local storage" - unmanagedFatal = "Pods declare no controller (use --force to override)" - unmanagedWarning = "deleting Pods that declare no controller" - AWSNodeGroupLabel = "alpha.eksctl.io/nodegroup-name" - AzureNodeGroupLabel = "kubernetes.azure.com/agentpool" - GcpNodeGroupLabel = "cloud.google.com/gke-nodepool" - KopsNodeGroupLabel = "kops.k8s.io/instancegroup" - AWSEKSNodeGroupLabel = "eks.amazonaws.com/nodegroup" + daemonSetFatal = "DaemonSet-managed Pods (use --ignore-daemonsets to ignore)" + daemonSetWarning = "ignoring DaemonSet-managed Pods" + localStorageFatal = "Pods with local storage (use --delete-emptydir-data to override)" + localStorageWarning = "deleting Pods with local storage" + unmanagedFatal = "Pods declare no controller (use --force to override)" + unmanagedWarning = "deleting Pods that declare no controller" + AWSNodeGroupLabel = "alpha.eksctl.io/nodegroup-name" + AzureNodeGroupLabel = "kubernetes.azure.com/agentpool" + GcpNodeGroupLabel = "cloud.google.com/gke-nodepool" + KopsNodeGroupLabel = "kops.k8s.io/instancegroup" + AWSEKSNodeGroupLabel = "eks.amazonaws.com/nodegroup" + DaemonSetPodDeleteError = "cannot delete DaemonSet-managed Pods" ) // TODO: add any new nodeGrouplabel in this array diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 97da7376b0..703e66fcd6 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -3,6 +3,7 @@ package capacity import ( "context" "fmt" + "github.com/devtron-labs/common-lib/utils" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" @@ -638,7 +639,10 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean } request.NodeDrainHelper.K8sClientSet = k8sClientSet err = impl.deleteOrEvictPods(request.Name, request.NodeDrainHelper) - if err != nil { + if err != nil && strings.Contains(err.Error(), bean.DaemonSetPodDeleteError) { + impl.logger.Errorw("daemonSet-managed pods can't be deleted", "err", err, "nodeName", request.Name) + return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: "cannot delete DaemonSet-managed Pods"} + } else if err != nil { impl.logger.Errorw("error in deleting/evicting pods", "err", err, "nodeName", request.Name) return respMessage, err } diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 5387843ad0..97ea56b758 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -39,6 +39,7 @@ import ( repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" util3 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" + "net/http" "path/filepath" "strconv" "strings" @@ -203,9 +204,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot if len(preCiSteps) == 0 && isJob { - return 0, &util.ApiError{ - UserMessage: "No tasks are configured in this job pipeline", - } + return 0, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: "No tasks are configured in this job pipeline"} } savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfig, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) if err != nil { diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go index dec37692bc..8d8e80d0f1 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go @@ -132,37 +132,37 @@ func (impl PubSubClientServiceImpl) Subscribe(topic string, callback func(msg *m impl.Logger.Fatalw("error while subscribing to nats ", "stream", streamName, "topic", topic, "error", err) return err } - go impl.startListeningForEvents(processingBatchSize, channel, callback, topic) + go impl.startListeningForEvents(processingBatchSize, channel, callback) impl.Logger.Infow("Successfully subscribed with Nats", "stream", streamName, "topic", topic, "queue", queueName, "consumer", consumerName) return nil } -func (impl PubSubClientServiceImpl) startListeningForEvents(processingBatchSize int, channel chan *nats.Msg, callback func(msg *model.PubSubMsg), topic string) { +func (impl PubSubClientServiceImpl) startListeningForEvents(processingBatchSize int, channel chan *nats.Msg, callback func(msg *model.PubSubMsg)) { wg := new(sync.WaitGroup) for index := 0; index < processingBatchSize; index++ { wg.Add(1) - go impl.processMessages(wg, channel, callback, topic) + go impl.processMessages(wg, channel, callback) } wg.Wait() impl.Logger.Warn("msgs received Done from Nats side, going to end listening!!") } -func (impl PubSubClientServiceImpl) processMessages(wg *sync.WaitGroup, channel chan *nats.Msg, callback func(msg *model.PubSubMsg), topic string) { +func (impl PubSubClientServiceImpl) processMessages(wg *sync.WaitGroup, channel chan *nats.Msg, callback func(msg *model.PubSubMsg)) { defer wg.Done() for msg := range channel { - impl.processMsg(msg, callback, topic) + impl.processMsg(msg, callback) } } // TODO need to extend msg ack depending upon response from callback like error scenario -func (impl PubSubClientServiceImpl) processMsg(msg *nats.Msg, callback func(msg *model.PubSubMsg), topic string) { +func (impl PubSubClientServiceImpl) processMsg(msg *nats.Msg, callback func(msg *model.PubSubMsg)) { t1 := time.Now() - metrics.IncConsumingCount(topic) - defer metrics.IncConsumptionCount(topic) + metrics.IncConsumingCount(msg.Subject) + defer metrics.IncConsumptionCount(msg.Subject) defer func() { // wrapping this function in defer as directly calling Observe() will run immediately - metrics.NatsEventConsumptionTime.WithLabelValues(topic).Observe(float64(time.Since(t1).Milliseconds())) + metrics.NatsEventConsumptionTime.WithLabelValues(msg.Subject).Observe(float64(time.Since(t1).Milliseconds())) }() impl.TryCatchCallBack(msg, callback) } diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go index b074fd8781..d122409f55 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go @@ -905,7 +905,10 @@ func (impl K8sUtil) GetCoreV1ClientByRestConfig(restConfig *rest.Config) (*v12.C func (impl K8sUtil) GetNodesList(ctx context.Context, k8sClientSet *kubernetes.Clientset) (*v1.NodeList, error) { nodeList, err := k8sClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { + if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { impl.logger.Errorw("error in getting node list", "err", err) return nil, err } @@ -1286,7 +1289,10 @@ func (impl K8sUtil) GetApiResources(restConfig *rest.Config, includeOnlyVerb str } apiResourcesListFromK8s, err := discoveryClient.ServerPreferredResources() - if err != nil { + if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { //takes care when K8s is unable to handle the request for some resources Isk8sApiError := strings.Contains(err.Error(), "unable to retrieve the complete list of server APIs") switch Isk8sApiError { @@ -1428,6 +1434,7 @@ func (impl *K8sUtil) DeleteResource(ctx context.Context, restConfig *rest.Config if len(namespace) > 0 && namespaced { obj, err = resourceIf.Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { + err = &utils.ApiError{Code: "404", HttpStatusCode: 404, UserMessage: "error on getting resource"} impl.logger.Errorw("error in getting resource", "err", err, "resource", name, "namespace", namespace) return nil, err } @@ -1435,6 +1442,7 @@ func (impl *K8sUtil) DeleteResource(ctx context.Context, restConfig *rest.Config } else { obj, err = resourceIf.Get(ctx, name, metav1.GetOptions{}) if err != nil { + err = &utils.ApiError{Code: "404", HttpStatusCode: 404, UserMessage: "error on getting resource"} impl.logger.Errorw("error in getting resource", "err", err, "resource", name, "namespace", namespace) return nil, err } diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index 0bcd729a91..47d47118f7 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -24,6 +24,7 @@ const ( DEVTRON_APP_LABEL_KEY = "app" DEVTRON_APP_LABEL_VALUE1 = "devtron" DEVTRON_APP_LABEL_VALUE2 = "orchestrator" + DnsLookupNoSuchHostError = "no such host" ) type ClusterResourceListMap struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 39cad08c31..d186886867 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -354,7 +354,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5 +# github.com/devtron-labs/common-lib v0.0.10-0.20240118204514-a0fc3a647a4a ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/pubsub-lib From 31d1657035d8ceabbcabeed10661ceab2b4183eb Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 11:42:46 +0530 Subject: [PATCH 02/28] main merge --- go.sum | 8 +- .../pubsub-lib/PubSubClientService.go | 157 ++++++++++++------ .../common-lib/utils/k8s/K8sUtil.go | 10 +- .../devtron-labs/common-lib/utils/k8s/bean.go | 1 - vendor/modules.txt | 6 +- 5 files changed, 116 insertions(+), 66 deletions(-) diff --git a/go.sum b/go.sum index 2c2cfe9666..fd9dff6c69 100644 --- a/go.sum +++ b/go.sum @@ -221,10 +221,10 @@ github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsP github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADGYw5LqMnHqSkyIELsHCGF6PkrmM31V8rF7o= github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= -github.com/devtron-labs/authenticator v0.4.32 h1:JAIJ0WqTXWj2nW7b8so9wunNICQn7O1Qpkk8INpatcs= -github.com/devtron-labs/authenticator v0.4.32/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5 h1:+Nh2SMzAdgBr1tgdKAlF5cN0CvTPUj1V/sI5aRUrZnE= -github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5/go.mod h1:pBThgympEjsza6GShqNNGCPBFXNDx0DGMc7ID/VHTAw= +github.com/devtron-labs/authenticator v0.4.33 h1:FpAV3ZgFluaRFcMwPpwxr/mwSipJ16XRvgABq3BzP5Y= +github.com/devtron-labs/authenticator v0.4.33/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= +github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e h1:tPD2kWowGhgH3g5TRZMKgrvuNAgpFE6E6sxZaTAHNSU= +github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go index dec37692bc..57a84ef932 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/PubSubClientService.go @@ -14,9 +14,18 @@ import ( "time" ) +const NATS_MSG_LOG_PREFIX = "NATS_LOG" + +type ValidateMsg func(msg model.PubSubMsg) bool + +// LoggerFunc is used to log the message before passing to callback function. +// it expects logg message and key value pairs to be returned. +// if keysAndValues is empty, it will log whole model.PubSubMsg +type LoggerFunc func(msg model.PubSubMsg) (logMsg string, keysAndValues []interface{}) + type PubSubClientService interface { Publish(topic string, msg string) error - Subscribe(topic string, callback func(msg *model.PubSubMsg)) error + Subscribe(topic string, callback func(msg *model.PubSubMsg), loggerFunc LoggerFunc, validations ...ValidateMsg) error } type PubSubClientServiceImpl struct { @@ -77,7 +86,12 @@ func (impl PubSubClientServiceImpl) Publish(topic string, msg string) error { return nil } -func (impl PubSubClientServiceImpl) Subscribe(topic string, callback func(msg *model.PubSubMsg)) error { +// Subscribe method is used to subscribe to the given topic(+required), +// this creates blocking process to continuously fetch messages from nats server published on this topic. +// invokes callback(+required) func for each message received. +// loggerFunc(+optional) is invoked before passing the message to the callback function. +// validations(+optional) methods were called before passing the message to the callback func. +func (impl PubSubClientServiceImpl) Subscribe(topic string, callback func(msg *model.PubSubMsg), loggerFunc LoggerFunc, validations ...ValidateMsg) error { impl.Logger.Infow("Subscribed to pubsub client", "topic", topic) natsTopic := GetNatsTopic(topic) streamName := natsTopic.streamName @@ -91,80 +105,61 @@ func (impl PubSubClientServiceImpl) Subscribe(topic string, callback func(msg *m if streamConfig.Retention == nats.WorkQueuePolicy { deliveryOption = nats.DeliverAll() } - processingBatchSize := NatsConsumerWiseConfigMapping[consumerName].NatsMsgProcessingBatchSize - msgBufferSize := NatsConsumerWiseConfigMapping[consumerName].NatsMsgBufferSize - - // Converting provided ack wait (int) into duration for comparing with nats-server config - ackWait := time.Duration(NatsConsumerWiseConfigMapping[consumerName].AckWaitInSecs) * time.Second - - // Get the current Consumer config from NATS-server - info, err := natsClient.JetStrCtxt.ConsumerInfo(streamName, consumerName) - - if err != nil { - impl.Logger.Errorw("unable to retrieve consumer info from NATS-server", - "stream", streamName, - "consumer", consumerName, - "err", err) - - } else { - // Update NATS Consumer config if new changes detected - // Currently only checking for AckWait, but can be done for other editable properties as well - if ackWait > 0 && info.Config.AckWait != ackWait { + consumerConfig := NatsConsumerWiseConfigMapping[consumerName] + processingBatchSize := consumerConfig.NatsMsgProcessingBatchSize + msgBufferSize := consumerConfig.GetNatsMsgBufferSize() - updatedConfig := info.Config - updatedConfig.AckWait = ackWait - - _, err = natsClient.JetStrCtxt.UpdateConsumer(streamName, &updatedConfig) + // Converting provided ack wait (int) into duration for comparing with nats-server config + ackWait := time.Duration(consumerConfig.AckWaitInSecs) * time.Second - if err != nil { - impl.Logger.Errorw("failed to update Consumer config", - "received consumer config", info.Config, - "err", err) - } - } - } + // Update consumer config if new changes detected + impl.updateConsumer(natsClient, streamName, consumerName, &consumerConfig) channel := make(chan *nats.Msg, msgBufferSize) - _, err = natsClient.JetStrCtxt.ChanQueueSubscribe(topic, queueName, channel, nats.Durable(consumerName), deliveryOption, nats.ManualAck(), + _, err := natsClient.JetStrCtxt.ChanQueueSubscribe(topic, queueName, channel, + nats.Durable(consumerName), + deliveryOption, + nats.ManualAck(), + nats.AckWait(ackWait), // if ackWait is 0 , nats sets this option to 30secs by default nats.BindStream(streamName)) if err != nil { impl.Logger.Fatalw("error while subscribing to nats ", "stream", streamName, "topic", topic, "error", err) return err } - go impl.startListeningForEvents(processingBatchSize, channel, callback, topic) + go impl.startListeningForEvents(processingBatchSize, channel, callback, loggerFunc, validations...) impl.Logger.Infow("Successfully subscribed with Nats", "stream", streamName, "topic", topic, "queue", queueName, "consumer", consumerName) return nil } -func (impl PubSubClientServiceImpl) startListeningForEvents(processingBatchSize int, channel chan *nats.Msg, callback func(msg *model.PubSubMsg), topic string) { +func (impl PubSubClientServiceImpl) startListeningForEvents(processingBatchSize int, channel chan *nats.Msg, callback func(msg *model.PubSubMsg), loggerFunc LoggerFunc, validations ...ValidateMsg) { wg := new(sync.WaitGroup) for index := 0; index < processingBatchSize; index++ { wg.Add(1) - go impl.processMessages(wg, channel, callback, topic) + go impl.processMessages(wg, channel, callback, loggerFunc, validations...) } wg.Wait() impl.Logger.Warn("msgs received Done from Nats side, going to end listening!!") } -func (impl PubSubClientServiceImpl) processMessages(wg *sync.WaitGroup, channel chan *nats.Msg, callback func(msg *model.PubSubMsg), topic string) { +func (impl PubSubClientServiceImpl) processMessages(wg *sync.WaitGroup, channel chan *nats.Msg, callback func(msg *model.PubSubMsg), loggerFunc LoggerFunc, validations ...ValidateMsg) { defer wg.Done() for msg := range channel { - impl.processMsg(msg, callback, topic) + impl.processMsg(msg, callback, loggerFunc, validations...) } } // TODO need to extend msg ack depending upon response from callback like error scenario -func (impl PubSubClientServiceImpl) processMsg(msg *nats.Msg, callback func(msg *model.PubSubMsg), topic string) { +func (impl PubSubClientServiceImpl) processMsg(msg *nats.Msg, callback func(msg *model.PubSubMsg), loggerFunc LoggerFunc, validations ...ValidateMsg) { t1 := time.Now() - metrics.IncConsumingCount(topic) - defer metrics.IncConsumptionCount(topic) + metrics.IncConsumingCount(msg.Subject) + defer metrics.IncConsumptionCount(msg.Subject) defer func() { // wrapping this function in defer as directly calling Observe() will run immediately - metrics.NatsEventConsumptionTime.WithLabelValues(topic).Observe(float64(time.Since(t1).Milliseconds())) + metrics.NatsEventConsumptionTime.WithLabelValues(msg.Subject).Observe(float64(time.Since(t1).Milliseconds())) }() - impl.TryCatchCallBack(msg, callback) + impl.TryCatchCallBack(msg, callback, loggerFunc, validations...) } func (impl PubSubClientServiceImpl) publishPanicError(msg *nats.Msg, panicErr error) (err error) { @@ -190,14 +185,36 @@ func (impl PubSubClientServiceImpl) publishPanicError(msg *nats.Msg, panicErr er } // TryCatchCallBack is a fail-safe method to use callback function -func (impl PubSubClientServiceImpl) TryCatchCallBack(msg *nats.Msg, callback func(msg *model.PubSubMsg)) { - subMsg := &model.PubSubMsg{Data: string(msg.Data)} +func (impl PubSubClientServiceImpl) TryCatchCallBack(msg *nats.Msg, callback func(msg *model.PubSubMsg), loggerFunc LoggerFunc, validations ...ValidateMsg) { + var msgDeliveryCount uint64 = 0 + if metadata, err := msg.Metadata(); err == nil { + msgDeliveryCount = metadata.NumDelivered + } + natsMsgId := msg.Header.Get(model.NatsMsgId) + subMsg := &model.PubSubMsg{Data: string(msg.Data), MsgDeliverCount: msgDeliveryCount, MsgId: natsMsgId} + + // call loggersFunc + impl.Log(loggerFunc, msg.Subject, *subMsg) + + // run validations + for _, validation := range validations { + if !validation(*subMsg) { + impl.Logger.Warnw("nats: message validation failed, not processing the message...", "subject", msg.Subject, "msg", string(msg.Data)) + return + } + } defer func() { // Acknowledge the message delivery err := msg.Ack() if err != nil { impl.Logger.Errorw("nats: unable to acknowledge the message", "subject", msg.Subject, "msg", string(msg.Data)) } + + // publish metrics for msg delivery count if msgDeliveryCount > 1 + if msgDeliveryCount > 1 { + metrics.NatsEventDeliveryCount.WithLabelValues(msg.Subject, natsMsgId).Observe(float64(msgDeliveryCount)) + } + // Panic recovery handling if panicInfo := recover(); panicInfo != nil { impl.Logger.Warnw("nats: found panic error", "subject", msg.Subject, "payload", string(msg.Data), "logs", string(debug.Stack())) @@ -214,12 +231,6 @@ func (impl PubSubClientServiceImpl) TryCatchCallBack(msg *nats.Msg, callback fun callback(subMsg) } -func (impl PubSubClientServiceImpl) printTimeDiff(t0 time.Time, msg *nats.Msg, timeLimitInMillSecs int64) { - t1 := time.Since(t0) - if t1.Milliseconds() > timeLimitInMillSecs { - impl.Logger.Debugw("time took to process msg: ", msg, "time :", t1) - } -} func (impl PubSubClientServiceImpl) getStreamConfig(streamName string) *nats.StreamConfig { configJson := NatsStreamWiseConfigMapping[streamName].StreamConfig streamCfg := &nats.StreamConfig{} @@ -235,3 +246,47 @@ func (impl PubSubClientServiceImpl) getStreamConfig(streamName string) *nats.Str return streamCfg } + +// Updates NATS Consumer config if new changes detected +// if consumer didn't exist, this will just return +func (impl PubSubClientServiceImpl) updateConsumer(natsClient *NatsClient, streamName string, consumerName string, overrideConfig *NatsConsumerConfig) { + + // Get the current Consumer config from NATS-server + info, err := natsClient.JetStrCtxt.ConsumerInfo(streamName, consumerName) + if err != nil { + impl.Logger.Errorw("unable to retrieve consumer info from NATS-server", "stream", streamName, "consumer", consumerName, "err", err) + return + } + + existingConfig := info.Config + updatesDetected := false + + // Currently only checking for AckWait,MaxAckPending but can be done for other editable properties as well + if ackWaitOverride := time.Duration(overrideConfig.AckWaitInSecs) * time.Second; ackWaitOverride > 0 && existingConfig.AckWait != ackWaitOverride { + existingConfig.AckWait = ackWaitOverride + updatesDetected = true + } + + if messageBufferSize := overrideConfig.GetNatsMsgBufferSize(); messageBufferSize > 0 && existingConfig.MaxAckPending != messageBufferSize { + existingConfig.MaxAckPending = messageBufferSize + updatesDetected = true + } + + if updatesDetected { + _, err = natsClient.JetStrCtxt.UpdateConsumer(streamName, &existingConfig) + if err != nil { + impl.Logger.Errorw("failed to update Consumer config", "received consumer config", info.Config, "err", err) + } + } + return +} + +func (impl PubSubClientServiceImpl) Log(loggerFunc LoggerFunc, topic string, subMsg model.PubSubMsg) { + logMsg, metaSlice := loggerFunc(subMsg) + logMsg = fmt.Sprintf("%s:%s", NATS_MSG_LOG_PREFIX, logMsg) + if len(metaSlice) == 0 { + metaSlice = []interface{}{"msgId", subMsg.MsgId, "msg", subMsg.Data} + } + metaSlice = append(metaSlice, "topic", topic) + impl.Logger.Infow(logMsg, metaSlice...) +} diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go index ea6a537a1b..798aeb92e6 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go @@ -943,10 +943,7 @@ func (impl K8sServiceImpl) GetCoreV1ClientByRestConfig(restConfig *rest.Config) func (impl K8sServiceImpl) GetNodesList(ctx context.Context, k8sClientSet *kubernetes.Clientset) (*v1.NodeList, error) { nodeList, err := k8sClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { impl.logger.Errorw("error in getting node list", "err", err) return nil, err } @@ -1327,10 +1324,7 @@ func (impl K8sServiceImpl) GetApiResources(restConfig *rest.Config, includeOnlyV } apiResourcesListFromK8s, err := discoveryClient.ServerPreferredResources() - if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { //takes care when K8s is unable to handle the request for some resources Isk8sApiError := strings.Contains(err.Error(), "unable to retrieve the complete list of server APIs") switch Isk8sApiError { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index df22b3337b..f27ce9676f 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -24,7 +24,6 @@ const ( DEVTRON_APP_LABEL_KEY = "app" DEVTRON_APP_LABEL_VALUE1 = "devtron" DEVTRON_APP_LABEL_VALUE2 = "orchestrator" - DnsLookupNoSuchHostError = "no such host" ) type ClusterConfig struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 39cad08c31..400bb905e9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -346,7 +346,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.32 +# github.com/devtron-labs/authenticator v0.4.33 ## explicit; go 1.16 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -354,9 +354,11 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.9-0.20231226070212-c47f7a07ebf5 +# github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage +github.com/devtron-labs/common-lib/constants +github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib github.com/devtron-labs/common-lib/pubsub-lib/metrics github.com/devtron-labs/common-lib/pubsub-lib/model From f71ae9976c0bff0bec0ed9b81749d90e0fa6de3d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 12:03:15 +0530 Subject: [PATCH 03/28] DeleteCiPipeline 5xx handled --- pkg/pipeline/BuildPipelineConfigService.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 511d302517..241e8d23bc 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -40,6 +40,7 @@ import ( "github.com/juju/errors" "go.opentelemetry.io/otel" "go.uber.org/zap" + "net/http" "strconv" "strings" "time" @@ -1995,6 +1996,8 @@ func (impl *CiPipelineConfigServiceImpl) DeleteCiPipeline(request *bean.CiPatchR } if len(workflowMapping) > 0 { return nil, &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, + Code: "200", InternalMessage: "Please delete deployment pipelines for this workflow first and try again.", UserDetailMessage: fmt.Sprintf("Please delete deployment pipelines for this workflow first and try again."), UserMessage: fmt.Sprintf("Please delete deployment pipelines for this workflow first and try again.")} From 886b02bac17bb7091c994bbdb14e0077aafeeeee Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 14:42:17 +0530 Subject: [PATCH 04/28] common lib version change with my changes --- go.mod | 2 +- go.sum | 4 +-- .../common-lib/utils/k8s/K8sUtil.go | 27 +++++++++++++++++-- .../devtron-labs/common-lib/utils/k8s/bean.go | 1 + vendor/modules.txt | 2 +- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d59b4fcb9a..2209b17357 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.33 - github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e + github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index fd9dff6c69..28e5c67034 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.33 h1:FpAV3ZgFluaRFcMwPpwxr/mwSipJ16XRvgABq3BzP5Y= github.com/devtron-labs/authenticator v0.4.33/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e h1:tPD2kWowGhgH3g5TRZMKgrvuNAgpFE6E6sxZaTAHNSU= -github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= +github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 h1:9LJtAFJ+4t5TybVhrLE3eLVGQvn6whX84ChinaiBqTI= +github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go index 798aeb92e6..743a96d29d 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go @@ -118,6 +118,23 @@ type K8sService interface { GetClientForInCluster() (*v12.CoreV1Client, error) GetCoreV1Client(clusterConfig *ClusterConfig) (*v12.CoreV1Client, error) GetRestConfigByCluster(clusterConfig *ClusterConfig) (*restclient.Config, error) + GetResource(ctx context.Context, namespace string, name string, gvk schema.GroupVersionKind, restConfig *rest.Config) (*ManifestResponse, error) + UpdateResource(ctx context.Context, restConfig *rest.Config, gvk schema.GroupVersionKind, namespace string, k8sRequestPatch string) (*ManifestResponse, error) + DeleteResource(ctx context.Context, restConfig *rest.Config, gvk schema.GroupVersionKind, namespace string, name string, forceDelete bool) (*ManifestResponse, error) + GetPodListByLabel(namespace, label string, clientSet *kubernetes.Clientset) ([]v1.Pod, error) + ExtractK8sServerMajorAndMinorVersion(k8sServerVersion *version.Info) (int, int, error) + GetK8sServerVersion(clientSet *kubernetes.Clientset) (*version.Info, error) + DecodeGroupKindversion(data string) (*schema.GroupVersionKind, error) + GetApiResources(restConfig *rest.Config, includeOnlyVerb string) ([]*K8sApiResource, error) + CreateResources(ctx context.Context, restConfig *rest.Config, manifest string, gvk schema.GroupVersionKind, namespace string) (*ManifestResponse, error) + PatchResourceRequest(ctx context.Context, restConfig *rest.Config, pt types.PatchType, manifest string, name string, namespace string, gvk schema.GroupVersionKind) (*ManifestResponse, error) + GetResourceList(ctx context.Context, restConfig *rest.Config, gvk schema.GroupVersionKind, namespace string) (*ResourceListResponse, bool, error) + GetResourceIfWithAcceptHeader(restConfig *rest.Config, groupVersionKind schema.GroupVersionKind) (resourceIf dynamic.NamespaceableResourceInterface, namespaced bool, err error) + GetPodLogs(ctx context.Context, restConfig *rest.Config, name string, namespace string, sinceTime *metav1.Time, tailLines int, follow bool, containerName string, isPrevContainerLogsEnabled bool) (io.ReadCloser, error) + ListEvents(restConfig *rest.Config, namespace string, groupVersionKind schema.GroupVersionKind, ctx context.Context, name string) (*v1.EventList, error) + GetResourceIf(restConfig *rest.Config, groupVersionKind schema.GroupVersionKind) (resourceIf dynamic.NamespaceableResourceInterface, namespaced bool, err error) + FetchConnectionStatusForCluster(k8sClientSet *kubernetes.Clientset) error + CreateK8sClientSet(restConfig *rest.Config) (*kubernetes.Clientset, error) } func NewK8sUtil(logger *zap.SugaredLogger, runTimeConfig *client.RuntimeConfig) *K8sServiceImpl { @@ -943,7 +960,10 @@ func (impl K8sServiceImpl) GetCoreV1ClientByRestConfig(restConfig *rest.Config) func (impl K8sServiceImpl) GetNodesList(ctx context.Context, k8sClientSet *kubernetes.Clientset) (*v1.NodeList, error) { nodeList, err := k8sClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { + if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { impl.logger.Errorw("error in getting node list", "err", err) return nil, err } @@ -1324,7 +1344,10 @@ func (impl K8sServiceImpl) GetApiResources(restConfig *rest.Config, includeOnlyV } apiResourcesListFromK8s, err := discoveryClient.ServerPreferredResources() - if err != nil { + if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { //takes care when K8s is unable to handle the request for some resources Isk8sApiError := strings.Contains(err.Error(), "unable to retrieve the complete list of server APIs") switch Isk8sApiError { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index f27ce9676f..df22b3337b 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -24,6 +24,7 @@ const ( DEVTRON_APP_LABEL_KEY = "app" DEVTRON_APP_LABEL_VALUE1 = "devtron" DEVTRON_APP_LABEL_VALUE2 = "orchestrator" + DnsLookupNoSuchHostError = "no such host" ) type ClusterConfig struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 400bb905e9..184b6490e0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -354,7 +354,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e +# github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/constants From 3c77ae7a72d4c7ca3634db855d6f74e927ac5646 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 15:26:32 +0530 Subject: [PATCH 05/28] cluster delete 5xx fix --- api/cluster/ClusterRestHandler.go | 10 +++++++++- internal/util/ErrorUtil.go | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api/cluster/ClusterRestHandler.go b/api/cluster/ClusterRestHandler.go index 6eca541156..be51b05037 100644 --- a/api/cluster/ClusterRestHandler.go +++ b/api/cluster/ClusterRestHandler.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "errors" + "github.com/devtron-labs/common-lib/utils" "net/http" "strconv" "time" @@ -611,8 +612,15 @@ func (impl ClusterRestHandlerImpl) DeleteCluster(w http.ResponseWriter, r *http. //RBAC enforcer Ends err = impl.deleteService.DeleteCluster(&bean, userId) if err != nil { + errCode := http.StatusInternalServerError + if apiErr, ok := err.(*utils.ApiError); ok { + switch errCode { + case http.StatusBadRequest: + errCode = apiErr.HttpStatusCode + } + } impl.logger.Errorw("error in deleting cluster", "err", err, "id", bean.Id, "name", bean.ClusterName) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + common.WriteJsonResp(w, err, nil, errCode) return } common.WriteJsonResp(w, err, CLUSTER_DELETE_SUCCESS_RESP, http.StatusOK) diff --git a/internal/util/ErrorUtil.go b/internal/util/ErrorUtil.go index 52df683fc6..c01aef8d03 100644 --- a/internal/util/ErrorUtil.go +++ b/internal/util/ErrorUtil.go @@ -36,6 +36,10 @@ func (e *ApiError) Error() string { return e.InternalMessage } +func (e *ApiError) StatusCode() int { + return e.HttpStatusCode +} + // default internal will be set func (e *ApiError) ErrorfInternal(format string, a ...interface{}) error { return &ApiError{InternalMessage: fmt.Sprintf(format, a...)} From 8137dfc19612b60601f2a53439802f16cda444f3 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 17:29:10 +0530 Subject: [PATCH 06/28] revert some changes --- api/cluster/ClusterRestHandler.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/api/cluster/ClusterRestHandler.go b/api/cluster/ClusterRestHandler.go index be51b05037..6eca541156 100644 --- a/api/cluster/ClusterRestHandler.go +++ b/api/cluster/ClusterRestHandler.go @@ -21,7 +21,6 @@ import ( "context" "encoding/json" "errors" - "github.com/devtron-labs/common-lib/utils" "net/http" "strconv" "time" @@ -612,15 +611,8 @@ func (impl ClusterRestHandlerImpl) DeleteCluster(w http.ResponseWriter, r *http. //RBAC enforcer Ends err = impl.deleteService.DeleteCluster(&bean, userId) if err != nil { - errCode := http.StatusInternalServerError - if apiErr, ok := err.(*utils.ApiError); ok { - switch errCode { - case http.StatusBadRequest: - errCode = apiErr.HttpStatusCode - } - } impl.logger.Errorw("error in deleting cluster", "err", err, "id", bean.Id, "name", bean.ClusterName) - common.WriteJsonResp(w, err, nil, errCode) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } common.WriteJsonResp(w, err, CLUSTER_DELETE_SUCCESS_RESP, http.StatusOK) From 78ed8248a81debb3e976ff835660ded04082386e Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 18:02:35 +0530 Subject: [PATCH 07/28] DrainNode 5xx handling at rest handler --- api/k8s/capacity/k8sCapacityRestHandler.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/k8s/capacity/k8sCapacityRestHandler.go b/api/k8s/capacity/k8sCapacityRestHandler.go index 3d363d1b22..f854af79e2 100644 --- a/api/k8s/capacity/k8sCapacityRestHandler.go +++ b/api/k8s/capacity/k8sCapacityRestHandler.go @@ -3,6 +3,7 @@ package capacity import ( "encoding/json" "errors" + "github.com/devtron-labs/common-lib/utils" "net/http" "strconv" @@ -369,8 +370,15 @@ func (handler *K8sCapacityRestHandlerImpl) DrainNode(w http.ResponseWriter, r *h } resp, err := handler.k8sCapacityService.DrainNode(r.Context(), &nodeDrainReq) if err != nil { + errCode := http.StatusInternalServerError + if apiErr, ok := err.(*utils.ApiError); ok { + switch errCode { + case http.StatusBadRequest: + errCode = apiErr.HttpStatusCode + } + } handler.logger.Errorw("error in draining node", "err", err, "req", nodeDrainReq) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + common.WriteJsonResp(w, err, nil, errCode) return } common.WriteJsonResp(w, nil, resp, http.StatusOK) From 5f020b0201d7790ad669fc41450e0e416c0bd790 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 18:28:49 +0530 Subject: [PATCH 08/28] DrainNode 5xx handling at when getting node name --- pkg/k8s/capacity/k8sCapacityService.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 5250d11c0c..ea760ea617 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" + bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" @@ -625,7 +626,10 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean } //get node node, err := impl.K8sUtil.GetNodeByName(context.Background(), k8sClientSet, request.Name) - if err != nil { + if err != nil && strings.Contains(err.Error(), bean2.NOT_FOUND) { + impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) + return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + } else if err != nil { impl.logger.Errorw("error in getting node", "err", err) return respMessage, err } From 4e2f60c55df7aa1441db1f55c4d7f6cf53de68a3 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 18:45:16 +0530 Subject: [PATCH 09/28] fix --- api/k8s/capacity/k8sCapacityRestHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/k8s/capacity/k8sCapacityRestHandler.go b/api/k8s/capacity/k8sCapacityRestHandler.go index f854af79e2..b670e0a6ed 100644 --- a/api/k8s/capacity/k8sCapacityRestHandler.go +++ b/api/k8s/capacity/k8sCapacityRestHandler.go @@ -373,7 +373,7 @@ func (handler *K8sCapacityRestHandlerImpl) DrainNode(w http.ResponseWriter, r *h errCode := http.StatusInternalServerError if apiErr, ok := err.(*utils.ApiError); ok { switch errCode { - case http.StatusBadRequest: + case http.StatusNotFound: errCode = apiErr.HttpStatusCode } } From f1f924a78a3751495d7d6ca5feecea5e5decc3ba Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 19:01:16 +0530 Subject: [PATCH 10/28] revert --- pkg/k8s/capacity/k8sCapacityService.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index ea760ea617..5250d11c0c 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" @@ -626,10 +625,7 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean } //get node node, err := impl.K8sUtil.GetNodeByName(context.Background(), k8sClientSet, request.Name) - if err != nil && strings.Contains(err.Error(), bean2.NOT_FOUND) { - impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) - return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} - } else if err != nil { + if err != nil { impl.logger.Errorw("error in getting node", "err", err) return respMessage, err } From 02838b5b1486ccaf8dcbf212ccebc084289a333e Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 19:42:16 +0530 Subject: [PATCH 11/28] minor fixes --- api/k8s/capacity/k8sCapacityRestHandler.go | 5 ++++- pkg/k8s/capacity/k8sCapacityService.go | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/api/k8s/capacity/k8sCapacityRestHandler.go b/api/k8s/capacity/k8sCapacityRestHandler.go index b670e0a6ed..d66a9c3b35 100644 --- a/api/k8s/capacity/k8sCapacityRestHandler.go +++ b/api/k8s/capacity/k8sCapacityRestHandler.go @@ -3,6 +3,7 @@ package capacity import ( "encoding/json" "errors" + "fmt" "github.com/devtron-labs/common-lib/utils" "net/http" "strconv" @@ -372,9 +373,11 @@ func (handler *K8sCapacityRestHandlerImpl) DrainNode(w http.ResponseWriter, r *h if err != nil { errCode := http.StatusInternalServerError if apiErr, ok := err.(*utils.ApiError); ok { + errCode = apiErr.HttpStatusCode switch errCode { case http.StatusNotFound: - errCode = apiErr.HttpStatusCode + errorMessage := apiErr.UserMessage + err = fmt.Errorf("%s: %w", errorMessage, err) } } handler.logger.Errorw("error in draining node", "err", err, "req", nodeDrainReq) diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 5250d11c0c..464e905c6b 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" + bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" @@ -625,7 +626,10 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean } //get node node, err := impl.K8sUtil.GetNodeByName(context.Background(), k8sClientSet, request.Name) - if err != nil { + if err != nil && strings.Contains(err.Error(), bean2.NOT_FOUND) { + impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) + return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + } else if err != nil { impl.logger.Errorw("error in getting node", "err", err) return respMessage, err } @@ -641,7 +645,7 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean err = impl.deleteOrEvictPods(request.Name, request.NodeDrainHelper) if err != nil && strings.Contains(err.Error(), bean.DaemonSetPodDeleteError) { impl.logger.Errorw("daemonSet-managed pods can't be deleted", "err", err, "nodeName", request.Name) - return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: "cannot delete DaemonSet-managed Pods"} + return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: bean.DaemonSetPodDeleteError} } else if err != nil { impl.logger.Errorw("error in deleting/evicting pods", "err", err, "nodeName", request.Name) return respMessage, err From fa3c65be644740c5616e6ecee4eb69bf79b5c8f7 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 19:50:53 +0530 Subject: [PATCH 12/28] main merge --- go.sum | 4 ++-- vendor/modules.txt | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index fd9dff6c69..28e5c67034 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.33 h1:FpAV3ZgFluaRFcMwPpwxr/mwSipJ16XRvgABq3BzP5Y= github.com/devtron-labs/authenticator v0.4.33/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e h1:tPD2kWowGhgH3g5TRZMKgrvuNAgpFE6E6sxZaTAHNSU= -github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= +github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 h1:9LJtAFJ+4t5TybVhrLE3eLVGQvn6whX84ChinaiBqTI= +github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/modules.txt b/vendor/modules.txt index 400bb905e9..b266c2c573 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -354,9 +354,12 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.10-0.20240119134712-5ed4eed5fc0e +# github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage +github.com/devtron-labs/common-lib/cloud-provider-identifier +github.com/devtron-labs/common-lib/cloud-provider-identifier/bean +github.com/devtron-labs/common-lib/cloud-provider-identifier/providers github.com/devtron-labs/common-lib/constants github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib From 220b204e14c59b8696a8fdbf50985e22f09ef1bd Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 22 Jan 2024 19:58:26 +0530 Subject: [PATCH 13/28] minor fixes --- api/k8s/capacity/k8sCapacityRestHandler.go | 13 +------------ internal/util/ErrorUtil.go | 4 ---- pkg/k8s/capacity/k8sCapacityService.go | 6 +++--- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/api/k8s/capacity/k8sCapacityRestHandler.go b/api/k8s/capacity/k8sCapacityRestHandler.go index d66a9c3b35..3d363d1b22 100644 --- a/api/k8s/capacity/k8sCapacityRestHandler.go +++ b/api/k8s/capacity/k8sCapacityRestHandler.go @@ -3,8 +3,6 @@ package capacity import ( "encoding/json" "errors" - "fmt" - "github.com/devtron-labs/common-lib/utils" "net/http" "strconv" @@ -371,17 +369,8 @@ func (handler *K8sCapacityRestHandlerImpl) DrainNode(w http.ResponseWriter, r *h } resp, err := handler.k8sCapacityService.DrainNode(r.Context(), &nodeDrainReq) if err != nil { - errCode := http.StatusInternalServerError - if apiErr, ok := err.(*utils.ApiError); ok { - errCode = apiErr.HttpStatusCode - switch errCode { - case http.StatusNotFound: - errorMessage := apiErr.UserMessage - err = fmt.Errorf("%s: %w", errorMessage, err) - } - } handler.logger.Errorw("error in draining node", "err", err, "req", nodeDrainReq) - common.WriteJsonResp(w, err, nil, errCode) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } common.WriteJsonResp(w, nil, resp, http.StatusOK) diff --git a/internal/util/ErrorUtil.go b/internal/util/ErrorUtil.go index c01aef8d03..52df683fc6 100644 --- a/internal/util/ErrorUtil.go +++ b/internal/util/ErrorUtil.go @@ -36,10 +36,6 @@ func (e *ApiError) Error() string { return e.InternalMessage } -func (e *ApiError) StatusCode() int { - return e.HttpStatusCode -} - // default internal will be set func (e *ApiError) ErrorfInternal(format string, a ...interface{}) error { return &ApiError{InternalMessage: fmt.Sprintf(format, a...)} diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 464e905c6b..837b74e7ef 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -3,8 +3,8 @@ package capacity import ( "context" "fmt" - "github.com/devtron-labs/common-lib/utils" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/internal/util" bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" @@ -628,7 +628,7 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean node, err := impl.K8sUtil.GetNodeByName(context.Background(), k8sClientSet, request.Name) if err != nil && strings.Contains(err.Error(), bean2.NOT_FOUND) { impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) - return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} } else if err != nil { impl.logger.Errorw("error in getting node", "err", err) return respMessage, err @@ -645,7 +645,7 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean err = impl.deleteOrEvictPods(request.Name, request.NodeDrainHelper) if err != nil && strings.Contains(err.Error(), bean.DaemonSetPodDeleteError) { impl.logger.Errorw("daemonSet-managed pods can't be deleted", "err", err, "nodeName", request.Name) - return respMessage, &utils.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: bean.DaemonSetPodDeleteError} + return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} } else if err != nil { impl.logger.Errorw("error in deleting/evicting pods", "err", err, "nodeName", request.Name) return respMessage, err From 0f8a2c99cfb275928112591f6fe71e2b5dccf062 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 00:27:19 +0530 Subject: [PATCH 14/28] minor fixes in k8s capacity and app services --- pkg/k8s/application/k8sApplicationService.go | 5 ++++- pkg/k8s/capacity/k8sCapacityService.go | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 99e4771dd7..7e7364c1c8 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -444,7 +444,10 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResourceGVKWithoutAuthorization( return nil, err } allApiResources, err := impl.K8sUtil.GetApiResources(restConfig, bean3.LIST_VERB) - if err != nil { + if err != nil && strings.Contains(err.Error(), k8s2.DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { return nil, err } // FILTER STARTS diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 837b74e7ef..fe52f095ff 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -95,7 +95,10 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetail(ctx context.Context } clusterDetail := &bean.ClusterCapacityDetail{} nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil { + if err != nil && strings.Contains(err.Error(), k8s2.DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err } @@ -234,7 +237,10 @@ func (impl *K8sCapacityServiceImpl) GetNodeCapacityDetailsListByCluster(ctx cont impl.logger.Errorw("error in getting node metrics", "err", err) } nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil { + if err != nil && strings.Contains(err.Error(), k8s2.DnsLookupNoSuchHostError) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err } From 9c1bc5f7131d3c1b3edbb0c3ac02e82c0d2a271d Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 00:32:53 +0530 Subject: [PATCH 15/28] go mod version --- go.mod | 2 +- go.sum | 4 ++-- .../devtron-labs/common-lib/utils/k8s/K8sUtil.go | 10 ++-------- .../devtron-labs/common-lib/utils/k8s/bean.go | 1 - vendor/modules.txt | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 2209b17357..4da1b2bec8 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/devtron-labs/authenticator v0.4.33 - github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 + github.com/devtron-labs/common-lib v0.0.10 github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 diff --git a/go.sum b/go.sum index 28e5c67034..043c296b16 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADG github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/devtron-labs/authenticator v0.4.33 h1:FpAV3ZgFluaRFcMwPpwxr/mwSipJ16XRvgABq3BzP5Y= github.com/devtron-labs/authenticator v0.4.33/go.mod h1:ozNfT8WcruiSgnUbyp48WVfc41++W6xYXhKFp67lNTU= -github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 h1:9LJtAFJ+4t5TybVhrLE3eLVGQvn6whX84ChinaiBqTI= -github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= +github.com/devtron-labs/common-lib v0.0.10 h1:3ayOUwIedXTvBEj80mKJ0iLo06glZ9nzY/WFsYFpYGM= +github.com/devtron-labs/common-lib v0.0.10/go.mod h1:95/DizzVXu1kHap/VwEvdxwgd+BvPVYc0bJzt8yqGDU= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2 h1:/IEIsJTxDZ3hv8uOoCaqdWCXqcv7nCAgX9AP/v84dUY= github.com/devtron-labs/protos v0.0.0-20230503113602-282404f70fd2/go.mod h1:l85jxWHlcSo910hdUfRycL40yGzC6glE93V1sVxVPto= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go index 743a96d29d..751ea3bcea 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/K8sUtil.go @@ -960,10 +960,7 @@ func (impl K8sServiceImpl) GetCoreV1ClientByRestConfig(restConfig *rest.Config) func (impl K8sServiceImpl) GetNodesList(ctx context.Context, k8sClientSet *kubernetes.Clientset) (*v1.NodeList, error) { nodeList, err := k8sClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { impl.logger.Errorw("error in getting node list", "err", err) return nil, err } @@ -1344,10 +1341,7 @@ func (impl K8sServiceImpl) GetApiResources(restConfig *rest.Config, includeOnlyV } apiResourcesListFromK8s, err := discoveryClient.ServerPreferredResources() - if err != nil && strings.Contains(err.Error(), DnsLookupNoSuchHostError) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &utils.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { //takes care when K8s is unable to handle the request for some resources Isk8sApiError := strings.Contains(err.Error(), "unable to retrieve the complete list of server APIs") switch Isk8sApiError { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index df22b3337b..f27ce9676f 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -24,7 +24,6 @@ const ( DEVTRON_APP_LABEL_KEY = "app" DEVTRON_APP_LABEL_VALUE1 = "devtron" DEVTRON_APP_LABEL_VALUE2 = "orchestrator" - DnsLookupNoSuchHostError = "no such host" ) type ClusterConfig struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index b266c2c573..1cf47e3478 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -354,7 +354,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.0.11-0.20240122075510-f26a39f40c59 +# github.com/devtron-labs/common-lib v0.0.10 ## explicit; go 1.20 github.com/devtron-labs/common-lib/blob-storage github.com/devtron-labs/common-lib/cloud-provider-identifier From b618848d0d9acf5027b1d90d95ec6ee04d742c8c Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 00:48:31 +0530 Subject: [PATCH 16/28] make fixes --- pkg/k8s/application/bean/bean.go | 4 ++++ pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/k8s/capacity/k8sCapacityService.go | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/k8s/application/bean/bean.go b/pkg/k8s/application/bean/bean.go index ed18919aa1..f05ffad70f 100644 --- a/pkg/k8s/application/bean/bean.go +++ b/pkg/k8s/application/bean/bean.go @@ -21,6 +21,10 @@ const ( ArgoInstalledType = 1 // Identifier for ArgoCD deployment ) +const ( + DnsLookupNoSuchHostError = "no such host" +) + type ResourceInfo struct { PodName string `json:"podName"` } diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 7e7364c1c8..f1538e1ef0 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -444,7 +444,7 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResourceGVKWithoutAuthorization( return nil, err } allApiResources, err := impl.K8sUtil.GetApiResources(restConfig, bean3.LIST_VERB) - if err != nil && strings.Contains(err.Error(), k8s2.DnsLookupNoSuchHostError) { + if err != nil && strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) { impl.logger.Errorw("k8s cluster unreachable", "err", err) return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} } else if err != nil { diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index fe52f095ff..0670ef7872 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -9,6 +9,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" + bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" "github.com/devtron-labs/devtron/pkg/k8s/capacity/bean" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -95,7 +96,7 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetail(ctx context.Context } clusterDetail := &bean.ClusterCapacityDetail{} nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil && strings.Contains(err.Error(), k8s2.DnsLookupNoSuchHostError) { + if err != nil && strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) { impl.logger.Errorw("k8s cluster unreachable", "err", err) return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} } else if err != nil { @@ -237,7 +238,7 @@ func (impl *K8sCapacityServiceImpl) GetNodeCapacityDetailsListByCluster(ctx cont impl.logger.Errorw("error in getting node metrics", "err", err) } nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil && strings.Contains(err.Error(), k8s2.DnsLookupNoSuchHostError) { + if err != nil && strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) { impl.logger.Errorw("k8s cluster unreachable", "err", err) return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} } else if err != nil { From 6c7f74e4803b47b7c242c0beeaf81fb621bda103 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 01:27:31 +0530 Subject: [PATCH 17/28] timeout error also introduced while fetching api-resources of a cluster --- pkg/k8s/application/bean/bean.go | 1 + pkg/k8s/application/k8sApplicationService.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/k8s/application/bean/bean.go b/pkg/k8s/application/bean/bean.go index f05ffad70f..6c6a05a2de 100644 --- a/pkg/k8s/application/bean/bean.go +++ b/pkg/k8s/application/bean/bean.go @@ -23,6 +23,7 @@ const ( const ( DnsLookupNoSuchHostError = "no such host" + TimeoutError = "timeout" ) type ResourceInfo struct { diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index f1538e1ef0..ea59170f1e 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -444,7 +444,7 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResourceGVKWithoutAuthorization( return nil, err } allApiResources, err := impl.K8sUtil.GetApiResources(restConfig, bean3.LIST_VERB) - if err != nil && strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) { + if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { impl.logger.Errorw("k8s cluster unreachable", "err", err) return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} } else if err != nil { From b79ca1b2b05a7fefce721c406762f27328332f21 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 01:44:25 +0530 Subject: [PATCH 18/28] fix --- pkg/k8s/capacity/k8sCapacityService.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 0670ef7872..00719e0835 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -96,7 +96,7 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetail(ctx context.Context } clusterDetail := &bean.ClusterCapacityDetail{} nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil && strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) { + if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { impl.logger.Errorw("k8s cluster unreachable", "err", err) return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} } else if err != nil { @@ -238,7 +238,7 @@ func (impl *K8sCapacityServiceImpl) GetNodeCapacityDetailsListByCluster(ctx cont impl.logger.Errorw("error in getting node metrics", "err", err) } nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil && strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) { + if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { impl.logger.Errorw("k8s cluster unreachable", "err", err) return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} } else if err != nil { From d58cfc63030a9163066b290728df621c5edc4a0e Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 02:01:56 +0530 Subject: [PATCH 19/28] cd-pipeline/patch cd delete req 5xx handled for cluster unreachable (checkifnsexist) --- api/helm-app/HelmAppService.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index 2b5d992d03..b3feeef3af 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -7,6 +7,7 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/helm-app/models" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" "github.com/go-pg/pg" "google.golang.org/grpc/codes" "net/http" @@ -556,7 +557,10 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error return false, err } exists, err := impl.K8sUtil.CheckIfNsExists(app.Namespace, v12Client) - if err != nil { + if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } else if err != nil { impl.logger.Errorw("error in checking if namespace exists or not", "error", err, "clusterConfig", config) return false, err } From 2d137568bcfd8dc283e224a392f281804e249736 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 11:15:01 +0530 Subject: [PATCH 20/28] app already exist 5xx --- pkg/appStore/deployment/service/AppStoreDeploymentService.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/appStore/deployment/service/AppStoreDeploymentService.go b/pkg/appStore/deployment/service/AppStoreDeploymentService.go index 6207684df2..c826747ba4 100644 --- a/pkg/appStore/deployment/service/AppStoreDeploymentService.go +++ b/pkg/appStore/deployment/service/AppStoreDeploymentService.go @@ -543,6 +543,7 @@ func (impl AppStoreDeploymentServiceImpl) createAppForAppStore(createRequest *be if app1 != nil && app1.Id > 0 { impl.logger.Infow(" app already exists", "name", createRequest.AppName) err = &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, Code: constants.AppAlreadyExists.Code, InternalMessage: "app already exists", UserMessage: fmt.Sprintf("app already exists with name %s", createRequest.AppName), @@ -585,6 +586,7 @@ func (impl AppStoreDeploymentServiceImpl) createAppForAppStore(createRequest *be return nil, err } err = &util.ApiError{ + HttpStatusCode: http.StatusBadRequest, Code: constants.AppAlreadyExists.Code, InternalMessage: "app already exists", UserMessage: fmt.Sprintf("app already exists with name %s", createRequest.AppName), From 35867652a2a4712b324b2d83a564e5795972b5ec Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 13:06:48 +0530 Subject: [PATCH 21/28] code refactoring --- api/helm-app/ClusterUtils.go | 33 +++++++++++++++++ api/helm-app/HelmAppService.go | 10 ++--- cmd/external-app/wire_gen.go | 30 +++++++-------- pkg/k8s/application/bean/bean.go | 5 --- pkg/k8s/application/k8sApplicationService.go | 9 +++-- pkg/k8s/capacity/bean/bean.go | 23 ++++++------ pkg/k8s/capacity/k8sCapacityService.go | 39 +++++++++++--------- 7 files changed, 90 insertions(+), 59 deletions(-) create mode 100644 api/helm-app/ClusterUtils.go diff --git a/api/helm-app/ClusterUtils.go b/api/helm-app/ClusterUtils.go new file mode 100644 index 0000000000..8fcd64893b --- /dev/null +++ b/api/helm-app/ClusterUtils.go @@ -0,0 +1,33 @@ +package client + +import ( + "strings" +) + +const ( + DaemonSetPodDeleteError = "cannot delete DaemonSet-managed Pods" + DnsLookupNoSuchHostError = "no such host" + TimeoutError = "timeout" + NotFound = "not found" +) + +func IsClusterUnReachableError(err error) bool { + if strings.Contains(err.Error(), DnsLookupNoSuchHostError) || strings.Contains(err.Error(), TimeoutError) { + return true + } + return false +} + +func IsNodeNotFoundError(err error) bool { + if strings.Contains(err.Error(), NotFound) { + return true + } + return false +} + +func IsDaemonSetPodDeleteError(err error) bool { + if strings.Contains(err.Error(), DaemonSetPodDeleteError) { + return true + } + return false +} diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index b3feeef3af..0dc9132477 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -7,7 +7,6 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/helm-app/models" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" "github.com/go-pg/pg" "google.golang.org/grpc/codes" "net/http" @@ -557,10 +556,11 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error return false, err } exists, err := impl.K8sUtil.CheckIfNsExists(app.Namespace, v12Client) - if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { + if IsClusterUnReachableError(err) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } impl.logger.Errorw("error in checking if namespace exists or not", "error", err, "clusterConfig", config) return false, err } diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index ef258c929a..6b4b2333fa 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -146,7 +146,7 @@ func InitializeApp() (*App, error) { userAuditServiceImpl := user.NewUserAuditServiceImpl(sugaredLogger, userAuditRepositoryImpl) userServiceImpl := user.NewUserServiceImpl(userAuthRepositoryImpl, sugaredLogger, userRepositoryImpl, roleGroupRepositoryImpl, sessionManager, userCommonServiceImpl, userAuditServiceImpl) ssoLoginRepositoryImpl := sso.NewSSOLoginRepositoryImpl(db) - k8sUtil := k8s.NewK8sUtil(sugaredLogger, runtimeConfig) + k8sServiceImpl := k8s.NewK8sUtil(sugaredLogger, runtimeConfig) devtronSecretConfig, err := util2.GetDevtronSecretName() if err != nil { return nil, err @@ -157,7 +157,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(sugaredLogger, ssoLoginRepositoryImpl, k8sUtil, devtronSecretConfig, userAuthOidcHelperImpl) + ssoLoginServiceImpl := sso.NewSSOLoginServiceImpl(sugaredLogger, ssoLoginRepositoryImpl, k8sServiceImpl, devtronSecretConfig, userAuthOidcHelperImpl) ssoLoginRestHandlerImpl := sso2.NewSsoLoginRestHandlerImpl(validate, sugaredLogger, enforcerImpl, userServiceImpl, ssoLoginServiceImpl) ssoLoginRouterImpl := sso2.NewSsoLoginRouterImpl(ssoLoginRestHandlerImpl) teamRepositoryImpl := team.NewTeamRepositoryImpl(db) @@ -166,12 +166,12 @@ func InitializeApp() (*App, error) { teamServiceImpl := team.NewTeamServiceImpl(sugaredLogger, teamRepositoryImpl, userAuthServiceImpl) clusterRepositoryImpl := repository2.NewClusterRepositoryImpl(db, sugaredLogger) v := informer.NewGlobalMapClusterNamespace() - k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, runtimeConfig, k8sUtil) - clusterServiceImpl := cluster.NewClusterServiceImpl(clusterRepositoryImpl, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) + k8sInformerFactoryImpl := informer.NewK8sInformerFactoryImpl(sugaredLogger, v, runtimeConfig, k8sServiceImpl) + clusterServiceImpl := cluster.NewClusterServiceImpl(clusterRepositoryImpl, sugaredLogger, k8sServiceImpl, k8sInformerFactoryImpl, userAuthRepositoryImpl, userRepositoryImpl, roleGroupRepositoryImpl) appStatusRepositoryImpl := appStatus.NewAppStatusRepositoryImpl(db, sugaredLogger) environmentRepositoryImpl := repository2.NewEnvironmentRepositoryImpl(db, sugaredLogger, appStatusRepositoryImpl) attributesRepositoryImpl := repository3.NewAttributesRepositoryImpl(db) - environmentServiceImpl := cluster.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImpl, sugaredLogger, k8sUtil, k8sInformerFactoryImpl, userAuthServiceImpl, attributesRepositoryImpl) + environmentServiceImpl := cluster.NewEnvironmentServiceImpl(environmentRepositoryImpl, clusterServiceImpl, sugaredLogger, k8sServiceImpl, k8sInformerFactoryImpl, userAuthServiceImpl, attributesRepositoryImpl) chartRepoRepositoryImpl := chartRepoRepository.NewChartRepoRepositoryImpl(db) acdAuthConfig, err := util3.GetACDAuthConfig() if err != nil { @@ -182,7 +182,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sUtil, clusterServiceImpl, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) + chartRepositoryServiceImpl := chartRepo.NewChartRepositoryServiceImpl(sugaredLogger, chartRepoRepositoryImpl, k8sServiceImpl, clusterServiceImpl, acdAuthConfig, httpClient, serverEnvConfigServerEnvConfig) installedAppRepositoryImpl := repository4.NewInstalledAppRepositoryImpl(sugaredLogger, db) helmClientConfig, err := client2.GetConfig() if err != nil { @@ -199,7 +199,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl, k8sUtil, helmReleaseConfig) + helmAppServiceImpl := client2.NewHelmAppServiceImpl(sugaredLogger, clusterServiceImpl, helmAppClientImpl, pumpImpl, enforcerUtilHelmImpl, serverDataStoreServerDataStore, serverEnvConfigServerEnvConfig, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, pipelineRepositoryImpl, installedAppRepositoryImpl, appRepositoryImpl, clusterRepositoryImpl, k8sServiceImpl, helmReleaseConfig) dockerArtifactStoreRepositoryImpl := repository5.NewDockerArtifactStoreRepositoryImpl(db) dockerRegistryIpsConfigRepositoryImpl := repository5.NewDockerRegistryIpsConfigRepositoryImpl(db) ociRegistryConfigRepositoryImpl := repository5.NewOCIRegistryConfigRepositoryImpl(db) @@ -248,15 +248,15 @@ func InitializeApp() (*App, error) { attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentCommonServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) - k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sUtil, clusterServiceImpl) - environmentRestHandlerImpl := cluster2.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceImpl, k8sUtil, k8sCommonServiceImpl) + k8sCommonServiceImpl := k8s2.NewK8sCommonServiceImpl(sugaredLogger, k8sServiceImpl, clusterServiceImpl) + environmentRestHandlerImpl := cluster2.NewEnvironmentRestHandlerImpl(environmentServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceImpl, k8sServiceImpl, k8sCommonServiceImpl) environmentRouterImpl := cluster2.NewEnvironmentRouterImpl(environmentRestHandlerImpl) k8sResourceHistoryRepositoryImpl := repository7.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) ephemeralContainersRepositoryImpl := repository2.NewEphemeralContainersRepositoryImpl(db) ephemeralContainerServiceImpl := cluster.NewEphemeralContainerServiceImpl(ephemeralContainersRepositoryImpl, sugaredLogger) - terminalSessionHandlerImpl := terminal.NewTerminalSessionHandlerImpl(environmentServiceImpl, clusterServiceImpl, sugaredLogger, k8sUtil, ephemeralContainerServiceImpl) - k8sApplicationServiceImpl, err := application.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImpl, pumpImpl, helmAppServiceImpl, k8sUtil, acdAuthConfig, k8sResourceHistoryServiceImpl, k8sCommonServiceImpl, terminalSessionHandlerImpl, ephemeralContainerServiceImpl, ephemeralContainersRepositoryImpl) + terminalSessionHandlerImpl := terminal.NewTerminalSessionHandlerImpl(environmentServiceImpl, clusterServiceImpl, sugaredLogger, k8sServiceImpl, ephemeralContainerServiceImpl) + k8sApplicationServiceImpl, err := application.NewK8sApplicationServiceImpl(sugaredLogger, clusterServiceImpl, pumpImpl, helmAppServiceImpl, k8sServiceImpl, acdAuthConfig, k8sResourceHistoryServiceImpl, k8sCommonServiceImpl, terminalSessionHandlerImpl, ephemeralContainerServiceImpl, ephemeralContainersRepositoryImpl) if err != nil { return nil, err } @@ -301,7 +301,7 @@ func InitializeApp() (*App, error) { } moduleRepositoryImpl := moduleRepo.NewModuleRepositoryImpl(db) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sUtil, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl) + telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl) if err != nil { return nil, err } @@ -321,7 +321,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - moduleCacheServiceImpl := module.NewModuleCacheServiceImpl(sugaredLogger, k8sUtil, moduleEnvConfig, serverEnvConfigServerEnvConfig, serverDataStoreServerDataStore, moduleRepositoryImpl, teamServiceImpl) + moduleCacheServiceImpl := module.NewModuleCacheServiceImpl(sugaredLogger, k8sServiceImpl, moduleEnvConfig, serverEnvConfigServerEnvConfig, serverDataStoreServerDataStore, moduleRepositoryImpl, teamServiceImpl) moduleServiceHelperImpl := module.NewModuleServiceHelperImpl(serverEnvConfigServerEnvConfig) moduleResourceStatusRepositoryImpl := moduleRepo.NewModuleResourceStatusRepositoryImpl(db) moduleDataStoreModuleDataStore := moduleDataStore.InitModuleDataStore() @@ -349,7 +349,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - k8sCapacityServiceImpl := capacity.NewK8sCapacityServiceImpl(sugaredLogger, clusterServiceImpl, k8sApplicationServiceImpl, k8sUtil, k8sCommonServiceImpl, clusterCronServiceImpl) + k8sCapacityServiceImpl := capacity.NewK8sCapacityServiceImpl(sugaredLogger, clusterServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, k8sCommonServiceImpl, clusterCronServiceImpl) k8sCapacityRestHandlerImpl := capacity2.NewK8sCapacityRestHandlerImpl(sugaredLogger, k8sCapacityServiceImpl, userServiceImpl, enforcerImpl, clusterServiceImpl, environmentServiceImpl, clusterRbacServiceImpl) k8sCapacityRouterImpl := capacity2.NewK8sCapacityRouterImpl(k8sCapacityRestHandlerImpl) webhookHelmServiceImpl := webhookHelm.NewWebhookHelmServiceImpl(sugaredLogger, helmAppServiceImpl, clusterServiceImpl, chartRepositoryServiceImpl, attributesServiceImpl) @@ -366,7 +366,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - userTerminalAccessServiceImpl, err := clusterTerminalAccess.NewUserTerminalAccessServiceImpl(sugaredLogger, terminalAccessRepositoryImpl, userTerminalSessionConfig, k8sCommonServiceImpl, terminalSessionHandlerImpl, k8sCapacityServiceImpl, k8sUtil) + userTerminalAccessServiceImpl, err := clusterTerminalAccess.NewUserTerminalAccessServiceImpl(sugaredLogger, terminalAccessRepositoryImpl, userTerminalSessionConfig, k8sCommonServiceImpl, terminalSessionHandlerImpl, k8sCapacityServiceImpl, k8sServiceImpl) if err != nil { return nil, err } diff --git a/pkg/k8s/application/bean/bean.go b/pkg/k8s/application/bean/bean.go index 6c6a05a2de..ed18919aa1 100644 --- a/pkg/k8s/application/bean/bean.go +++ b/pkg/k8s/application/bean/bean.go @@ -21,11 +21,6 @@ const ( ArgoInstalledType = 1 // Identifier for ArgoCD deployment ) -const ( - DnsLookupNoSuchHostError = "no such host" - TimeoutError = "timeout" -) - type ResourceInfo struct { PodName string `json:"podName"` } diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index ea59170f1e..2649fc5c3d 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -444,10 +444,11 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResourceGVKWithoutAuthorization( return nil, err } allApiResources, err := impl.K8sUtil.GetApiResources(restConfig, bean3.LIST_VERB) - if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { + if client.IsClusterUnReachableError(err) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } return nil, err } // FILTER STARTS diff --git a/pkg/k8s/capacity/bean/bean.go b/pkg/k8s/capacity/bean/bean.go index 3d2ae29384..95e35c8644 100644 --- a/pkg/k8s/capacity/bean/bean.go +++ b/pkg/k8s/capacity/bean/bean.go @@ -27,18 +27,17 @@ const NamespaceAll string = "" // below const set is used for pod filters const ( - daemonSetFatal = "DaemonSet-managed Pods (use --ignore-daemonsets to ignore)" - daemonSetWarning = "ignoring DaemonSet-managed Pods" - localStorageFatal = "Pods with local storage (use --delete-emptydir-data to override)" - localStorageWarning = "deleting Pods with local storage" - unmanagedFatal = "Pods declare no controller (use --force to override)" - unmanagedWarning = "deleting Pods that declare no controller" - AWSNodeGroupLabel = "alpha.eksctl.io/nodegroup-name" - AzureNodeGroupLabel = "kubernetes.azure.com/agentpool" - GcpNodeGroupLabel = "cloud.google.com/gke-nodepool" - KopsNodeGroupLabel = "kops.k8s.io/instancegroup" - AWSEKSNodeGroupLabel = "eks.amazonaws.com/nodegroup" - DaemonSetPodDeleteError = "cannot delete DaemonSet-managed Pods" + daemonSetFatal = "DaemonSet-managed Pods (use --ignore-daemonsets to ignore)" + daemonSetWarning = "ignoring DaemonSet-managed Pods" + localStorageFatal = "Pods with local storage (use --delete-emptydir-data to override)" + localStorageWarning = "deleting Pods with local storage" + unmanagedFatal = "Pods declare no controller (use --force to override)" + unmanagedWarning = "deleting Pods that declare no controller" + AWSNodeGroupLabel = "alpha.eksctl.io/nodegroup-name" + AzureNodeGroupLabel = "kubernetes.azure.com/agentpool" + GcpNodeGroupLabel = "cloud.google.com/gke-nodepool" + KopsNodeGroupLabel = "kops.k8s.io/instancegroup" + AWSEKSNodeGroupLabel = "eks.amazonaws.com/nodegroup" ) // TODO: add any new nodeGrouplabel in this array diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index 00719e0835..bf661a9eda 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -4,12 +4,11 @@ import ( "context" "fmt" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" + client "github.com/devtron-labs/devtron/api/helm-app" "github.com/devtron-labs/devtron/internal/util" - bean2 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" - bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" "github.com/devtron-labs/devtron/pkg/k8s/capacity/bean" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -96,10 +95,11 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetail(ctx context.Context } clusterDetail := &bean.ClusterCapacityDetail{} nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { + if client.IsClusterUnReachableError(err) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err } @@ -238,10 +238,11 @@ func (impl *K8sCapacityServiceImpl) GetNodeCapacityDetailsListByCluster(ctx cont impl.logger.Errorw("error in getting node metrics", "err", err) } nodeList, err := impl.K8sUtil.GetNodesList(ctx, k8sClientSet) - if err != nil && (strings.Contains(err.Error(), bean3.DnsLookupNoSuchHostError) || strings.Contains(err.Error(), bean3.TimeoutError)) { - impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} - } else if err != nil { + if err != nil { + if client.IsClusterUnReachableError(err) { + impl.logger.Errorw("k8s cluster unreachable", "err", err) + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + } impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err } @@ -633,10 +634,11 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean } //get node node, err := impl.K8sUtil.GetNodeByName(context.Background(), k8sClientSet, request.Name) - if err != nil && strings.Contains(err.Error(), bean2.NOT_FOUND) { - impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) - return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} - } else if err != nil { + if err != nil { + if client.IsNodeNotFoundError(err) { + impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) + return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + } impl.logger.Errorw("error in getting node", "err", err) return respMessage, err } @@ -650,10 +652,11 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean } request.NodeDrainHelper.K8sClientSet = k8sClientSet err = impl.deleteOrEvictPods(request.Name, request.NodeDrainHelper) - if err != nil && strings.Contains(err.Error(), bean.DaemonSetPodDeleteError) { - impl.logger.Errorw("daemonSet-managed pods can't be deleted", "err", err, "nodeName", request.Name) - return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} - } else if err != nil { + if err != nil { + if client.IsDaemonSetPodDeleteError(err) { + impl.logger.Errorw("daemonSet-managed pods can't be deleted", "err", err, "nodeName", request.Name) + return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + } impl.logger.Errorw("error in deleting/evicting pods", "err", err, "nodeName", request.Name) return respMessage, err } From 6d80c4aa80c4d2c4d86206334460ec24fac0ad43 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 13:22:09 +0530 Subject: [PATCH 22/28] fix --- api/helm-app/HelmAppService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index 0dc9132477..ab156f3a1b 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -559,7 +559,7 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error if err != nil { if IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + return false, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} } impl.logger.Errorw("error in checking if namespace exists or not", "error", err, "clusterConfig", config) return false, err From c6dc9b1492af0506db250ff2993334b9a53b9813 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 13:41:19 +0530 Subject: [PATCH 23/28] fix --- api/helm-app/HelmAppService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index ab156f3a1b..286eaa9229 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -559,7 +559,7 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error if err != nil { if IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return false, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: err.Error()} } impl.logger.Errorw("error in checking if namespace exists or not", "error", err, "clusterConfig", config) return false, err From 3de1ba66c4a5dc4130d75d6dd208d4f4c38a7f9a Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Tue, 23 Jan 2024 17:58:52 +0530 Subject: [PATCH 24/28] removing code 200 from all util Api errors --- api/helm-app/HelmAppService.go | 2 +- pkg/delete/DeleteServiceExtended.go | 8 ++++---- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/k8s/capacity/k8sCapacityService.go | 8 ++++---- pkg/pipeline/BuildPipelineConfigService.go | 1 - pkg/pipeline/CiService.go | 2 +- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index 286eaa9229..5a4c5be5d3 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -559,7 +559,7 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error if err != nil { if IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: err.Error()} + return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} } impl.logger.Errorw("error in checking if namespace exists or not", "error", err, "clusterConfig", config) return false, err diff --git a/pkg/delete/DeleteServiceExtended.go b/pkg/delete/DeleteServiceExtended.go index 39ac917e43..66486b33df 100644 --- a/pkg/delete/DeleteServiceExtended.go +++ b/pkg/delete/DeleteServiceExtended.go @@ -63,7 +63,7 @@ func (impl DeleteServiceExtendedImpl) DeleteCluster(deleteRequest *cluster.Clust } if len(env) > 0 { impl.logger.Errorw("err in deleting cluster, found env in this cluster", "clusterName", deleteRequest.ClusterName, "err", err) - return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related environments before deleting this cluster"} + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: " Please delete all related environments before deleting this cluster"} } err = impl.clusterService.DeleteFromDb(deleteRequest, userId) if err != nil { @@ -91,13 +91,13 @@ func (impl DeleteServiceExtendedImpl) DeleteEnvironment(deleteRequest *cluster.E } if len(installedApps) > 0 && len(pipelines) > 0 { impl.logger.Errorw("err in deleting env, found cd pipelines and helm apps in this env", "envName", deleteRequest.Environment, "err", err) - return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related cd pipelines and helm apps before deleting this environment"} + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: " Please delete all related cd pipelines and helm apps before deleting this environment"} } else if len(installedApps) > 0 { impl.logger.Errorw("err in deleting env, found helm apps in this env", "envName", deleteRequest.Environment, "err", err) - return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related helm apps before deleting this environment"} + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: " Please delete all related helm apps before deleting this environment"} } else if len(pipelines) > 0 { impl.logger.Errorw("err in deleting env, found cd pipelines in this env", "envName", deleteRequest.Environment, "err", err) - return &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: " Please delete all related cd pipelines before deleting this environment"} + return &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: " Please delete all related cd pipelines before deleting this environment"} } err = impl.environmentService.Delete(deleteRequest, userId) diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 2649fc5c3d..567ab4b593 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -447,7 +447,7 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResourceGVKWithoutAuthorization( if err != nil { if client.IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "k8s cluster unreachable"} } return nil, err } diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index bf661a9eda..40d9c9f8de 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -98,7 +98,7 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetail(ctx context.Context if err != nil { if client.IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "k8s cluster unreachable"} } impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err @@ -241,7 +241,7 @@ func (impl *K8sCapacityServiceImpl) GetNodeCapacityDetailsListByCluster(ctx cont if err != nil { if client.IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: "k8s cluster unreachable"} + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "k8s cluster unreachable"} } impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err @@ -637,7 +637,7 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean if err != nil { if client.IsNodeNotFoundError(err) { impl.logger.Errorw("node not found", "err", err, "nodeName", request.Name) - return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, UserMessage: err.Error()} } impl.logger.Errorw("error in getting node", "err", err) return respMessage, err @@ -655,7 +655,7 @@ func (impl *K8sCapacityServiceImpl) DrainNode(ctx context.Context, request *bean if err != nil { if client.IsDaemonSetPodDeleteError(err) { impl.logger.Errorw("daemonSet-managed pods can't be deleted", "err", err, "nodeName", request.Name) - return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: err.Error()} + return respMessage, &util.ApiError{HttpStatusCode: http.StatusNotFound, UserMessage: err.Error()} } impl.logger.Errorw("error in deleting/evicting pods", "err", err, "nodeName", request.Name) return respMessage, err diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 241e8d23bc..50a2bc44f6 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -1997,7 +1997,6 @@ func (impl *CiPipelineConfigServiceImpl) DeleteCiPipeline(request *bean.CiPatchR if len(workflowMapping) > 0 { return nil, &util.ApiError{ HttpStatusCode: http.StatusBadRequest, - Code: "200", InternalMessage: "Please delete deployment pipelines for this workflow first and try again.", UserDetailMessage: fmt.Sprintf("Please delete deployment pipelines for this workflow first and try again."), UserMessage: fmt.Sprintf("Please delete deployment pipelines for this workflow first and try again.")} diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index e1aca00642..7cc5e28546 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -205,7 +205,7 @@ func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot if len(preCiSteps) == 0 && isJob { - return 0, &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: "200", UserMessage: "No tasks are configured in this job pipeline"} + return 0, &util.ApiError{HttpStatusCode: http.StatusNotFound, UserMessage: "No tasks are configured in this job pipeline"} } savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfig, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) if err != nil { From 4c8dd31e69e1a0be0eb51a98476d9f00789a8151 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 24 Jan 2024 12:00:22 +0530 Subject: [PATCH 25/28] fix --- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/k8s/capacity/k8sCapacityService.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 4b25638e96..e3907f715a 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -447,7 +447,7 @@ func (impl *K8sApplicationServiceImpl) GetAllApiResourceGVKWithoutAuthorization( if err != nil { if client.IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "k8s cluster unreachable"} + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} } return nil, err } diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index eef7d99b13..3404c6b4bf 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -98,7 +98,7 @@ func (impl *K8sCapacityServiceImpl) GetClusterCapacityDetail(ctx context.Context if err != nil { if client.IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "k8s cluster unreachable"} + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} } impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err @@ -241,7 +241,7 @@ func (impl *K8sCapacityServiceImpl) GetNodeCapacityDetailsListByCluster(ctx cont if err != nil { if client.IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "k8s cluster unreachable"} + return nil, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} } impl.logger.Errorw("error in getting node list", "err", err, "clusterId", cluster.Id) return nil, err From 93238ce207a683b274e5bdaa7ccf4df2853090ef Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 2 Feb 2024 12:10:38 +0530 Subject: [PATCH 26/28] minor fix --- pkg/appStore/deployment/service/InstalledAppService.go | 3 --- pkg/pipeline/CiHandler.go | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/appStore/deployment/service/InstalledAppService.go b/pkg/appStore/deployment/service/InstalledAppService.go index c74ae09bd8..10a10d3afe 100644 --- a/pkg/appStore/deployment/service/InstalledAppService.go +++ b/pkg/appStore/deployment/service/InstalledAppService.go @@ -919,9 +919,6 @@ func (impl *InstalledAppServiceImpl) FetchChartNotes(installedAppId int, envId i installedApp, err := impl.installedAppRepository.FetchNotes(installedAppId) if err != nil && err != pg.ErrNoRows { return "", err - } else if err == pg.ErrNoRows { - impl.logger.Errorw("installed app not found or may have been deleted", "installedAppId", installedAppId, "envId", envId) - return "", &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "400", UserMessage: "Installed app not found in database or may have been deleted", InternalMessage: err.Error()} } installedAppVerison, err := impl.installedAppRepository.GetInstalledAppVersionByInstalledAppIdAndEnvId(installedAppId, envId) if err != nil { diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 9d4192730c..2b48e6bf3c 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -612,11 +612,11 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, er // Terminate workflow err = impl.workflowService.TerminateWorkflow(workflow.ExecutorType, workflow.Name, workflow.Namespace, restConfig, isExt, env) - if err != nil && !strings.Contains(err.Error(), "cannot find workflow") { + if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { impl.Logger.Errorw("cannot terminate wf", "err", err) return 0, err - } else if err != nil { - return 0, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} } workflow.Status = executors.WorkflowCancel From 0ed1c519a8813e8fc05fcaed9c3ce0a3ca5729ba Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 16 Feb 2024 11:48:28 +0530 Subject: [PATCH 27/28] wire fix --- api/helm-app/{ => service}/ClusterUtils.go | 2 +- pkg/k8s/capacity/k8sCapacityService.go | 2 +- wire_gen.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename api/helm-app/{ => service}/ClusterUtils.go (97%) diff --git a/api/helm-app/ClusterUtils.go b/api/helm-app/service/ClusterUtils.go similarity index 97% rename from api/helm-app/ClusterUtils.go rename to api/helm-app/service/ClusterUtils.go index 8fcd64893b..1e4f33c8da 100644 --- a/api/helm-app/ClusterUtils.go +++ b/api/helm-app/service/ClusterUtils.go @@ -1,4 +1,4 @@ -package client +package service import ( "strings" diff --git a/pkg/k8s/capacity/k8sCapacityService.go b/pkg/k8s/capacity/k8sCapacityService.go index f9ecc1be4c..0159947112 100644 --- a/pkg/k8s/capacity/k8sCapacityService.go +++ b/pkg/k8s/capacity/k8sCapacityService.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" - client "github.com/devtron-labs/devtron/api/helm-app" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/k8s" diff --git a/wire_gen.go b/wire_gen.go index 436b40ccbf..815d24ca14 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -765,7 +765,7 @@ func InitializeApp() (*App, error) { deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl) pipelineHistoryRestHandlerImpl := history2.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineHistoryRouterImpl := history3.NewPipelineHistoryRouterImpl(pipelineHistoryRestHandlerImpl) - pipelineStatusTimelineRestHandlerImpl := status2.NewPipelineStatusTimelineRestHandlerImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, enforcerUtilImpl, enforcerImpl) + pipelineStatusTimelineRestHandlerImpl := status2.NewPipelineStatusTimelineRestHandlerImpl(sugaredLogger, userServiceImpl, pipelineStatusTimelineServiceImpl, enforcerUtilImpl, enforcerImpl, cdApplicationStatusUpdateHandlerImpl, pipelineBuilderImpl) pipelineStatusRouterImpl := status3.NewPipelineStatusRouterImpl(pipelineStatusTimelineRestHandlerImpl) appWorkflowRestHandlerImpl := workflow.NewAppWorkflowRestHandlerImpl(sugaredLogger, userServiceImpl, appWorkflowServiceImpl, teamServiceImpl, enforcerImpl, pipelineBuilderImpl, appRepositoryImpl, enforcerUtilImpl) appWorkflowRouterImpl := workflow2.NewAppWorkflowRouterImpl(appWorkflowRestHandlerImpl) From 1f04e60ccc046ad7fe517ab3da05a7a3c78d6a05 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Mon, 19 Feb 2024 14:05:16 +0530 Subject: [PATCH 28/28] minor fixes --- api/helm-app/service/ClusterUtils.go | 4 +++- api/helm-app/service/HelmAppService.go | 2 +- .../installedApp/service/EAMode/EAModeDeploymentService.go | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/helm-app/service/ClusterUtils.go b/api/helm-app/service/ClusterUtils.go index 1e4f33c8da..199d200bef 100644 --- a/api/helm-app/service/ClusterUtils.go +++ b/api/helm-app/service/ClusterUtils.go @@ -9,10 +9,12 @@ const ( DnsLookupNoSuchHostError = "no such host" TimeoutError = "timeout" NotFound = "not found" + ConnectionRefused = "connection refused" ) func IsClusterUnReachableError(err error) bool { - if strings.Contains(err.Error(), DnsLookupNoSuchHostError) || strings.Contains(err.Error(), TimeoutError) { + if strings.Contains(err.Error(), DnsLookupNoSuchHostError) || strings.Contains(err.Error(), TimeoutError) || + strings.Contains(err.Error(), ConnectionRefused) { return true } return false diff --git a/api/helm-app/service/HelmAppService.go b/api/helm-app/service/HelmAppService.go index 3f77c7e977..d0efce9c0f 100644 --- a/api/helm-app/service/HelmAppService.go +++ b/api/helm-app/service/HelmAppService.go @@ -590,7 +590,7 @@ func (impl *HelmAppServiceImpl) checkIfNsExists(app *AppIdentifier) (bool, error if err != nil { if IsClusterUnReachableError(err) { impl.logger.Errorw("k8s cluster unreachable", "err", err) - return false, &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + return false, &util.ApiError{HttpStatusCode: http.StatusUnprocessableEntity, UserMessage: err.Error()} } impl.logger.Errorw("error in checking if namespace exists or not", "error", err, "clusterConfig", config) return false, err diff --git a/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go b/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go index 36b4ced1b9..78e05979d2 100644 --- a/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go +++ b/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go @@ -9,6 +9,7 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" + "net/http" "time" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" @@ -142,6 +143,10 @@ func (impl *EAModeDeploymentServiceImpl) DeleteInstalledApp(ctx context.Context, isInstalled, err := impl.helmAppService.IsReleaseInstalled(ctx, appIdentifier) if err != nil { + if client.IsClusterUnReachableError(err) { + impl.Logger.Errorw("k8s cluster unreachable", "err", err) + return &util.ApiError{HttpStatusCode: http.StatusUnprocessableEntity, UserMessage: err.Error()} + } impl.Logger.Errorw("error in checking if helm release is installed or not", "error", err, "appIdentifier", appIdentifier) return err }