diff --git a/api/helm-app/HelmAppService.go b/api/helm-app/HelmAppService.go index b8b1348577..8485431c26 100644 --- a/api/helm-app/HelmAppService.go +++ b/api/helm-app/HelmAppService.go @@ -8,6 +8,7 @@ import ( "github.com/devtron-labs/devtron/api/helm-app/models" repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/go-pg/pg" + "google.golang.org/grpc/codes" "net/http" "reflect" "strconv" @@ -520,6 +521,14 @@ func (impl *HelmAppServiceImpl) DeleteApplication(ctx context.Context, app *AppI deleteApplicationResponse, err := impl.helmAppClient.DeleteApplication(ctx, req) if err != nil { + code, message := util.GetGRPCDetailedError(err) + if code == codes.NotFound { + return nil, &util.ApiError{ + Code: "404", + HttpStatusCode: 200, + UserMessage: message, + } + } impl.logger.Errorw("error in deleting helm application", "err", err) return nil, errors.New(util.GetGRPCErrorDetailedMessage(err)) } diff --git a/api/restHandler/AppWorkflowRestHandler.go b/api/restHandler/AppWorkflowRestHandler.go index e243cfba78..ef7e33a262 100644 --- a/api/restHandler/AppWorkflowRestHandler.go +++ b/api/restHandler/AppWorkflowRestHandler.go @@ -163,6 +163,8 @@ func (handler AppWorkflowRestHandlerImpl) DeleteAppWorkflow(w http.ResponseWrite if err != nil { if _, ok := err.(*util.ApiError); ok { handler.Logger.Warnw("error on deleting", "err", err) + common.WriteJsonResp(w, err, []byte("Creation Failed"), http.StatusOK) + return } else { handler.Logger.Errorw("error on deleting", "err", err) } diff --git a/api/restHandler/app/BuildPipelineRestHandler.go b/api/restHandler/app/BuildPipelineRestHandler.go index 4a38551c44..dfebe712c9 100644 --- a/api/restHandler/app/BuildPipelineRestHandler.go +++ b/api/restHandler/app/BuildPipelineRestHandler.go @@ -1382,6 +1382,7 @@ func (handler PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrite common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return } + queryVars := r.URL.Query() vars := mux.Vars(r) workflowId, err := strconv.Atoi(vars["workflowId"]) if err != nil { @@ -1395,6 +1396,13 @@ func (handler PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrite common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + var forceAbort bool + forceAbort, err = strconv.ParseBool(queryVars.Get("forceAbort")) + if err != nil { + handler.Logger.Errorw("request err, CancelWorkflow", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } handler.Logger.Infow("request payload, CancelWorkflow", "workflowId", workflowId, "pipelineId", pipelineId) ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId) @@ -1434,7 +1442,7 @@ func (handler PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrite //RBAC - resp, err := handler.ciHandler.CancelBuild(workflowId) + resp, err := handler.ciHandler.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/internal/util/ErrorUtil.go b/internal/util/ErrorUtil.go index 4ca26b6df6..52df683fc6 100644 --- a/internal/util/ErrorUtil.go +++ b/internal/util/ErrorUtil.go @@ -20,6 +20,7 @@ package util import ( "fmt" "github.com/go-pg/pg" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -55,3 +56,10 @@ func GetGRPCErrorDetailedMessage(err error) string { } return err.Error() } + +func GetGRPCDetailedError(err error) (codes.Code, string) { + if errStatus, ok := status.FromError(err); ok { + return errStatus.Code(), errStatus.Message() + } + return codes.Unknown, err.Error() +} diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 26cc1cdc92..6ff2385e6a 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -8,11 +8,13 @@ import ( k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/helm-app" + util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/cluster" bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" "github.com/devtron-labs/devtron/util" "go.opentelemetry.io/otel" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" @@ -92,6 +94,10 @@ func (impl *K8sCommonServiceImpl) UpdateResource(ctx context.Context, request *R resp, err := impl.K8sUtil.UpdateResource(ctx, restConfig, resourceIdentifier.GroupVersionKind, resourceIdentifier.Namespace, request.K8sRequest.Patch) if err != nil { impl.logger.Errorw("error in updating resource", "err", err, "clusterId", clusterId) + statusError, ok := err.(*errors.StatusError) + if ok { + err = &util2.ApiError{Code: "400", HttpStatusCode: int(statusError.ErrStatus.Code), UserMessage: statusError.Error()} + } return nil, err } return resp, nil diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 307cc7c3ff..d9cde718ed 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -69,7 +69,7 @@ type CiHandler interface { FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) - CancelBuild(workflowId int) (int, error) + CancelBuild(workflowId int, forceAbort bool) (int, error) GetRunningWorkflowLogs(pipelineId int, workflowId int) (*bufio.Reader, func() error, error) GetHistoricBuildLogs(pipelineId int, workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) @@ -163,6 +163,7 @@ const Running = "Running" const Starting = "Starting" const POD_DELETED_MESSAGE = "pod deleted" const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate" +const ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort" func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { @@ -581,15 +582,22 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return ciWorkLowResponses, nil } -func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) { +func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { workflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("err", "err", err) return 0, err } if !(string(v1alpha1.NodePending) == workflow.Status || string(v1alpha1.NodeRunning) == workflow.Status) { - impl.Logger.Warn("cannot cancel build, build not in progress") - return 0, errors.New("cannot cancel build, build not in progress") + if forceAbort { + return impl.cancelBuildAfterStartWorkflowStage(workflow) + } else { + return 0, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a ci + if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.cancelBuildAfterStartWorkflowStage(workflow) } isExt := workflow.Namespace != DefaultCiWorkflowNamespace var env *repository3.Environment @@ -603,9 +611,11 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) { // Terminate workflow err = impl.workflowService.TerminateWorkflow(workflow.ExecutorType, workflow.Name, workflow.Namespace, restConfig, isExt, env) - if err != nil { + if err != nil && !strings.Contains(err.Error(), "cannot find workflow") { impl.Logger.Errorw("cannot terminate wf", "err", err) return 0, err + } else if err != nil { + return 0, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} } workflow.Status = executors.WorkflowCancel @@ -635,6 +645,18 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) { return workflow.Id, nil } +func (impl *CiHandlerImpl) cancelBuildAfterStartWorkflowStage(workflow *pipelineConfig.CiWorkflow) (int, error) { + workflow.Status = executors.WorkflowCancel + workflow.PodStatus = string(bean.Failed) + workflow.Message = ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.ciWorkflowRepository.UpdateWorkFlow(workflow) + if err != nil { + impl.Logger.Errorw("error in updating workflow status", "err", err) + return 0, err + } + return workflow.Id, nil +} + func (impl *CiHandlerImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { env, err := impl.envRepository.FindById(workflow.EnvironmentId) if err != nil { @@ -785,13 +807,13 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) if logStream == nil || err != nil { if !ciWorkflow.BlobStorageEnabled { - return nil, nil, errors.New("logs-not-stored-in-repository") + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == executors.WorkflowCancel { impl.Logger.Errorw("err", "err", err) return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt) } impl.Logger.Errorw("err", "err", err) - return nil, nil, err + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} } logReader := bufio.NewReader(logStream) return logReader, cleanUp, err diff --git a/pkg/pipeline/WebhookService.go b/pkg/pipeline/WebhookService.go index 6168d00e96..23cee294d2 100644 --- a/pkg/pipeline/WebhookService.go +++ b/pkg/pipeline/WebhookService.go @@ -30,6 +30,7 @@ import ( util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository" types2 "github.com/devtron-labs/devtron/pkg/pipeline/types" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -177,6 +178,10 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C impl.logger.Errorw("cannot get saved wf", "err", err) return 0, err } + // if workflow already cancelled then return, this state arises when user force aborts a ci + if savedWorkflow.Status == executors.WorkflowCancel { + return 0, err + } savedWorkflow.Status = string(v1alpha1.NodeSucceeded) impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow) err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow) diff --git a/pkg/pipeline/WorkflowDagExecutor.go b/pkg/pipeline/WorkflowDagExecutor.go index d8fa8d3b1b..7fa3e5be4d 100644 --- a/pkg/pipeline/WorkflowDagExecutor.go +++ b/pkg/pipeline/WorkflowDagExecutor.go @@ -3589,10 +3589,10 @@ func (impl *WorkflowDagExecutorImpl) createHelmAppForCdPipeline(overrideRequest impl.logger.Errorw("error in helm install custom chart", "err", err) return false, err } - if util.GetGRPCErrorDetailedMessage(err) == context.Canceled.Error() { err = errors.New(pipelineConfig.NEW_DEPLOYMENT_INITIATED) } + // IMP: update cd pipeline to mark deployment app created, even if helm install fails // If the helm install fails, it still creates the app in failed state, so trying to // re-create the app results in error from helm that cannot re-use name which is still in use