Skip to content
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
7d21dd2
letting build abort even after starting stage of ci_workflow
prakash100198 Dec 12, 2023
416d458
error 500 removed
adi6859 Dec 14, 2023
1727ca4
forceAbort query param in cancle stage
prakash100198 Dec 14, 2023
d8506c4
Merge branch 'main' into unable-to-abort-ci-fix
prakash100198 Dec 14, 2023
b3855e2
minor fix
prakash100198 Dec 14, 2023
1b63728
minor fix
prakash100198 Dec 14, 2023
e7d91ed
error 500 removed
adi6859 Dec 14, 2023
de2b7a1
some random change
adi6859 Dec 14, 2023
74a054d
fix
prakash100198 Dec 15, 2023
f60981a
some error messaging changed
adi6859 Dec 15, 2023
e657a93
if workflow already cancelled then return from ci success event
prakash100198 Dec 18, 2023
84b8c7d
helm uninstall issue fixed
adi6859 Dec 18, 2023
e8d5073
status code changes
ShashwatDadhich Dec 19, 2023
8abc912
changed err code and msg in getWorkflowLogs in cihandler
prakash100198 Dec 19, 2023
16bc217
k8s issue fixed for 409
adi6859 Dec 19, 2023
481ad10
when workflow not found to terminate then throw bad req instead of 5xx
prakash100198 Dec 19, 2023
56bf5e7
fix
prakash100198 Dec 19, 2023
61bec54
Merge remote-tracking branch 'origin/unable-to-abort-ci-fix' into err…
adi6859 Dec 19, 2023
87e0f6f
Merge remote-tracking branch 'origin/500-status-fix' into error-5xx
adi6859 Dec 19, 2023
72be29b
Merge branch 'main' into error-5xx
prakash100198 Dec 21, 2023
5960126
fix
prakash100198 Dec 26, 2023
e7c155c
incorporated grpc code check in cd-pipeline/trigger api in createHelm…
prakash100198 Dec 26, 2023
68a6387
small fix
prakash100198 Dec 26, 2023
403ba29
minor refactoring
prakash100198 Dec 26, 2023
5343258
revert cd-pipeline/trigger 5xx fix
prakash100198 Dec 27, 2023
a5e2736
bug fix
prakash100198 Dec 28, 2023
866ca98
Merge branch 'main' into error-5xx
prakash100198 Dec 28, 2023
9ad3042
Merge branch 'main' into error-5xx
prakash100198 Jan 3, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions api/helm-app/HelmAppService.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/devtron-labs/devtron/api/helm-app/models"
repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry"
"github.com/go-pg/pg"
"google.golang.org/grpc/codes"
"net/http"
"reflect"
"strconv"
Expand Down Expand Up @@ -520,6 +521,14 @@ func (impl *HelmAppServiceImpl) DeleteApplication(ctx context.Context, app *AppI

deleteApplicationResponse, err := impl.helmAppClient.DeleteApplication(ctx, req)
if err != nil {
code, message := util.GetGRPCDetailedError(err)
if code == codes.NotFound {
return nil, &util.ApiError{
Code: "404",
HttpStatusCode: 200,
UserMessage: message,
}
}
impl.logger.Errorw("error in deleting helm application", "err", err)
return nil, errors.New(util.GetGRPCErrorDetailedMessage(err))
}
Expand Down
2 changes: 2 additions & 0 deletions api/restHandler/AppWorkflowRestHandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ func (handler AppWorkflowRestHandlerImpl) DeleteAppWorkflow(w http.ResponseWrite
if err != nil {
if _, ok := err.(*util.ApiError); ok {
handler.Logger.Warnw("error on deleting", "err", err)
common.WriteJsonResp(w, err, []byte("Creation Failed"), http.StatusOK)
return
} else {
handler.Logger.Errorw("error on deleting", "err", err)
}
Expand Down
10 changes: 9 additions & 1 deletion api/restHandler/app/BuildPipelineRestHandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -1377,6 +1377,7 @@ func (handler PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrite
common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
return
}
queryVars := r.URL.Query()
vars := mux.Vars(r)
workflowId, err := strconv.Atoi(vars["workflowId"])
if err != nil {
Expand All @@ -1390,6 +1391,13 @@ func (handler PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrite
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
return
}
var forceAbort bool
forceAbort, err = strconv.ParseBool(queryVars.Get("forceAbort"))
if err != nil {
handler.Logger.Errorw("request err, CancelWorkflow", "err", err)
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
return
}
handler.Logger.Infow("request payload, CancelWorkflow", "workflowId", workflowId, "pipelineId", pipelineId)

ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId)
Expand Down Expand Up @@ -1429,7 +1437,7 @@ func (handler PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrite

//RBAC

resp, err := handler.ciHandler.CancelBuild(workflowId)
resp, err := handler.ciHandler.CancelBuild(workflowId, forceAbort)
if err != nil {
handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId)
if util.IsErrNoRows(err) {
Expand Down
8 changes: 8 additions & 0 deletions internal/util/ErrorUtil.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package util
import (
"fmt"
"github.com/go-pg/pg"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

Expand Down Expand Up @@ -55,3 +56,10 @@ func GetGRPCErrorDetailedMessage(err error) string {
}
return err.Error()
}

func GetGRPCDetailedError(err error) (codes.Code, string) {
if errStatus, ok := status.FromError(err); ok {
return errStatus.Code(), errStatus.Message()
}
return codes.Unknown, err.Error()
}
6 changes: 6 additions & 0 deletions pkg/k8s/K8sCommonService.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import (
k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean"
"github.com/devtron-labs/devtron/api/bean"
"github.com/devtron-labs/devtron/api/helm-app"
util2 "github.com/devtron-labs/devtron/internal/util"
"github.com/devtron-labs/devtron/pkg/cluster"
bean3 "github.com/devtron-labs/devtron/pkg/k8s/application/bean"
"github.com/devtron-labs/devtron/util"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/version"
Expand Down Expand Up @@ -92,6 +94,10 @@ func (impl *K8sCommonServiceImpl) UpdateResource(ctx context.Context, request *R
resp, err := impl.K8sUtil.UpdateResource(ctx, restConfig, resourceIdentifier.GroupVersionKind, resourceIdentifier.Namespace, request.K8sRequest.Patch)
if err != nil {
impl.logger.Errorw("error in updating resource", "err", err, "clusterId", clusterId)
statusError, ok := err.(*errors.StatusError)
if ok {
err = &util2.ApiError{Code: "400", HttpStatusCode: int(statusError.ErrStatus.Code), UserMessage: statusError.Error()}
}
return nil, err
}
return resp, nil
Expand Down
34 changes: 28 additions & 6 deletions pkg/pipeline/CiHandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ type CiHandler interface {
FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error)
FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error)
//FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error)
CancelBuild(workflowId int) (int, error)
CancelBuild(workflowId int, forceAbort bool) (int, error)

GetRunningWorkflowLogs(pipelineId int, workflowId int) (*bufio.Reader, func() error, error)
GetHistoricBuildLogs(pipelineId int, workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error)
Expand Down Expand Up @@ -163,6 +163,7 @@ const Running = "Running"
const Starting = "Starting"
const POD_DELETED_MESSAGE = "pod deleted"
const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate"
const ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort"

func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error {

Expand Down Expand Up @@ -580,15 +581,22 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int
return ciWorkLowResponses, nil
}

func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) {
func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) {
workflow, err := impl.ciWorkflowRepository.FindById(workflowId)
if err != nil {
impl.Logger.Errorw("err", "err", err)
return 0, err
}
if !(string(v1alpha1.NodePending) == workflow.Status || string(v1alpha1.NodeRunning) == workflow.Status) {
impl.Logger.Warn("cannot cancel build, build not in progress")
return 0, errors.New("cannot cancel build, build not in progress")
if forceAbort {
return impl.cancelBuildAfterStartWorkflowStage(workflow)
} else {
return 0, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"}
}
}
//this arises when someone deletes the workflow in resource browser and wants to force abort a ci
if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort {
return impl.cancelBuildAfterStartWorkflowStage(workflow)
}
isExt := workflow.Namespace != DefaultCiWorkflowNamespace
var env *repository3.Environment
Expand All @@ -602,9 +610,11 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) {

// Terminate workflow
err = impl.workflowService.TerminateWorkflow(workflow.ExecutorType, workflow.Name, workflow.Namespace, restConfig, isExt, env)
if err != nil {
if err != nil && !strings.Contains(err.Error(), "cannot find workflow") {
impl.Logger.Errorw("cannot terminate wf", "err", err)
return 0, err
} else {
return 0, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()}
}

workflow.Status = executors.WorkflowCancel
Expand Down Expand Up @@ -634,6 +644,18 @@ func (impl *CiHandlerImpl) CancelBuild(workflowId int) (int, error) {
return workflow.Id, nil
}

func (impl *CiHandlerImpl) cancelBuildAfterStartWorkflowStage(workflow *pipelineConfig.CiWorkflow) (int, error) {
workflow.Status = executors.WorkflowCancel
workflow.PodStatus = string(bean.Failed)
workflow.Message = ABORT_MESSAGE_AFTER_STARTING_STAGE
err := impl.ciWorkflowRepository.UpdateWorkFlow(workflow)
if err != nil {
impl.Logger.Errorw("error in updating workflow status", "err", err)
return 0, err
}
return workflow.Id, nil
}

func (impl *CiHandlerImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) {
env, err := impl.envRepository.FindById(workflow.EnvironmentId)
if err != nil {
Expand Down Expand Up @@ -789,7 +811,7 @@ func (impl *CiHandlerImpl) getWorkflowLogs(pipelineId int, ciWorkflow *pipelineC
return impl.getLogsFromRepository(pipelineId, ciWorkflow, clusterConfig, isExt)
}
impl.Logger.Errorw("err", "err", err)
return nil, nil, err
return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()}
}
logReader := bufio.NewReader(logStream)
return logReader, cleanUp, err
Expand Down
5 changes: 5 additions & 0 deletions pkg/pipeline/WebhookService.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
util2 "github.com/devtron-labs/devtron/internal/util"
"github.com/devtron-labs/devtron/pkg/app"
"github.com/devtron-labs/devtron/pkg/pipeline/bean"
"github.com/devtron-labs/devtron/pkg/pipeline/executors"
repository2 "github.com/devtron-labs/devtron/pkg/pipeline/repository"
types2 "github.com/devtron-labs/devtron/pkg/pipeline/types"
repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository"
Expand Down Expand Up @@ -177,6 +178,10 @@ func (impl WebhookServiceImpl) HandleCiSuccessEvent(ciPipelineId int, request *C
impl.logger.Errorw("cannot get saved wf", "err", err)
return 0, err
}
// if workflow already cancelled then return, this state arises when user force aborts a ci
if savedWorkflow.Status == executors.WorkflowCancel {
return 0, err
}
savedWorkflow.Status = string(v1alpha1.NodeSucceeded)
impl.logger.Debugw("updating workflow ", "savedWorkflow", savedWorkflow)
err = impl.ciWorkflowRepository.UpdateWorkFlow(savedWorkflow)
Expand Down