fix: Build get Stuck Indefinitely in case of K8s Driver When Node Termination Occurs

client/argocdServer/ArgoClientWrapperService.go

@@ -28,6 +28,7 @@ import (
 	"github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/caarlos0/env"
 	"github.com/devtron-labs/common-lib/async"
+	"github.com/devtron-labs/common-lib/utils/retryFunc"
 	"github.com/devtron-labs/devtron/client/argocdServer/adapter"
 	"github.com/devtron-labs/devtron/client/argocdServer/application"
 	"github.com/devtron-labs/devtron/client/argocdServer/bean"
@@ -41,7 +42,6 @@ import (
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/git"
-	"github.com/devtron-labs/devtron/util/retryFunc"
 	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"

env_gen.md

@@ -25,6 +25,14 @@
  | USE_DEPLOYMENT_CONFIG_DATA | bool |false | use deployment config data from deployment_config table |  | true |
 
 
+## CI_BUILDX Related Environment Variables
+| Key   | Type     | Default Value     | Description       | Example       | Deprecated       |
+|-------|----------|-------------------|-------------------|-----------------------|------------------|
+ | ASYNC_BUILDX_CACHE_EXPORT | bool |false | To enable async container image cache export |  | false |
+ | BUILDX_CACHE_MODE_MIN | bool |false | To set build cache mode to minimum in buildx |  | false |
+ | BUILDX_INTERRUPTION_MAX_RETRY | int |3 | Maximum number of retries for buildx builder interruption |  | false |
+
+
 ## CI_RUNNER Related Environment Variables
 | Key   | Type     | Default Value     | Description       | Example       | Deprecated       |
 |-------|----------|-------------------|-------------------|-----------------------|------------------|
@@ -136,10 +144,8 @@
  | ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT | int |1 | Delay on retrying the maifest commit the on gitops |  | false |
  | ARGO_REPO_REGISTER_RETRY_COUNT | int |4 | Retry count for registering a GitOps repository to ArgoCD | 3 | false |
  | ARGO_REPO_REGISTER_RETRY_DELAY | int |5 | Delay (in Seconds) between the retries for registering a GitOps repository to ArgoCD | 5 | false |
- | ASYNC_BUILDX_CACHE_EXPORT | bool |false | To enable async container image cache export |  | false |
  | BATCH_SIZE | int |5 | there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go. |  | false |
  | BLOB_STORAGE_ENABLED | bool |false |  |  | false |
- | BUILDX_CACHE_MODE_MIN | bool |false | To set build cache mode to minimum in buildx |  | false |
  | CD_HOST | string |localhost | Host for the devtron stack |  | false |
  | CD_NAMESPACE | string |devtroncd |  |  | false |
  | CD_PORT | string |8000 | Port for pre/post-cd |  | false |

go.mod

@@ -315,7 +315,7 @@ require (
 replace (
 	github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13
 	github.com/cyphar/filepath-securejoin v0.4.1 => github.com/cyphar/filepath-securejoin v0.3.6 // indirect
-	github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250616095652-3ee00d35b864
-	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250616095652-3ee00d35b864
+	github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250623221428-f565cfab8993
+	github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250623221428-f565cfab8993
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1
 )

go.sum

@@ -237,10 +237,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq
 github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU=
 github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA=
-github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250616095652-3ee00d35b864 h1:nemhi7OwDE4zTRyMYbeW8rCgCa3FnetL8Izt30gUn7k=
-github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250616095652-3ee00d35b864/go.mod h1:9LCkYfiWaEKIBkmxw9jX1GujvEMyHwmDtVsatffAkeU=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250616095652-3ee00d35b864 h1:MNpsaUvDgvaiHcYVzT8tPJ7UyY8QiBekpWezfPYu4nA=
-github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250616095652-3ee00d35b864/go.mod h1:/Ciy9tD9OxZOWBDPIasM448H7uvSo4+ZJiExpfwBZpA=
+github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250623221428-f565cfab8993 h1:s7XPhNlueKV1ZEX9K/wz8Wu9AX/61LDdFyC1DeqcTDE=
+github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250623221428-f565cfab8993/go.mod h1:9LCkYfiWaEKIBkmxw9jX1GujvEMyHwmDtVsatffAkeU=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250623221428-f565cfab8993 h1:x4n8S5ENofGbXsGy8JVq2gXeESLU25dLdO+VjTupsN8=
+github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250623221428-f565cfab8993/go.mod h1:/Ciy9tD9OxZOWBDPIasM448H7uvSo4+ZJiExpfwBZpA=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU=
 github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y=
 github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs=

pkg/build/trigger/HandlerService.go

@@ -84,9 +84,11 @@ type HandlerService interface {
 	DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error)
 }
 
-type BuildxCacheFlags struct {
-	BuildxCacheModeMin     bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" `
-	AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"`
+// CATEGORY=CI_BUILDX
+type BuildxGlobalFlags struct {
+	BuildxCacheModeMin         bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" `
+	AsyncBuildxCacheExport     bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"`
+	BuildxInterruptionMaxRetry int  `env:"BUILDX_INTERRUPTION_MAX_RETRY" envDefault:"3" description:"Maximum number of retries for buildx builder interruption"`
 }
 
 type HandlerServiceImpl struct {
@@ -105,7 +107,7 @@ type HandlerServiceImpl struct {
 	config                       *types.CiConfig
 	scopedVariableManager        variables.ScopedVariableManager
 	ciCdPipelineOrchestrator     pipeline2.CiCdPipelineOrchestrator
-	buildxCacheFlags             *BuildxCacheFlags
+	buildxGlobalFlags            *BuildxGlobalFlags
 	attributeService             attributes.AttributesService
 	pluginInputVariableParser    pipeline2.PluginInputVariableParser
 	globalPluginService          plugin.GlobalPluginService
@@ -145,10 +147,10 @@ func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.W
 	K8sUtil *k8s.K8sServiceImpl,
 	asyncRunnable *async.Runnable,
 ) *HandlerServiceImpl {
-	buildxCacheFlags := &BuildxCacheFlags{}
+	buildxCacheFlags := &BuildxGlobalFlags{}
 	err := env.Parse(buildxCacheFlags)
 	if err != nil {
-		Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err)
+		Logger.Infow("error occurred while parsing BuildxGlobalFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err)
 	}
 	cis := &HandlerServiceImpl{
 		Logger:                       Logger,
@@ -165,7 +167,7 @@ func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.W
 		scopedVariableManager:        scopedVariableManager,
 		customTagService:             customTagService,
 		ciCdPipelineOrchestrator:     ciCdPipelineOrchestrator,
-		buildxCacheFlags:             buildxCacheFlags,
+		buildxGlobalFlags:            buildxCacheFlags,
 		attributeService:             attributeService,
 		pluginInputVariableParser:    pluginInputVariableParser,
 		globalPluginService:          globalPluginService,
@@ -769,9 +771,9 @@ func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types
 	} else {
 		workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE
 	}
-	workflowRequest, err = impl.updateWorkflowRequestWithBuildCacheData(workflowRequest, scope)
+	workflowRequest, err = impl.updateWorkflowRequestWithBuildxFlags(workflowRequest, scope)
 	if err != nil {
-		impl.Logger.Errorw("error, updateWorkflowRequestWithBuildCacheData", "workflowRequest", workflowRequest, "err", err)
+		impl.Logger.Errorw("error, updateWorkflowRequestWithBuildxFlags", "workflowRequest", workflowRequest, "err", err)
 		return nil, nil, nil, nil, err
 	}
 	if impl.canSetK8sDriverData(workflowRequest) {

pkg/build/trigger/HandlerService_ent.go

@@ -65,10 +65,11 @@ func (impl *HandlerServiceImpl) updateWorkflowRequestWithEntSupportData(workflow
 	return workflowRequest
 }
 
-func (impl *HandlerServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest,
+func (impl *HandlerServiceImpl) updateWorkflowRequestWithBuildxFlags(workflowRequest *types.WorkflowRequest,
 	scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) {
-	workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin
-	workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport
+	workflowRequest.BuildxCacheModeMin = impl.buildxGlobalFlags.BuildxCacheModeMin
+	workflowRequest.AsyncBuildxCacheExport = impl.buildxGlobalFlags.AsyncBuildxCacheExport
+	workflowRequest.BuildxInterruptionMaxRetry = impl.buildxGlobalFlags.BuildxInterruptionMaxRetry
 	return workflowRequest, nil
 }
 

pkg/deployment/gitOps/git/GitOperationService.go

@@ -18,8 +18,8 @@ package git
 
 import (
 	"context"
-	"errors"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/retryFunc"
 	bean2 "github.com/devtron-labs/devtron/api/bean"
 	apiBean "github.com/devtron-labs/devtron/api/bean/gitOps"
 	"github.com/devtron-labs/devtron/internal/util"
@@ -28,7 +28,6 @@ import (
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean"
 	globalUtil "github.com/devtron-labs/devtron/util"
-	"github.com/devtron-labs/devtron/util/retryFunc"
 	dirCopy "github.com/otiai10/copy"
 	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
@@ -284,10 +283,7 @@ func (impl *GitOperationServiceImpl) CommitValues(ctx context.Context, chartGitA
 }
 
 func (impl *GitOperationServiceImpl) isRetryableGitCommitError(err error) bool {
-	if retryErr := (&retryFunc.RetryableError{}); errors.As(err, &retryErr) {
-		return true
-	}
-	return false
+	return retryFunc.IsRetryableError(err)
 }
 
 func (impl *GitOperationServiceImpl) CreateRepository(ctx context.Context, dto *apiBean.GitOpsConfigDto, userId int32) (string, bool, bool, error) {

pkg/deployment/gitOps/git/GitServiceAzure.go

@@ -21,10 +21,10 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/retryFunc"
 	bean2 "github.com/devtron-labs/devtron/api/bean/gitOps"
 	globalUtil "github.com/devtron-labs/devtron/util"
 	"github.com/devtron-labs/devtron/util/gitUtil"
-	"github.com/devtron-labs/devtron/util/retryFunc"
 	"github.com/microsoft/azure-devops-go-api/azuredevops"
 	"github.com/microsoft/azure-devops-go-api/azuredevops/git"
 	"go.uber.org/zap"

pkg/deployment/gitOps/git/GitServiceBitbucket.go

@@ -21,9 +21,9 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/retryFunc"
 	bean2 "github.com/devtron-labs/devtron/api/bean/gitOps"
 	"github.com/devtron-labs/devtron/util"
-	"github.com/devtron-labs/devtron/util/retryFunc"
 	"github.com/devtron-labs/go-bitbucket"
 	"go.uber.org/zap"
 	"io/ioutil"

pkg/deployment/gitOps/git/GitServiceGithub.go

@@ -21,10 +21,10 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/retryFunc"
 	"github.com/devtron-labs/common-lib/utils/runTime"
 	bean2 "github.com/devtron-labs/devtron/api/bean/gitOps"
 	globalUtil "github.com/devtron-labs/devtron/util"
-	"github.com/devtron-labs/devtron/util/retryFunc"
 	"github.com/google/go-github/github"
 	"go.uber.org/zap"
 	"golang.org/x/oauth2"

pkg/deployment/gitOps/git/GitServiceGitlab.go

@@ -20,10 +20,10 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/retryFunc"
 	bean2 "github.com/devtron-labs/devtron/api/bean/gitOps"
 	"github.com/devtron-labs/devtron/pkg/deployment/gitOps/git/bean"
 	"github.com/devtron-labs/devtron/util"
-	"github.com/devtron-labs/devtron/util/retryFunc"
 	_ "github.com/hashicorp/go-retryablehttp"
 	"github.com/xanzy/go-gitlab"
 	"go.uber.org/zap"

pkg/pipeline/types/Workflow.go

@@ -151,6 +151,7 @@ type WorkflowRequest struct {
 	Scope                       resourceQualifiers.Scope
 	BuildxCacheModeMin          bool   `json:"buildxCacheModeMin"`
 	AsyncBuildxCacheExport      bool   `json:"asyncBuildxCacheExport"`
+	BuildxInterruptionMaxRetry  int    `json:"buildxInterruptionMaxRetry"`
 	UseDockerApiToGetDigest     bool   `json:"useDockerApiToGetDigest"`
 	HostUrl                     string `json:"hostUrl"`
 	WorkflowRequestEnt

vendor/modules.txt

@@ -523,15 +523,15 @@ github.com/davecgh/go-spew/spew
 # github.com/deckarep/golang-set v1.8.0
 ## explicit; go 1.17
 github.com/deckarep/golang-set
-# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250616095652-3ee00d35b864
+# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250623221428-f565cfab8993
 ## explicit; go 1.24.0
 github.com/devtron-labs/authenticator/apiToken
 github.com/devtron-labs/authenticator/client
 github.com/devtron-labs/authenticator/jwt
 github.com/devtron-labs/authenticator/middleware
 github.com/devtron-labs/authenticator/oidc
 github.com/devtron-labs/authenticator/password
-# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250616095652-3ee00d35b864
+# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250623221428-f565cfab8993
 ## explicit; go 1.24.0
 github.com/devtron-labs/common-lib/async
 github.com/devtron-labs/common-lib/blob-storage
@@ -560,6 +560,7 @@ github.com/devtron-labs/common-lib/utils/k8s/health
 github.com/devtron-labs/common-lib/utils/k8sObjectsUtil
 github.com/devtron-labs/common-lib/utils/registry
 github.com/devtron-labs/common-lib/utils/remoteConnection/bean
+github.com/devtron-labs/common-lib/utils/retryFunc
 github.com/devtron-labs/common-lib/utils/runTime
 github.com/devtron-labs/common-lib/utils/workFlow
 github.com/devtron-labs/common-lib/utils/yaml
@@ -2569,5 +2570,5 @@ xorm.io/xorm/log
 xorm.io/xorm/names
 xorm.io/xorm/schemas
 xorm.io/xorm/tags
-# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250616095652-3ee00d35b864
-# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250616095652-3ee00d35b864
+# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250623221428-f565cfab8993
+# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250623221428-f565cfab8993