Skip to content

Commit 8a47b97

Browse files
prakash100198prakash100198
committed
Merge branch 'develop' into rename-secret-to-cm-cluster
2 parents 483d6a2 + 8c6fb75 commit 8a47b97

File tree

6 files changed

+75
-62
lines changed

6 files changed

+75
-62
lines changed

‎ci-runner/app/CiCdProcessor.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,22 @@ func (impl *CiCdProcessor) ProcessCiCdEvent(ciCdRequest *helper.CiCdTriggerEvent
100100
if logLevel == "" || logLevel == "DEBUG" {
101101
log.Println(util.DEVTRON, " ci-cd request details -----> ", ciCdRequest)
102102
}
103+
impl.handleEventBasedOnType(ciCdRequest, &exitCode)
104+
impl.HandleCleanup(*ciCdRequest, &exitCode, util.Source_Defer)
105+
return
106+
}
103107

104-
defer impl.HandleCleanup(*ciCdRequest, &exitCode, util.Source_Defer)
108+
func (impl *CiCdProcessor) handleEventBasedOnType(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) {
109+
defer func() { //recover in this function allows us to process further cleanup even if the code crashes
110+
if r := recover(); r != nil {
111+
log.Println("recovered from panic in handleEventBasedOnType:", r)
112+
}
113+
}()
105114
if helper.IsCIOrJobTypeEvent(ciCdRequest.Type) {
106-
impl.ciStage.HandleCIEvent(ciCdRequest, &exitCode)
115+
impl.ciStage.HandleCIEvent(ciCdRequest, exitCode)
107116
} else {
108-
impl.cdStage.HandleCDEvent(ciCdRequest, &exitCode)
117+
impl.cdStage.HandleCDEvent(ciCdRequest, exitCode)
109118
}
110-
return
111119
}
112120

113121
func (impl *CiCdProcessor) CleanUpBuildxK8sDriver(ciCdRequest helper.CiCdTriggerEvent, wg *sync.WaitGroup) {

‎ci-runner/executor/stage/cdStages.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,14 @@ func (impl *CdStage) HandleCDEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
5656
return
5757
}
5858

59-
func (impl *CdStage) handleCDEvent(ciCdRequest *helper.CiCdTriggerEvent) (*helper.HandleCdEventResponse, error) {
59+
func (impl *CdStage) handleCDEvent(ciCdRequest *helper.CiCdTriggerEvent) (resp *helper.HandleCdEventResponse, err error) {
60+
defer func() { //recover in this function allows us to send event even if the code crashes
61+
if r := recover(); r != nil {
62+
log.Println("recovered from panic in handleCDEvent:", r)
63+
err = fmt.Errorf("panic occurred during CD event handling")
64+
}
65+
}()
6066
var artifactUploaded bool
61-
var err error
6267
var allPluginArtifacts *helper.PluginArtifacts
6368

6469
allPluginArtifacts, err = impl.runCDStages(ciCdRequest)

‎ci-runner/executor/stage/ciStages.go‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ func NewCiStage(gitManager helper.GitManager, dockerHelper helper.DockerHelper,
7474
}
7575
}
7676

77-
func deferCIEvent(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) {
78-
log.Println(util.DEVTRON, "defer CI stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded)
77+
func sendCIEventResult(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool, err error) (exitCode int) {
78+
log.Println(util.DEVTRON, "sendCIEventResult CI stage data.", "err: ", err, "artifactUploaded: ", artifactUploaded)
7979
if err != nil {
8080
var stageError *helper.CiStageError
8181
if errors.As(err, &stageError) {
@@ -99,13 +99,23 @@ func deferCIEvent(ciRequest *helper.CommonWorkflowRequest, artifactUploaded bool
9999
}
100100

101101
func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCode *int) {
102-
var artifactUploaded bool
103-
var err error
102+
artifactUploaded, err := impl.handleCIEvent(ciCdRequest)
103+
if err != nil {
104+
//log error and send event result
105+
log.Println("ci stage error: ", err)
106+
}
107+
*exitCode = sendCIEventResult(ciCdRequest.CommonWorkflowRequest, artifactUploaded, err)
108+
}
109+
110+
func (impl *CiStage) handleCIEvent(ciCdRequest *helper.CiCdTriggerEvent) (artifactUploaded bool, err error) {
111+
defer func() { //recover in this function allows us to send event even if the code crashes
112+
if r := recover(); r != nil {
113+
log.Println("recovered from panic in handleCIEvent:", r)
114+
err = fmt.Errorf("panic occurred during CI event handling")
115+
}
116+
}()
104117
ciRequest := ciCdRequest.CommonWorkflowRequest
105118
ciContext := cicxt.BuildCiContext(context.Background(), ciRequest.EnableSecretMasking)
106-
defer func() {
107-
*exitCode = deferCIEvent(ciRequest, artifactUploaded, err)
108-
}()
109119
artifactUploaded, err = impl.runCIStages(ciContext, ciCdRequest)
110120
log.Println(util.DEVTRON, artifactUploaded, err)
111121
var artifactUploadErr error
@@ -116,16 +126,16 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
116126

117127
if err != nil {
118128
log.Println(util.DEVTRON, err)
119-
return
129+
return artifactUploaded, err
120130
}
121131

122132
if artifactUploadErr != nil {
123133
log.Println(util.DEVTRON, "error in artifact upload: ", artifactUploadErr)
124134
if ciCdRequest.CommonWorkflowRequest.IsExtRun {
125135
log.Println(util.DEVTRON, "Ignoring artifactUploadErr")
126-
return
136+
return artifactUploaded, err
127137
}
128-
return
138+
return artifactUploaded, err
129139
}
130140

131141
// sync cache
@@ -148,7 +158,7 @@ func (impl *CiStage) HandleCIEvent(ciCdRequest *helper.CiCdTriggerEvent, exitCod
148158
if err != nil {
149159
log.Println("error in cache push", err)
150160
}
151-
return
161+
return artifactUploaded, err
152162
}
153163

154164
// TODO: take as tech debt and break this function into parts for better code readability
@@ -672,17 +682,10 @@ func pullCache(metrics *helper.CIMetrics, ciCdRequest *helper.CiCdTriggerEvent)
672682
log.Println(util.DEVTRON, " cache-pull")
673683
start := time.Now()
674684
metrics.CacheDownStartTime = start
675-
676-
defer func() {
677-
log.Println(util.DEVTRON, " /cache-pull")
678-
metrics.CacheDownDuration = time.Since(start).Seconds()
679-
}()
680-
681685
err := helper.GetCache(ciCdRequest.CommonWorkflowRequest)
682-
if err != nil {
683-
return err
684-
}
685-
return nil
686+
log.Println(util.DEVTRON, " /cache-pull")
687+
metrics.CacheDownDuration = time.Since(start).Seconds()
688+
return err
686689
}
687690

688691
func (impl *CiStage) prepareStep(ciCdRequest *helper.CiCdTriggerEvent, metrics *helper.CIMetrics, skipCheckout bool) error {

‎ci-runner/helper/DockerHelper.go‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -548,14 +548,14 @@ func (impl *DockerHelperImpl) getBuildxExportCacheFunc(ciContext cicxt.CiContext
548548
wg.Add(len(exportCacheCmds))
549549
for platform, exportCacheCmd := range exportCacheCmds {
550550
go func(platform, exportCacheCmd string) {
551-
defer wg.Done()
552551
log.Println("exporting build cache, platform : ", platform)
553552
log.Println(exportCacheCmd)
554553
err := impl.executeCmd(ciContext, exportCacheCmd)
555554
if err != nil {
556-
log.Println("error in exporting ", "err : ", err)
557-
return
555+
log.Println("error in exporting", "err:", err)
556+
//not returning as need to mark waitGroup done
558557
}
558+
wg.Done()
559559
}(platform, exportCacheCmd)
560560
}
561561
wg.Wait()
@@ -1006,27 +1006,31 @@ func (impl *DockerHelperImpl) CleanBuildxK8sDriver(ciContext cicxt.CiContext, no
10061006

10071007
func (impl *DockerHelperImpl) leaveNodesFromBuildxK8sDriver(ciContext cicxt.CiContext, nodeNames []string) error {
10081008
var err error
1009-
defer func() {
1010-
removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME)
1011-
fmt.Println(util.DEVTRON, " cmd : ", removeCmd)
1012-
execRemoveCmd := impl.GetCommandToExecute(removeCmd)
1013-
_ = impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd)
1014-
1015-
}()
1016-
10171009
for _, node := range nodeNames {
10181010
createCmd := fmt.Sprintf("docker buildx create --name=%s --node=%s --leave", BUILDX_K8S_DRIVER_NAME, node)
10191011
fmt.Println(util.DEVTRON, " cmd : ", createCmd)
10201012
execCreateCmd := impl.GetCommandToExecute(createCmd)
10211013
err = impl.cmdExecutor.RunCommand(ciContext, execCreateCmd)
10221014
if err != nil {
10231015
log.Println(util.DEVTRON, "error in leaving node : ", err)
1024-
return err
1016+
break
10251017
}
10261018
}
1019+
impl.removeBuildxDriver(ciContext) //driver cleanup
10271020
return err
10281021
}
10291022

1023+
func (impl *DockerHelperImpl) removeBuildxDriver(ciContext cicxt.CiContext) {
1024+
removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME)
1025+
fmt.Println(util.DEVTRON, " cmd : ", removeCmd)
1026+
execRemoveCmd := impl.GetCommandToExecute(removeCmd)
1027+
err := impl.cmdExecutor.RunCommand(ciContext, execRemoveCmd)
1028+
if err != nil {
1029+
log.Println("error in executing docker buildx remove command", "err", err)
1030+
//not returning error here as this is just a cleanup job, not making it blocking
1031+
}
1032+
}
1033+
10301034
// this function is deprecated, use cmdExecutor.RunCommand instead
10311035
func (impl *DockerHelperImpl) runCmd(cmd string) (error, *bytes.Buffer) {
10321036
fmt.Println(util.DEVTRON, " cmd : ", cmd)

‎ci-runner/helper/GitCliManager.go‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -53,35 +53,30 @@ const GIT_AKS_PASS = "/git-ask-pass.sh"
5353
const DefaultRemoteName = "origin"
5454

5555
func (impl *GitCliManagerImpl) Fetch(gitContext GitContext, rootDir string) (response, errMsg string, err error) {
56-
5756
log.Println(util.DEVTRON, "git fetch ", "location", rootDir)
5857
cmd := exec.Command("git", "-C", rootDir, "fetch", "origin", "--tags", "--force")
59-
60-
tlsPathInfo, err := git_manager.CreateFilesForTlsData(git_manager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), git_manager.TLS_FILES_DIR)
61-
if err != nil {
62-
//making it non-blocking
63-
log.Println("error encountered in createFilesForTlsData", "err", err)
64-
}
65-
defer git_manager.DeleteTlsFiles(tlsPathInfo)
66-
67-
output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Auth.Username, gitContext.Auth.Password, tlsPathInfo)
68-
log.Println(util.DEVTRON, "fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
69-
return output, errMsg, err
58+
return impl.handleTLSDataAndExecuteCommand(cmd, gitContext)
7059
}
7160

7261
func (impl *GitCliManagerImpl) Checkout(gitContext GitContext, rootDir string, checkout string) (response, errMsg string, err error) {
7362
log.Println(util.DEVTRON, "git checkout ", "location", rootDir)
7463
cmd := exec.Command("git", "-C", rootDir, "checkout", checkout, "--force")
64+
return impl.handleTLSDataAndExecuteCommand(cmd, gitContext)
65+
}
7566

67+
func (impl *GitCliManagerImpl) handleTLSDataAndExecuteCommand(cmd *exec.Cmd, gitContext GitContext) (response, errMsg string, err error) {
7668
tlsPathInfo, err := git_manager.CreateFilesForTlsData(git_manager.BuildTlsData(gitContext.TLSKey, gitContext.TLSCertificate, gitContext.CACert, gitContext.TLSVerificationEnabled), git_manager.TLS_FILES_DIR)
7769
if err != nil {
7870
//making it non-blocking
7971
log.Println("error encountered in createFilesForTlsData", "err", err)
8072
}
81-
defer git_manager.DeleteTlsFiles(tlsPathInfo)
8273

8374
output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Auth.Username, gitContext.Auth.Password, tlsPathInfo)
84-
log.Println(util.DEVTRON, "checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
75+
log.Println(util.DEVTRON, "checkout output", "opt", output, "errMsg", errMsg, "error", err)
76+
77+
//deleting tls files
78+
git_manager.DeleteTlsFiles(tlsPathInfo)
79+
8580
return output, errMsg, err
8681
}
8782

‎ci-runner/util/Util.go‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -235,15 +235,13 @@ func ExecuteWithStageInfoLog(stageName string, stageExecutor func() error) (err
235235
func ExecuteWithStageInfoLogWithMetadata(stageName string, metadata interface{}, stageExecutor func() error) (err error) {
236236
startDockerStageInfo := newStageInfo(stageName).withCurrentStartTime().withMetadata(metadata)
237237
startDockerStageInfo.log()
238-
defer func() {
239-
status := success
240-
if err != nil {
241-
status = failure
242-
}
243-
startDockerStageInfo.withStatus(status).withCurrentEndTime().log()
244-
}()
245-
246-
return stageExecutor()
238+
err = stageExecutor()
239+
status := success
240+
if err != nil {
241+
status = failure
242+
}
243+
startDockerStageInfo.withStatus(status).withCurrentEndTime().log()
244+
return
247245
}
248246

249247
func GenerateBuildkitdContent(host string) string {

0 commit comments

Comments
 (0)