From 1418dc7e7ca1840d125f4e36b8675fbd2561d2a1 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 10 Mar 2025 18:49:25 +0530 Subject: [PATCH 1/6] wip: fix gitops metrics --- pkg/deployment/gitOps/git/GitServiceAzure.go | 1 + pkg/deployment/gitOps/git/GitServiceBitbucket.go | 1 + pkg/deployment/gitOps/git/GitServiceGithub.go | 1 + pkg/deployment/gitOps/git/GitServiceGitlab.go | 1 + util/helper.go | 7 ++++++- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/deployment/gitOps/git/GitServiceAzure.go b/pkg/deployment/gitOps/git/GitServiceAzure.go index ff5a6d5dbd..be5c83c427 100644 --- a/pkg/deployment/gitOps/git/GitServiceAzure.go +++ b/pkg/deployment/gitOps/git/GitServiceAzure.go @@ -325,6 +325,7 @@ func (impl GitAzureClient) repoExists(repoName, projectName string) (repoUrl str notFoundStatus := 404 if err != nil { if e, ok := err.(azuredevops.WrappedError); ok && *e.StatusCode == notFoundStatus { + err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status return "", false, isRepoEmpty, nil } else { return "", false, isRepoEmpty, err diff --git a/pkg/deployment/gitOps/git/GitServiceBitbucket.go b/pkg/deployment/gitOps/git/GitServiceBitbucket.go index 243f399e11..31522f1b87 100644 --- a/pkg/deployment/gitOps/git/GitServiceBitbucket.go +++ b/pkg/deployment/gitOps/git/GitServiceBitbucket.go @@ -192,6 +192,7 @@ func (impl GitBitbucketClient) repoExists(repoOptions *bitbucket.RepositoryOptio repo, err := impl.client.Repositories.Repository.Get(repoOptions) if repo == nil && err.Error() == BITBUCKET_REPO_NOT_FOUND_ERROR { + err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status return "", false, nil } if err != nil { diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index 2ddb364e07..4eed4d0d80 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -115,6 +115,7 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { if IsRepoNotFound(err) { repoExists = false + err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status } else { impl.logger.Errorw("error in creating github repo", "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err diff --git a/pkg/deployment/gitOps/git/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go index bcabae1d5b..5a6c48e676 100644 --- a/pkg/deployment/gitOps/git/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -299,6 +299,7 @@ func (impl GitLabClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl stri if err != nil { impl.logger.Debugw("gitlab get project err", "pid", pid, "err", err) if res != nil && res.StatusCode == 404 { + err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status return "", isRepoEmpty, nil } return "", isRepoEmpty, err diff --git a/util/helper.go b/util/helper.go index d2f632fc31..c8d3e00c27 100644 --- a/util/helper.go +++ b/util/helper.go @@ -19,7 +19,9 @@ package util import ( "archive/tar" "compress/gzip" + "context" "encoding/json" + errors2 "errors" "fmt" "github.com/devtron-labs/devtron/internal/middleware" "github.com/juju/errors" @@ -270,7 +272,10 @@ func TriggerCIMetrics(Metrics CIMetrics, exposeCIMetrics bool, PipelineName stri func TriggerGitOpsMetrics(operation string, method string, startTime time.Time, err error) { status := "Success" - if err != nil { + + if err != nil && errors2.Is(err, context.Canceled) { + status = "Failed: context canceled" + } else if err != nil { status = "Failed" } middleware.GitOpsDuration.WithLabelValues(operation, method, status).Observe(time.Since(startTime).Seconds()) From 11acef7dd44a6445009c5bbd41c0f24ba958daba Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 12 Mar 2025 01:05:46 +0530 Subject: [PATCH 2/6] fix pull metrics --- pkg/deployment/gitOps/git/GitServiceAzure.go | 1 - pkg/deployment/gitOps/git/GitServiceBitbucket.go | 1 - pkg/deployment/gitOps/git/GitServiceGitlab.go | 1 - pkg/deployment/gitOps/git/commandManager/GitCliManager.go | 5 ++--- .../gitOps/git/commandManager/GitCommandBaseManager.go | 3 +++ pkg/deployment/gitOps/git/commandManager/utils.go | 8 ++++++++ 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/deployment/gitOps/git/GitServiceAzure.go b/pkg/deployment/gitOps/git/GitServiceAzure.go index be5c83c427..ff5a6d5dbd 100644 --- a/pkg/deployment/gitOps/git/GitServiceAzure.go +++ b/pkg/deployment/gitOps/git/GitServiceAzure.go @@ -325,7 +325,6 @@ func (impl GitAzureClient) repoExists(repoName, projectName string) (repoUrl str notFoundStatus := 404 if err != nil { if e, ok := err.(azuredevops.WrappedError); ok && *e.StatusCode == notFoundStatus { - err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status return "", false, isRepoEmpty, nil } else { return "", false, isRepoEmpty, err diff --git a/pkg/deployment/gitOps/git/GitServiceBitbucket.go b/pkg/deployment/gitOps/git/GitServiceBitbucket.go index 31522f1b87..243f399e11 100644 --- a/pkg/deployment/gitOps/git/GitServiceBitbucket.go +++ b/pkg/deployment/gitOps/git/GitServiceBitbucket.go @@ -192,7 +192,6 @@ func (impl GitBitbucketClient) repoExists(repoOptions *bitbucket.RepositoryOptio repo, err := impl.client.Repositories.Repository.Get(repoOptions) if repo == nil && err.Error() == BITBUCKET_REPO_NOT_FOUND_ERROR { - err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status return "", false, nil } if err != nil { diff --git a/pkg/deployment/gitOps/git/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go index 5a6c48e676..bcabae1d5b 100644 --- a/pkg/deployment/gitOps/git/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -299,7 +299,6 @@ func (impl GitLabClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUrl stri if err != nil { impl.logger.Debugw("gitlab get project err", "pid", pid, "err", err) if res != nil && res.StatusCode == 404 { - err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status return "", isRepoEmpty, nil } return "", isRepoEmpty, err diff --git a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go index b7ca057492..ce8a84079b 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go @@ -21,7 +21,6 @@ import ( git_manager "github.com/devtron-labs/common-lib/git-manager" "github.com/devtron-labs/devtron/util" "os/exec" - "strings" "time" ) @@ -76,12 +75,12 @@ func (impl *GitCliManagerImpl) Pull(ctx GitContext, targetRevision string, repoR if err != nil { return err } + response, errMsg, err := impl.PullCli(ctx, repoRoot, targetRevision) if err != nil { impl.logger.Errorw("error in git pull from cli", "errMsg", errMsg, "err", err) } - - if strings.Contains(response, "already up-to-date") || strings.Contains(errMsg, "already up-to-date") { + if IsAlreadyUpToDateError(response, errMsg) { err = nil return nil } diff --git a/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go b/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go index f10894d1d1..dcdfb605ea 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go @@ -80,6 +80,9 @@ func (impl *GitManagerBaseImpl) ListBranch(ctx GitContext, rootDir string) (resp func (impl *GitManagerBaseImpl) PullCli(ctx GitContext, rootDir string, branch string) (response, errMsg string, err error) { start := time.Now() defer func() { + if IsAlreadyUpToDateError(response, errMsg) { + return + } util.TriggerGitOpsMetrics("Pull", "GitCli", start, err) }() impl.logger.Debugw("git pull ", "location", rootDir) diff --git a/pkg/deployment/gitOps/git/commandManager/utils.go b/pkg/deployment/gitOps/git/commandManager/utils.go index a58a5c9741..fd47d61b8a 100644 --- a/pkg/deployment/gitOps/git/commandManager/utils.go +++ b/pkg/deployment/gitOps/git/commandManager/utils.go @@ -20,6 +20,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" "os" "path/filepath" + "strings" ) func LocateGitRepo(path string) error { @@ -34,3 +35,10 @@ func LocateGitRepo(path string) error { } return nil } + +func IsAlreadyUpToDateError(response, errMsg string) bool { + if strings.Contains(response, "already up-to-date") || strings.Contains(errMsg, "already up-to-date") { + return true + } + return false +} From ac13bc8577d7a3613b99af57a7273e0c1dd626c2 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 12 Mar 2025 01:16:24 +0530 Subject: [PATCH 3/6] github create repo function --- pkg/deployment/gitOps/git/GitServiceGithub.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index 4eed4d0d80..c9f6586fe7 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -106,6 +106,9 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git var err error start := time.Now() defer func() { + if IsRepoNotFound(err) { + return + } globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) }() @@ -115,7 +118,7 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { if IsRepoNotFound(err) { repoExists = false - err = nil // so that TriggerGitOpsMetrics does not get incremented with Failed status + err = nil } else { impl.logger.Errorw("error in creating github repo", "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err From 1947613f5730678a5619527ad6ba869a75a79068 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 12 Mar 2025 19:20:03 +0530 Subject: [PATCH 4/6] wip: metrics changed from defer function --- .../argocdServer/ArgoClientWrapperService.go | 2 +- cmd/external-app/wire_gen.go | 2 +- .../gitOps/git/GitOperationService.go | 31 ++++---------- pkg/deployment/gitOps/git/GitOpsClient.go | 2 +- pkg/deployment/gitOps/git/GitOpsHelper.go | 12 ++++-- pkg/deployment/gitOps/git/GitServiceAzure.go | 30 ++++++++++---- .../gitOps/git/GitServiceBitbucket.go | 34 ++++++++++----- pkg/deployment/gitOps/git/GitServiceGithub.go | 41 +++++++++++-------- pkg/deployment/gitOps/git/GitServiceGitlab.go | 28 +++++++++---- .../git/commandManager/GitCliManager.go | 15 +++---- .../commandManager/GitCommandBaseManager.go | 11 +++-- util/retryFunc/RetryFunction.go | 4 +- 12 files changed, 124 insertions(+), 88 deletions(-) diff --git a/client/argocdServer/ArgoClientWrapperService.go b/client/argocdServer/ArgoClientWrapperService.go index 4b028eede5..4e3b634a13 100644 --- a/client/argocdServer/ArgoClientWrapperService.go +++ b/client/argocdServer/ArgoClientWrapperService.go @@ -348,7 +348,7 @@ func (impl *ArgoClientWrapperServiceImpl) RegisterGitOpsRepoInArgoWithRetry(ctx return nil } - callback := func() error { + callback := func(int) error { return impl.createRepoInArgoCd(ctx, grpcConfig, gitOpsRepoUrl) } argoCdErr := retryFunc.Retry(callback, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index f172415218..0d91a63f76 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/deployment/gitOps/git/GitOperationService.go b/pkg/deployment/gitOps/git/GitOperationService.go index a9b4e1c820..2f96fc0333 100644 --- a/pkg/deployment/gitOps/git/GitOperationService.go +++ b/pkg/deployment/gitOps/git/GitOperationService.go @@ -51,7 +51,6 @@ type GitOperationService interface { PushChartToGitOpsRepoForHelmApp(ctx context.Context, pushChartToGitRequest *bean.PushChartToGitRequestDTO, requirementsConfig, valuesConfig *ChartConfig) (*commonBean.ChartGitAttribute, string, error) CreateRepository(ctx context.Context, dto *apiBean.GitOpsConfigDto, userId int32) (string, bool, bool, error) - GetRepoUrlByRepoName(repoName string) (string, error) GetClonedDir(ctx context.Context, chartDir, repoUrl, targetRevision string) (string, error) ReloadGitOpsProvider() error @@ -170,7 +169,7 @@ func (impl *GitOperationServiceImpl) PushChartToGitRepo(ctx context.Context, git commit, err := impl.gitFactory.GitOpsHelper.CommitAndPushAllChanges(newCtx, clonedDir, targetRevision, "first commit", userName, userEmailId) if err != nil { impl.logger.Errorw("error in pushing git", "err", err) - callback := func() error { + callback := func(int) error { commit, err = impl.updateRepoAndPushAllChanges(newCtx, clonedDir, repoUrl, targetRevision, tempReferenceTemplateDir, dir, userName, userEmailId, impl.gitFactory.GitOpsHelper) return err @@ -264,8 +263,12 @@ func (impl *GitOperationServiceImpl) CommitValues(ctx context.Context, chartGitA return commitHash, commitTime, err } gitOpsConfig := &apiBean.GitOpsConfigDto{BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId} - callback := func() error { - commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(newCtx, chartGitAttr, gitOpsConfig) + callback := func(retriesLeft int) error { + publishStatusConflictError := false + if retriesLeft <= 0 { + publishStatusConflictError = true + } + commitHash, commitTime, err = impl.gitFactory.Client.CommitValues(newCtx, chartGitAttr, gitOpsConfig, publishStatusConflictError) return err } err = retryFunc.Retry(callback, impl.isRetryableGitCommitError, @@ -303,26 +306,6 @@ func (impl *GitOperationServiceImpl) CreateRepository(ctx context.Context, dto * return repoUrl, isNew, isEmpty, nil } -func (impl *GitOperationServiceImpl) GetRepoUrlByRepoName(repoName string) (string, error) { - repoUrl := "" - bitbucketMetadata, err := impl.gitOpsConfigReadService.GetBitbucketMetadata() - if err != nil { - impl.logger.Errorw("error in getting bitbucket metadata", "err", err) - return repoUrl, err - } - dto := &apiBean.GitOpsConfigDto{ - GitRepoName: repoName, - BitBucketWorkspaceId: bitbucketMetadata.BitBucketWorkspaceId, - BitBucketProjectKey: bitbucketMetadata.BitBucketProjectKey, - } - repoUrl, _, err = impl.gitFactory.Client.GetRepoUrl(dto) - if err != nil { - //will allow to continue to persist status on next operation - impl.logger.Errorw("error in getting repo url", "err", err, "repoName", repoName) - } - return repoUrl, nil -} - // PushChartToGitOpsRepoForHelmApp pushes built chart to GitOps repo (Specific implementation for Helm Apps) // TODO refactoring: Make a common method for both PushChartToGitRepo and PushChartToGitOpsRepoForHelmApp func (impl *GitOperationServiceImpl) PushChartToGitOpsRepoForHelmApp(ctx context.Context, pushChartToGitRequest *bean.PushChartToGitRequestDTO, requirementsConfig, valuesConfig *ChartConfig) (*commonBean.ChartGitAttribute, string, error) { diff --git a/pkg/deployment/gitOps/git/GitOpsClient.go b/pkg/deployment/gitOps/git/GitOpsClient.go index a08047e525..dc622edd4b 100644 --- a/pkg/deployment/gitOps/git/GitOpsClient.go +++ b/pkg/deployment/gitOps/git/GitOpsClient.go @@ -30,7 +30,7 @@ import ( type GitOpsClient interface { CreateRepository(ctx context.Context, config *gitOps.GitOpsConfigDto) (url string, isNew bool, isEmpty bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) - CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *gitOps.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) + CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *gitOps.GitOpsConfigDto, publishStatusConflictError bool) (commitHash string, commitTime time.Time, err error) GetRepoUrl(config *gitOps.GitOpsConfigDto) (repoUrl string, isRepoEmpty bool, err error) DeleteRepository(config *gitOps.GitOpsConfigDto) error CreateReadme(ctx context.Context, config *gitOps.GitOpsConfigDto) (string, error) diff --git a/pkg/deployment/gitOps/git/GitOpsHelper.go b/pkg/deployment/gitOps/git/GitOpsHelper.go index 9c5d02c7d4..a817445a46 100644 --- a/pkg/deployment/gitOps/git/GitOpsHelper.go +++ b/pkg/deployment/gitOps/git/GitOpsHelper.go @@ -97,12 +97,14 @@ func (impl *GitOpsHelper) Clone(url, targetDir, targetRevision string) (clonedDi func (impl *GitOpsHelper) Pull(repoRoot, targetRevision string) (err error) { start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("Pull", "GitService", start, err) - }() ctx := git.BuildGitContext(context.Background()).WithCredentials(impl.Auth). WithTLSData(impl.tlsConfig.CaData, impl.tlsConfig.TLSKeyData, impl.tlsConfig.TLSCertData, impl.isTlsEnabled) - return impl.gitCommandManager.Pull(ctx, targetRevision, repoRoot) + err = impl.gitCommandManager.Pull(ctx, targetRevision, repoRoot) + if err != nil { + util.TriggerGitOpsMetrics("Pull", "GitService", start, err) + return err + } + return nil } const PushErrorMessage = "failed to push some refs" @@ -129,8 +131,10 @@ func (impl *GitOpsHelper) pullFromBranch(ctx git.GitContext, rootDir, targetRevi impl.logger.Warnw("no branch found in git repo", "rootDir", rootDir) return "", "", err } + start := time.Now() response, errMsg, err := impl.gitCommandManager.PullCli(ctx, rootDir, branch) if err != nil { + util.TriggerGitOpsMetrics("Pull", "GitCli", start, err) impl.logger.Errorw("error on git pull", "branch", branch, "err", err) return response, errMsg, err } diff --git a/pkg/deployment/gitOps/git/GitServiceAzure.go b/pkg/deployment/gitOps/git/GitServiceAzure.go index ff5a6d5dbd..323196e220 100644 --- a/pkg/deployment/gitOps/git/GitServiceAzure.go +++ b/pkg/deployment/gitOps/git/GitServiceAzure.go @@ -107,19 +107,18 @@ func (impl GitAzureClient) CreateRepository(ctx context.Context, config *bean2.G repoExists bool ) start := time.Now() - defer func() { - globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) - }() detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error) url, repoExists, isEmpty, err = impl.repoExists(config.GitRepoName, impl.project) if err != nil { impl.logger.Errorw("error in communication with azure", "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) return "", false, isEmpty, detailedErrorGitOpsConfigActions } if repoExists { detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, GetRepoUrlStage) + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, nil) return url, false, isEmpty, detailedErrorGitOpsConfigActions } gitRepositoryCreateOptions := git.GitRepositoryCreateOptions{ @@ -138,6 +137,7 @@ func (impl GitAzureClient) CreateRepository(ctx context.Context, config *bean2.G impl.logger.Errorw("error in communication with azure", "err", err) } if err != nil || !repoExists { + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } } @@ -146,11 +146,14 @@ func (impl GitAzureClient) CreateRepository(ctx context.Context, config *bean2.G validated, err := impl.ensureProjectAvailabilityOnHttp(config.GitRepoName) if err != nil { impl.logger.Errorw("error in ensuring project availability azure", "project", config.GitRepoName, "err", err) + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err return *operationReference.WebUrl, true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneHttpStage) @@ -159,6 +162,7 @@ func (impl GitAzureClient) CreateRepository(ctx context.Context, config *bean2.G if err != nil { impl.logger.Errorw("error in creating readme azure", "project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CreateReadmeStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) return *operationReference.WebUrl, true, isEmpty, detailedErrorGitOpsConfigActions } isEmpty = false //As we have created readme, repo is no longer empty @@ -167,13 +171,17 @@ func (impl GitAzureClient) CreateRepository(ctx context.Context, config *bean2.G if err != nil { impl.logger.Errorw("error in ensuring project availability azure", "project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) return *operationReference.WebUrl, true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneSshStage) + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitAzureClient", start, nil) return *operationReference.WebUrl, true, isEmpty, detailedErrorGitOpsConfigActions } @@ -196,14 +204,15 @@ func (impl GitAzureClient) CreateReadme(ctx context.Context, config *bean2.GitOp UserName: config.Username, UserEmailId: config.UserEmailId, } - hash, _, err := impl.CommitValues(ctx, cfg, config) + hash, _, err := impl.CommitValues(ctx, cfg, config, true) if err != nil { impl.logger.Errorw("error in creating readme azure", "repo", config.GitRepoName, "err", err) } return hash, err } -func (impl GitAzureClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) { +func (impl GitAzureClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto, publishStatusConflictError bool) (commitHash string, commitTime time.Time, err error) { + start := time.Now() branch := config.TargetRevision if len(branch) == 0 { branch = globalUtil.GetDefaultTargetRevision() @@ -229,6 +238,7 @@ func (impl GitAzureClient) CommitValues(ctx context.Context, config *ChartConfig if err != nil { if e, ok := err.(azuredevops.WrappedError); !ok || *e.StatusCode >= 500 { impl.logger.Errorw("error in fetching branch from azure devops", "err", err) + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitAzureClient", start, err) return "", time.Time{}, err } } else if branchStat != nil { @@ -236,6 +246,7 @@ func (impl GitAzureClient) CommitValues(ctx context.Context, config *ChartConfig } } else { impl.logger.Errorw("error in fetching file from azure devops", "err", err) + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitAzureClient", start, err) return "", time.Time{}, err } } else { @@ -292,9 +303,13 @@ func (impl GitAzureClient) CommitValues(ctx context.Context, config *ChartConfig }) if e := (azuredevops.WrappedError{}); errors.As(err, &e) && e.StatusCode != nil && *e.StatusCode == http2.StatusConflict { impl.logger.Warn("conflict found in commit azure", "err", err, "config", config) + if publishStatusConflictError { + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitAzureClient", start, err) + } return "", time.Time{}, retryFunc.NewRetryableError(err) } else if err != nil { impl.logger.Errorw("error in commit azure", "err", err) + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitAzureClient", start, err) return "", time.Time{}, err } //gitPush.Commits @@ -305,6 +320,7 @@ func (impl GitAzureClient) CommitValues(ctx context.Context, config *ChartConfig commitAuthorTime = (*push.Commits)[0].Author.Date.Time } // push.Commits[0].CommitId + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitAzureClient", start, nil) return commitId, commitAuthorTime, nil } diff --git a/pkg/deployment/gitOps/git/GitServiceBitbucket.go b/pkg/deployment/gitOps/git/GitServiceBitbucket.go index 243f399e11..ec7f371100 100644 --- a/pkg/deployment/gitOps/git/GitServiceBitbucket.go +++ b/pkg/deployment/gitOps/git/GitServiceBitbucket.go @@ -106,9 +106,6 @@ func (impl GitBitbucketClient) GetRepoUrl(config *bean2.GitOpsConfigDto) (repoUr func (impl GitBitbucketClient) CreateRepository(ctx context.Context, config *bean2.GitOpsConfigDto) (url string, isNew bool, isEmpty bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) { var err error start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) - }() detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error) @@ -127,10 +124,12 @@ func (impl GitBitbucketClient) CreateRepository(ctx context.Context, config *bea if err != nil { impl.logger.Errorw("error in communication with bitbucket", "repoOptions", repoOptions, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", false, isEmpty, detailedErrorGitOpsConfigActions } if repoExists { detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, GetRepoUrlStage) + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, nil) return repoUrl, false, isEmpty, detailedErrorGitOpsConfigActions } _, err = impl.client.Repositories.Repository.Create(repoOptions) @@ -142,6 +141,7 @@ func (impl GitBitbucketClient) CreateRepository(ctx context.Context, config *bea impl.logger.Errorw("error in creating repo bitbucket", "repoOptions", repoOptions, "err", err) } if err != nil || !repoExists { + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } } @@ -153,10 +153,13 @@ func (impl GitBitbucketClient) CreateRepository(ctx context.Context, config *bea if err != nil { impl.logger.Errorw("error in ensuring project availability bitbucket", "repoName", repoOptions.RepoSlug, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneHttpStage) @@ -165,6 +168,7 @@ func (impl GitBitbucketClient) CreateRepository(ctx context.Context, config *bea if err != nil { impl.logger.Errorw("error in creating readme bitbucket", "repoName", repoOptions.RepoSlug, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CreateReadmeStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CreateReadmeStage) @@ -173,13 +177,17 @@ func (impl GitBitbucketClient) CreateRepository(ctx context.Context, config *bea if err != nil { impl.logger.Errorw("error in ensuring project availability bitbucket", "project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneSshStage) + util.TriggerGitOpsMetrics("CreateRepository", "GitBitbucketClient", start, nil) return repoUrl, true, isEmpty, detailedErrorGitOpsConfigActions } @@ -242,7 +250,7 @@ func (impl GitBitbucketClient) CreateReadme(ctx context.Context, config *bean2.G UserEmailId: config.UserEmailId, } cfg.SetBitBucketBaseDir(getDir()) - hash, _, err := impl.CommitValues(ctx, cfg, config) + hash, _, err := impl.CommitValues(ctx, cfg, config, false) if err != nil { impl.logger.Errorw("error in creating readme bitbucket", "repo", config.GitRepoName, "err", err) } @@ -270,16 +278,14 @@ func (impl GitBitbucketClient) cleanUp(cloneDir string) { } } -func (impl GitBitbucketClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) { +func (impl GitBitbucketClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto, publishStatusConflictError bool) (commitHash string, commitTime time.Time, err error) { start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) - }() homeDir, err := os.UserHomeDir() if err != nil { impl.logger.Errorw("error in getting home dir", "err", err) + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) return "", time.Time{}, err } bitbucketGitOpsDirPath := path.Join(homeDir, BITBUCKET_GITOPS_DIR, config.GetBitBucketBaseDir()) @@ -294,6 +300,7 @@ func (impl GitBitbucketClient) CommitValues(ctx context.Context, config *ChartCo err = ioutil.WriteFile(bitbucketCommitFilePath, []byte(config.FileContent), 0666) if err != nil { + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) impl.logger.Errorw("error in writing bitbucket commit file", "bitbucketCommitFilePath", bitbucketCommitFilePath, "err", err) return "", time.Time{}, err } @@ -319,8 +326,12 @@ func (impl GitBitbucketClient) CommitValues(ctx context.Context, config *ChartCo if err != nil { impl.logger.Errorw("error in committing file to bitbucket", "repoWriteOptions", repoWriteOptions, "err", err) if e := (&bitbucket.UnexpectedResponseStatusError{}); errors.As(err, &e) && strings.Contains(e.Error(), "500 Internal Server Error") { + if publishStatusConflictError { + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) + } return "", time.Time{}, retryFunc.NewRetryableError(err) } + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) return "", time.Time{}, err } commitOptions := &bitbucket.CommitsOptions{ @@ -330,6 +341,7 @@ func (impl GitBitbucketClient) CommitValues(ctx context.Context, config *ChartCo } commits, err := impl.client.Repositories.Commits.GetCommits(commitOptions) if err != nil { + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) impl.logger.Errorw("error in getting commits from bitbucket", "commitOptions", commitOptions, "err", err) return "", time.Time{}, err } @@ -339,8 +351,10 @@ func (impl GitBitbucketClient) CommitValues(ctx context.Context, config *ChartCo commitTimeString := commits.(map[string]interface{})["values"].([]interface{})[0].(map[string]interface{})["date"].(string) commitTime, err = time.Parse(time.RFC3339, commitTimeString) if err != nil { + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, err) impl.logger.Errorw("error in getting commitTime", "err", err) return "", time.Time{}, err } + util.TriggerGitOpsMetrics("CommitValues", "GitBitbucketClient", start, nil) return commitHash, commitTime, nil } diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index c9f6586fe7..5efbb6ae3e 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -103,14 +103,10 @@ func IsRepoNotFound(err error) bool { } func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.GitOpsConfigDto) (url string, isNew bool, isEmpty bool, detailedErrorGitOpsConfigActions DetailedErrorGitOpsConfigActions) { + var err error + start := time.Now() - defer func() { - if IsRepoNotFound(err) { - return - } - globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) - }() detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error) repoExists := true @@ -118,15 +114,16 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { if IsRepoNotFound(err) { repoExists = false - err = nil } else { impl.logger.Errorw("error in creating github repo", "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) return "", false, isEmpty, detailedErrorGitOpsConfigActions } } if repoExists { detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, GetRepoUrlStage) + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, nil) return url, false, isEmpty, detailedErrorGitOpsConfigActions } private := true @@ -138,15 +135,16 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git // Visibility: &visibility, }) if err1 != nil { - impl.logger.Errorw("error in creating github repo, ", "repo", config.GitRepoName, "err", err) - + impl.logger.Errorw("error in creating github repo, ", "repo", config.GitRepoName, "err", err1) url, isEmpty, err = impl.GetRepoUrl(config) if err != nil { impl.logger.Errorw("error in getting github repo", "repo", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CreateRepoStage] = err1 + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err1) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, GetRepoUrlStage) + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, nil) return url, false, isEmpty, detailedErrorGitOpsConfigActions } impl.logger.Infow("github repo created ", "r", r.CloneURL) @@ -156,10 +154,13 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in ensuring project availability github", "project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) return *r.CloneURL, true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneHttpStage) @@ -168,6 +169,7 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in creating readme github", "project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CreateReadmeStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) return *r.CloneURL, true, isEmpty, detailedErrorGitOpsConfigActions } isEmpty = false //As we have created readme, repo is no longer empty @@ -177,14 +179,18 @@ func (impl GitHubClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in ensuring project availability github", "project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) return *r.CloneURL, true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneSshStage) //_, err = impl.createReadme(name) + globalUtil.TriggerGitOpsMetrics("CreateRepository", "GitHubClient", start, nil) return *r.CloneURL, true, isEmpty, detailedErrorGitOpsConfigActions } @@ -206,19 +212,16 @@ func (impl GitHubClient) CreateReadme(ctx context.Context, config *bean2.GitOpsC UserName: config.Username, UserEmailId: config.UserEmailId, } - hash, _, err := impl.CommitValues(ctx, cfg, config) + hash, _, err := impl.CommitValues(ctx, cfg, config, false) if err != nil { impl.logger.Errorw("error in creating readme github", "repo", config.GitRepoName, "err", err) } return hash, err } -func (impl GitHubClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) { +func (impl GitHubClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto, publishStatusConflictErrorMetrics bool) (commitHash string, commitTime time.Time, err error) { start := time.Now() - defer func() { - globalUtil.TriggerGitOpsMetrics("CommitValues", "GitHubClient", start, err) - }() branch := config.TargetRevision if len(branch) == 0 { @@ -231,6 +234,7 @@ func (impl GitHubClient) CommitValues(ctx context.Context, config *ChartConfig, responseErr, ok := err.(*github.ErrorResponse) if !ok || responseErr.Response.StatusCode != 404 { impl.logger.Errorw("error in creating repo github", "err", err, "config", config) + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitHubClient", start, err) return "", time.Time{}, err } else { newFile = true @@ -260,15 +264,20 @@ func (impl GitHubClient) CommitValues(ctx context.Context, config *ChartConfig, c, httpRes, err := impl.client.Repositories.CreateFile(ctx, impl.org, config.ChartRepoName, path, options) if err != nil && httpRes != nil && httpRes.StatusCode == http2.StatusConflict { impl.logger.Warn("conflict found in commit github", "err", err, "config", config) + if publishStatusConflictErrorMetrics { + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitHubClient", start, err) + } return "", time.Time{}, retryFunc.NewRetryableError(err) } else if err != nil { impl.logger.Errorw("error in commit github", "err", err, "config", config) + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitHubClient", start, err) return "", time.Time{}, err } commitTime = time.Now() // default is current time, if found then will get updated accordingly if c != nil && c.Commit.Author != nil { commitTime = *c.Commit.Author.Date } + globalUtil.TriggerGitOpsMetrics("CommitValues", "GitHubClient", start, nil) return *c.SHA, commitTime, nil } diff --git a/pkg/deployment/gitOps/git/GitServiceGitlab.go b/pkg/deployment/gitOps/git/GitServiceGitlab.go index bcabae1d5b..0f0225f6cf 100644 --- a/pkg/deployment/gitOps/git/GitServiceGitlab.go +++ b/pkg/deployment/gitOps/git/GitServiceGitlab.go @@ -135,9 +135,6 @@ func (impl GitLabClient) CreateRepository(ctx context.Context, config *bean2.Git repoUrl string ) start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) - }() detailedErrorGitOpsConfigActions.StageErrorMap = make(map[string]error) impl.logger.Debugw("gitlab app create request ", "name", config.GitRepoName, "description", config.Description) @@ -145,10 +142,12 @@ func (impl GitLabClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in getting repo url ", "gitlab project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[GetRepoUrlStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", false, isEmpty, detailedErrorGitOpsConfigActions } if len(repoUrl) > 0 { detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, GetRepoUrlStage) + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, nil) return repoUrl, false, isEmpty, detailedErrorGitOpsConfigActions } else { url, err = impl.createProject(config.GitRepoName, config.Description) @@ -159,6 +158,7 @@ func (impl GitLabClient) CreateRepository(ctx context.Context, config *bean2.Git impl.logger.Errorw("error in getting repo url ", "gitlab project", config.GitRepoName, "err", err) } if err != nil || len(repoUrl) == 0 { + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } } @@ -169,10 +169,13 @@ func (impl GitLabClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in ensuring project availability ", "gitlab project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneHttpStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneHttpStage) @@ -180,6 +183,7 @@ func (impl GitLabClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in creating readme ", "gitlab project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CreateReadmeStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } isEmpty = false //As we have created readme, repo is no longer empty @@ -188,13 +192,17 @@ func (impl GitLabClient) CreateRepository(ctx context.Context, config *bean2.Git if err != nil { impl.logger.Errorw("error in ensuring project availability ", "gitlab project", config.GitRepoName, "err", err) detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } if !validated { - detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + err = fmt.Errorf("unable to validate project:%s in given time", config.GitRepoName) + detailedErrorGitOpsConfigActions.StageErrorMap[CloneSshStage] = err + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, err) return "", true, isEmpty, detailedErrorGitOpsConfigActions } detailedErrorGitOpsConfigActions.SuccessfulStages = append(detailedErrorGitOpsConfigActions.SuccessfulStages, CloneSshStage) + util.TriggerGitOpsMetrics("CreateRepository", "GitLabClient", start, nil) return url, true, isEmpty, detailedErrorGitOpsConfigActions } @@ -344,12 +352,9 @@ func (impl GitLabClient) checkIfFileExists(projectName, ref, file string) (exist return err == nil, err } -func (impl GitLabClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto) (commitHash string, commitTime time.Time, err error) { +func (impl GitLabClient) CommitValues(ctx context.Context, config *ChartConfig, gitOpsConfig *bean2.GitOpsConfigDto, publishStatusConflictError bool) (commitHash string, commitTime time.Time, err error) { start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("CommitValues", "GitLabClient", start, err) - }() branch := config.TargetRevision if len(branch) == 0 { @@ -374,13 +379,18 @@ func (impl GitLabClient) CommitValues(ctx context.Context, config *ChartConfig, config.ChartRepoName), actions, gitlab.WithContext(ctx)) if err != nil && httpRes != nil && httpRes.StatusCode == http.StatusBadRequest { impl.logger.Warn("conflict found in commit gitlab", "err", err, "config", config) + if publishStatusConflictError { + util.TriggerGitOpsMetrics("CommitValues", "GitLabClient", start, err) + } return "", time.Time{}, retryFunc.NewRetryableError(err) } else if err != nil { + util.TriggerGitOpsMetrics("CommitValues", "GitLabClient", start, err) return "", time.Time{}, err } commitTime = time.Now() //default is current time, if found then will get updated accordingly if c != nil { commitTime = *c.AuthoredDate } + util.TriggerGitOpsMetrics("CommitValues", "GitLabClient", start, nil) return c.ID, commitTime, err } diff --git a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go index ce8a84079b..3fc69abf75 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go @@ -66,24 +66,25 @@ func (impl *GitCliManagerImpl) CommitAndPush(ctx GitContext, repoRoot, targetRev } func (impl *GitCliManagerImpl) Pull(ctx GitContext, targetRevision string, repoRoot string) (err error) { + start := time.Now() - defer func() { - util.TriggerGitOpsMetrics("Pull", "GitService", start, err) - }() err = LocateGitRepo(repoRoot) if err != nil { + util.TriggerGitOpsMetrics("Pull", "GitService", start, err) return err } response, errMsg, err := impl.PullCli(ctx, repoRoot, targetRevision) if err != nil { + if IsAlreadyUpToDateError(response, errMsg) { + err = nil + return nil + } else { + util.TriggerGitOpsMetrics("Pull", "GitService", start, err) + } impl.logger.Errorw("error in git pull from cli", "errMsg", errMsg, "err", err) } - if IsAlreadyUpToDateError(response, errMsg) { - err = nil - return nil - } return err } diff --git a/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go b/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go index dcdfb605ea..4517c32012 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go @@ -79,12 +79,6 @@ func (impl *GitManagerBaseImpl) ListBranch(ctx GitContext, rootDir string) (resp func (impl *GitManagerBaseImpl) PullCli(ctx GitContext, rootDir string, branch string) (response, errMsg string, err error) { start := time.Now() - defer func() { - if IsAlreadyUpToDateError(response, errMsg) { - return - } - util.TriggerGitOpsMetrics("Pull", "GitCli", start, err) - }() impl.logger.Debugw("git pull ", "location", rootDir) cmd, cancel := impl.createCmdWithContext(ctx, "git", "-C", rootDir, "pull", "origin", branch, "--force") defer cancel() @@ -95,6 +89,11 @@ func (impl *GitManagerBaseImpl) PullCli(ctx GitContext, rootDir string, branch s } defer git_manager.DeleteTlsFiles(tlsPathInfo) output, errMsg, err := impl.runCommandWithCred(cmd, ctx.auth, tlsPathInfo) + if err != nil { + if !IsAlreadyUpToDateError(response, errMsg) { + util.TriggerGitOpsMetrics("Pull", "GitCli", start, err) + } + } impl.logger.Debugw("pull output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } diff --git a/util/retryFunc/RetryFunction.go b/util/retryFunc/RetryFunction.go index f59e803104..55f9ca431a 100644 --- a/util/retryFunc/RetryFunction.go +++ b/util/retryFunc/RetryFunction.go @@ -25,7 +25,7 @@ import ( ) // Retry performs a function with retries, delay, and a max number of attempts. -func Retry(fn func() error, shouldRetry func(err error) bool, maxRetries int, delay time.Duration, logger *zap.SugaredLogger) error { +func Retry(fn func(retriesLeft int) error, shouldRetry func(err error) bool, maxRetries int, delay time.Duration, logger *zap.SugaredLogger) error { var err error logger.Debugw("retrying function", "maxRetries", maxRetries, "delay", delay, @@ -33,7 +33,7 @@ func Retry(fn func() error, shouldRetry func(err error) bool, maxRetries int, de "path", fmt.Sprintf("%s:%d", runTime.GetCallerFileName(), runTime.GetCallerLineNumber())) for i := 0; i < maxRetries; i++ { logger.Debugw("function called with retry", "attempt", i+1, "maxRetries", maxRetries, "delay", delay) - err = fn() + err = fn(maxRetries - (i + 1)) if err == nil { return nil } From 807f7ef3232ebab40517743df6845d02448e5837 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 12 Mar 2025 19:54:25 +0530 Subject: [PATCH 5/6] fix metrics publish --- pkg/deployment/gitOps/git/GitServiceBitbucket.go | 2 +- pkg/deployment/gitOps/git/GitServiceGithub.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/deployment/gitOps/git/GitServiceBitbucket.go b/pkg/deployment/gitOps/git/GitServiceBitbucket.go index ec7f371100..4d3adc58f8 100644 --- a/pkg/deployment/gitOps/git/GitServiceBitbucket.go +++ b/pkg/deployment/gitOps/git/GitServiceBitbucket.go @@ -250,7 +250,7 @@ func (impl GitBitbucketClient) CreateReadme(ctx context.Context, config *bean2.G UserEmailId: config.UserEmailId, } cfg.SetBitBucketBaseDir(getDir()) - hash, _, err := impl.CommitValues(ctx, cfg, config, false) + hash, _, err := impl.CommitValues(ctx, cfg, config, true) if err != nil { impl.logger.Errorw("error in creating readme bitbucket", "repo", config.GitRepoName, "err", err) } diff --git a/pkg/deployment/gitOps/git/GitServiceGithub.go b/pkg/deployment/gitOps/git/GitServiceGithub.go index 5efbb6ae3e..7589b13a02 100644 --- a/pkg/deployment/gitOps/git/GitServiceGithub.go +++ b/pkg/deployment/gitOps/git/GitServiceGithub.go @@ -212,7 +212,7 @@ func (impl GitHubClient) CreateReadme(ctx context.Context, config *bean2.GitOpsC UserName: config.Username, UserEmailId: config.UserEmailId, } - hash, _, err := impl.CommitValues(ctx, cfg, config, false) + hash, _, err := impl.CommitValues(ctx, cfg, config, true) if err != nil { impl.logger.Errorw("error in creating readme github", "repo", config.GitRepoName, "err", err) } From 9782a71132ab265645e1d236ef9e608351920a7c Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 13 Mar 2025 13:22:15 +0530 Subject: [PATCH 6/6] self review changes --- pkg/deployment/gitOps/git/GitOpsHelper.go | 2 ++ pkg/deployment/gitOps/git/commandManager/GitCliManager.go | 1 + .../gitOps/git/commandManager/GitCommandBaseManager.go | 1 + 3 files changed, 4 insertions(+) diff --git a/pkg/deployment/gitOps/git/GitOpsHelper.go b/pkg/deployment/gitOps/git/GitOpsHelper.go index a817445a46..2c9b51b8d6 100644 --- a/pkg/deployment/gitOps/git/GitOpsHelper.go +++ b/pkg/deployment/gitOps/git/GitOpsHelper.go @@ -104,6 +104,7 @@ func (impl *GitOpsHelper) Pull(repoRoot, targetRevision string) (err error) { util.TriggerGitOpsMetrics("Pull", "GitService", start, err) return err } + util.TriggerGitOpsMetrics("Pull", "GitService", start, nil) return nil } @@ -138,6 +139,7 @@ func (impl *GitOpsHelper) pullFromBranch(ctx git.GitContext, rootDir, targetRevi impl.logger.Errorw("error on git pull", "branch", branch, "err", err) return response, errMsg, err } + util.TriggerGitOpsMetrics("Pull", "GitCli", start, nil) return response, errMsg, err } diff --git a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go index 3fc69abf75..d8ff2ba33f 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCliManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCliManager.go @@ -85,6 +85,7 @@ func (impl *GitCliManagerImpl) Pull(ctx GitContext, targetRevision string, repoR } impl.logger.Errorw("error in git pull from cli", "errMsg", errMsg, "err", err) } + util.TriggerGitOpsMetrics("Pull", "GitService", start, nil) return err } diff --git a/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go b/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go index 4517c32012..c18586466b 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCommandBaseManager.go @@ -94,6 +94,7 @@ func (impl *GitManagerBaseImpl) PullCli(ctx GitContext, rootDir string, branch s util.TriggerGitOpsMetrics("Pull", "GitCli", start, err) } } + util.TriggerGitOpsMetrics("Pull", "GitCli", start, nil) impl.logger.Debugw("pull output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err }