Merge remote-tracking branch 'origin/feat-flux-cd' into feat-flux-cd

Author: iamayushm

internal/sql/repository/chartConfig/PipelineOverrideRepository.go

@@ -23,6 +23,7 @@ import (
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
 	"github.com/devtron-labs/devtron/internal/util"
 	"github.com/devtron-labs/devtron/pkg/sql"
+	util2 "github.com/devtron-labs/devtron/util"
 	"github.com/go-pg/pg"
 	"github.com/go-pg/pg/orm"
 	"github.com/juju/errors"
@@ -68,6 +69,7 @@ type PipelineOverrideRepository interface {
 	GetByPipelineIdAndReleaseNo(pipelineId, releaseNo int) (pipelineOverrides []*PipelineOverride, err error)
 	GetAllRelease(appId, environmentId int) (pipelineOverrides []*PipelineOverride, err error)
 	FindByPipelineTriggerGitHash(gitHash string) (pipelineOverride *PipelineOverride, err error)
+	FindByPipelineLikeTriggerGitHash(gitHash string) (pipelineOverride *PipelineOverride, err error)
 	GetLatestRelease(appId, environmentId int) (pipelineOverrides *PipelineOverride, err error)
 	GetLatestReleaseForAppIds(appIds []int, envId int) (pipelineOverrides []*PipelineConfigOverrideMetadata, err error)
 	FindById(id int) (*PipelineOverride, error)
@@ -288,6 +290,16 @@ func (impl PipelineOverrideRepositoryImpl) FindByPipelineTriggerGitHash(gitHash
 	return pipelineOverride, err
 }
 
+func (impl PipelineOverrideRepositoryImpl) FindByPipelineLikeTriggerGitHash(gitHash string) (pipelineOverride *PipelineOverride, err error) {
+	pipelineOverride = &PipelineOverride{}
+	err = impl.dbConnection.Model(pipelineOverride).
+		Column("pipeline_override.*", "Pipeline", "CiArtifact").
+		Where("pipeline_override.git_hash LIKE ?", util2.GetLIKEClauseQueryParamEnd(gitHash)).
+		Order("id DESC").Limit(1).
+		Select()
+	return pipelineOverride, err
+}
+
 func (impl PipelineOverrideRepositoryImpl) FindById(id int) (*PipelineOverride, error) {
 	var pipelineOverride PipelineOverride
 	err := impl.dbConnection.Model(&pipelineOverride).

pkg/app/status/PipelineStatusTimelineService.go

@@ -93,7 +93,6 @@ type PipelineTimelineDetailDto struct {
 	StatusFetchCount           int                          `json:"statusFetchCount"`
 	WfrStatus                  string                       `json:"wfrStatus"`
 	DeploymentAppDeleteRequest bool                         `json:"deploymentAppDeleteRequest"`
-	DeploymentAppType          string                       `json:"deploymentAppType"`
 }
 
 type PipelineStatusTimelineDto struct {
@@ -294,7 +293,6 @@ func (impl *PipelineStatusTimelineServiceImpl) FetchTimelines(appId, envId, wfrI
 		StatusFetchCount:           statusFetchCount,
 		WfrStatus:                  wfrStatus,
 		DeploymentAppDeleteRequest: wfr.CdWorkflow.Pipeline.DeploymentAppDeleteRequest,
-		DeploymentAppType:          deploymentAppType,
 	}
 	return timelineDetail, nil
 }

pkg/deployment/trigger/devtronApps/deployStageHandlerCodeFlux.go

@@ -276,6 +276,9 @@ func (impl *HandlerServiceImpl) CreateHelmRelease(ctx context.Context, fluxCdSpe
 		},
 		Spec: helmv2.HelmReleaseSpec{
 			Interval: metav1.Duration{Duration: helmReleaseReconcileInterval}, //TODO
+			DriftDetection: &helmv2.DriftDetection{
+				Mode: helmv2.DriftDetectionEnabled,
+			},
 			Chart: &helmv2.HelmChartTemplate{
 				Spec: helmv2.HelmChartTemplateSpec{
 					ReconcileStrategy: "Revision",
@@ -310,6 +313,9 @@ func (impl *HandlerServiceImpl) UpdateHelmRelease(ctx context.Context, fluxCdSpe
 		impl.logger.Errorw("error in getting helm release", "name", name, "namespace", namespace, "err", err)
 		return nil, fmt.Errorf("failed to get HelmRelease: %w", err)
 	}
+	existing.Spec.DriftDetection = &helmv2.DriftDetection{
+		Mode: helmv2.DriftDetectionEnabled,
+	}
 	existing.Spec.Interval = metav1.Duration{Duration: helmReleaseReconcileInterval}
 	existing.Spec.Chart = &helmv2.HelmChartTemplate{
 		Spec: helmv2.HelmChartTemplateSpec{

pkg/workflow/dag/WorkflowDagExecutor.go

@@ -112,12 +112,13 @@ type WorkflowDagExecutor interface {
 	UpdateWorkflowRunnerStatusForDeployment(appIdentifier *helmBean.AppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, skipReleaseNotFound bool) bool
 
 	BuildCiArtifactRequestForWebhook(event pipeline.ExternalCiWebhookDto) (*bean2.CiArtifactWebhookRequest, error)
-	UpdateWorkflowRunnerStatusForFluxDeployment(appIdentifier *bean8.FluxAppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, hash string) bool
+	UpdateWorkflowRunnerStatusForFluxDeployment(appIdentifier *bean8.FluxAppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverride *chartConfig.PipelineOverride) bool
 }
 
 type WorkflowDagExecutorImpl struct {
 	logger                        *zap.SugaredLogger
 	pipelineRepository            pipelineConfig.PipelineRepository
+	pipelineOverrideRepository    chartConfig.PipelineOverrideRepository
 	cdWorkflowRepository          pipelineConfig.CdWorkflowRepository
 	ciArtifactRepository          repository.CiArtifactRepository
 	enforcerUtil                  rbac.EnforcerUtil
@@ -159,6 +160,7 @@ type WorkflowDagExecutorImpl struct {
 }
 
 func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository,
+	pipelineOverrideRepository chartConfig.PipelineOverrideRepository,
 	cdWorkflowRepository pipelineConfig.CdWorkflowRepository,
 	ciArtifactRepository repository.CiArtifactRepository,
 	enforcerUtil rbac.EnforcerUtil,
@@ -193,6 +195,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi
 ) *WorkflowDagExecutorImpl {
 	wde := &WorkflowDagExecutorImpl{logger: Logger,
 		pipelineRepository:            pipelineRepository,
+		pipelineOverrideRepository:    pipelineOverrideRepository,
 		cdWorkflowRepository:          cdWorkflowRepository,
 		ciArtifactRepository:          ciArtifactRepository,
 		enforcerUtil:                  enforcerUtil,
@@ -1299,7 +1302,8 @@ func (impl *WorkflowDagExecutorImpl) BuildCiArtifactRequestForWebhook(event pipe
 	return request, nil
 }
 
-func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForFluxDeployment(appIdentifier *bean8.FluxAppIdentifier, wfr *pipelineConfig.CdWorkflowRunner, hash string) bool {
+func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForFluxDeployment(appIdentifier *bean8.FluxAppIdentifier, wfr *pipelineConfig.CdWorkflowRunner,
+	pipelineOverride *chartConfig.PipelineOverride) bool {
 	fluxAppDetail, err := impl.fluxApplicationService.GetFluxAppDetail(context.Background(), appIdentifier)
 	if err != nil {
 		impl.logger.Errorw("error in getting helm app release status", "appIdentifier", appIdentifier, "err", err)
@@ -1310,7 +1314,7 @@ func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForFluxDeployment
 		wfr.FinishedOn = time.Now()
 		return true
 	}
-	if !matchShortHash(fluxAppDetail.LastObservedVersion, hash) {
+	if !impl.checkIfFluxPipelineEventIsValid(fluxAppDetail.LastObservedVersion, pipelineOverride) {
 		return false
 	}
 	wfr.FinishedOn = time.Now()
@@ -1331,24 +1335,39 @@ func (impl *WorkflowDagExecutorImpl) UpdateWorkflowRunnerStatusForFluxDeployment
 	return true
 }
 
-// matchShortHash compares the short Git hash embedded in the version string
+func (impl *WorkflowDagExecutorImpl) checkIfFluxPipelineEventIsValid(lastObservedVersion string, pipelineOverride *chartConfig.PipelineOverride) bool {
+	gitHash := getShortHash(lastObservedVersion)
+	if !strings.HasPrefix(pipelineOverride.GitHash, gitHash) {
+		pipelineOverrideByHash, err := impl.pipelineOverrideRepository.FindByPipelineLikeTriggerGitHash(gitHash)
+		if err != nil {
+			impl.logger.Errorw("error on update application status", "gitHash", gitHash, "err", err)
+			return false
+		}
+		if pipelineOverrideByHash == nil || pipelineOverrideByHash.CommitTime.Before(pipelineOverride.CommitTime) {
+			// we have received trigger hash which is committed before this apps actual gitHash stored by us
+			// this means that the hash stored by us will be synced later, so we will drop this event
+			return false
+		}
+	}
+	return true
+}
+
+// getShortHash gets the short Git hash embedded in the version string
 // with the beginning of the full Git commit hash.
 //
 // version: expected format like "4.22.1+<shortHash>.<buildNumber>"
 // fullHash: expected to be a full 40-character Git commit SHA
-//
-// Returns true if the short hash is a prefix of the full hash.
-func matchShortHash(version, fullHash string) bool {
+func getShortHash(version string) string {
 	// Split version string at '+' to extract metadata
 	parts := strings.Split(version, "+")
 	if len(parts) < 2 {
-		return false // No metadata found
+		return "" // No metadata found
 	}
 
 	// Metadata might look like "2b6c6b2.2" → short hash + build number
 	metaParts := strings.Split(parts[1], ".")
 	shortHash := metaParts[0] // Take only the short hash before the dot
 
 	// Compare short hash with prefix of full hash
-	return strings.HasPrefix(fullHash, shortHash)
+	return shortHash
 }

pkg/workflow/status/WorkflowStatusService.go

@@ -620,14 +620,16 @@ func (impl *WorkflowStatusServiceImpl) CheckFluxAppStatusPeriodicallyAndUpdateIn
 			Name:           wfr.CdWorkflow.Pipeline.DeploymentAppName,
 			IsKustomizeApp: false,
 		}
+		pipeline := wfr.CdWorkflow.Pipeline
 
-		pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByCdWorkflowId(wfr.CdWorkflowId)
+		// getting latest pipelineOverride for app (by appId and envId)
+		pipelineOverride, err := impl.pipelineOverrideRepository.FindLatestByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId, bean3.FluxCd)
 		if err != nil {
-			impl.logger.Errorw("error in getting latest pipeline override by cdWorkflowId", "err", err, "cdWorkflowId", wfr.CdWorkflowId)
+			impl.logger.Errorw("error in getting latest pipelineOverride by appId and envId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId)
 			return err
 		}
 
-		if isWfrUpdated := impl.workflowDagExecutor.UpdateWorkflowRunnerStatusForFluxDeployment(appIdentifier, wfr, pipelineOverride.GitHash); !isWfrUpdated {
+		if isWfrUpdated := impl.workflowDagExecutor.UpdateWorkflowRunnerStatusForFluxDeployment(appIdentifier, wfr, pipelineOverride); !isWfrUpdated {
 			continue
 		}
 

scripts/devtron-reference-helm-charts/reference-chart_5-2-0/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

scripts/devtron-reference-helm-charts/reference-chart_5-2-0/.image_descriptor_template.json

@@ -0,0 +1 @@
+{"server":{"deployment":{"image_tag":"{{.Tag}}","image":"{{.Name}}"}},"pipelineName": "{{.PipelineName}}","releaseVersion":"{{.ReleaseVersion}}","deploymentType": "{{.DeploymentType}}", "app": "{{.App}}", "env": "{{.Env}}", "appMetrics": {{.AppMetrics}}}

scripts/devtron-reference-helm-charts/reference-chart_5-2-0/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: reference-chart_5-2-0
+version: 5.2.0