From 8dfb77213d804e60d32df205feade0135bfee050 Mon Sep 17 00:00:00 2001 From: Kent Rancourt Date: Tue, 22 Jul 2025 21:10:58 -0400 Subject: [PATCH 1/2] apply app health cool down when desired revisions are unknown Signed-off-by: Kent Rancourt --- internal/health/checker/builtin/argocd.go | 85 +++++++++++-------- .../health/checker/builtin/argocd_test.go | 2 +- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/internal/health/checker/builtin/argocd.go b/internal/health/checker/builtin/argocd.go index c546ad4620..aa9e8e29ed 100644 --- a/internal/health/checker/builtin/argocd.go +++ b/internal/health/checker/builtin/argocd.go @@ -218,49 +218,60 @@ func (a *argocdChecker) getApplicationHealth( return kargoapi.HealthStateUnhealthy, appStatus, errors.Join(issues...) } - if len(desiredRevisions) > 0 { - if stageHealth, err := a.stageHealthForAppSync(app, desiredRevisions); err != nil { - return stageHealth, appStatus, err - } - // If we care about revisions, and recently finished an operation, we - // should wait for a cooldown period before assessing the health of the - // application. This is to ensure the health check has a chance to run - // after the sync operation has finished. - // - // xref: https://github.com/akuity/kargo/issues/2196 - // - // TODO: revisit this when https://github.com/argoproj/argo-cd/pull/18660 - // is merged and released. - if app.Status.OperationState != nil { - cooldown := time.Now() - if !app.Status.OperationState.FinishedAt.IsZero() { - cooldown = app.Status.OperationState.FinishedAt.Time - } - cooldown = cooldown.Add(10 * time.Second) - if duration := time.Until(cooldown); duration > 0 { - time.Sleep(duration) - // Re-fetch the application to get the latest state. - if err := a.argocdClient.Get(ctx, appKey, app); err != nil { - if apierrors.IsNotFound(err) { - err = fmt.Errorf( - "unable to find Argo CD Application %q in namespace %q", - appKey.Name, appKey.Namespace, - ) - } else { - err = fmt.Errorf( - "error finding Argo CD Application %q in namespace %q: %w", - appKey.Name, appKey.Namespace, err, - ) - } - return kargoapi.HealthStateUnknown, appStatus, err + // Argo CD has separate reconciliation loops for operations (like syncing) and + // assessing Application health. This means there is possibly a period in the + // moments after an Application sync has completed that its health status is + // stale. If the stale status indicates the Application is in a Healthy state, + // but would have indicated otherwise had the Health status not been stale, it + // creates the possibility of the Stage prematurely being considered Healthy, + // which can in-turn prompt a verification process to kick of prematurely. To + // work around this, we will not trust Application health if the operation + // state is non-nil and indicates the operation completed fewer than ten + // seconds ago. In such a case, we will pause until ten seconds have passed + // since operation completion. Note: This workaround assumes no/minimal clock + // drift between the Kargo controller and the Argo CD Application controller. + // + // TODO(krancour): This workaround can be revisited if/when + // https://github.com/argoproj/argo-cd/pull/21120 is merged, as (for newer + // versions of Argo CD) it will allow us to accurately determine whether an + // Application's health was last assessed before or after the most recently + // completed operation. + if app.Status.OperationState != nil && !app.Status.OperationState.FinishedAt.IsZero() { + coolDownPeriod := 10 * time.Second + timeDiff := time.Since(app.Status.OperationState.FinishedAt.Time) + if timeDiff < coolDownPeriod { + // Wait the remaining interval for the coolDownPeriod to have elapsed. + time.Sleep(coolDownPeriod - timeDiff) + // Re-fetch the application to get the latest state. + if err := a.argocdClient.Get(ctx, appKey, app); err != nil { + if apierrors.IsNotFound(err) { + err = fmt.Errorf( + "unable to find Argo CD Application %q in namespace %q", + appKey.Name, appKey.Namespace, + ) + } else { + err = fmt.Errorf( + "error finding Argo CD Application %q in namespace %q: %w", + appKey.Name, appKey.Namespace, err, + ) } + return kargoapi.HealthStateUnknown, appStatus, err } } } - // With all the above checks passed, we can now assume the Argo CD - // Application's health state is reliable. + // We can now assume the Argo CD Application's health state is reliable. stageHealth, err := a.stageHealthForAppHealth(app) + if err != nil || stageHealth != kargoapi.HealthStateHealthy { + // If there was an error or the App is not Healthy, we're done. + return stageHealth, appStatus, err + } + + // If we get to here, the App is Healthy and, so far, the Stage appears to be + // as well. If desiredRevisions are known, however, this needs to be factored + // into Stage health. Assess how App sources being synced to the correct or + // incorrect revisions affects overall Stage health: + stageHealth, err = a.stageHealthForAppSync(app, desiredRevisions) return stageHealth, appStatus, err } diff --git a/internal/health/checker/builtin/argocd_test.go b/internal/health/checker/builtin/argocd_test.go index e20a203a1b..0687749874 100644 --- a/internal/health/checker/builtin/argocd_test.go +++ b/internal/health/checker/builtin/argocd_test.go @@ -458,7 +458,7 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { Namespace: testApp.Namespace, Name: testApp.Name, }, - []string{"fake-version", "fake-commit", "another-fake-commit"}, + nil, ) elapsed := time.Since(app.Status.OperationState.FinishedAt.Time) require.NoError(t, err) From 2c96746e9e93bef522e612ac49cf48d3770d4e17 Mon Sep 17 00:00:00 2001 From: Kent Rancourt Date: Wed, 23 Jul 2025 14:05:34 -0400 Subject: [PATCH 2/2] revise approach to trusting/not trusting app health Signed-off-by: Kent Rancourt --- internal/controller/stages/regular_stages.go | 10 +- internal/health/checker/builtin/argocd.go | 108 +++++---- .../health/checker/builtin/argocd_test.go | 213 +++++++++++------- 3 files changed, 204 insertions(+), 127 deletions(-) diff --git a/internal/controller/stages/regular_stages.go b/internal/controller/stages/regular_stages.go index 222dd093f0..437287da16 100644 --- a/internal/controller/stages/regular_stages.go +++ b/internal/controller/stages/regular_stages.go @@ -2,6 +2,7 @@ package stages import ( "context" + "errors" "fmt" "slices" "strings" @@ -418,7 +419,14 @@ func (r *RegularStageReconciler) reconcile( { name: "assessing health", reconcile: func() (kargoapi.StageStatus, error) { - return r.assessHealth(ctx, stage), nil + status := r.assessHealth(ctx, stage) + if status.Health != nil && status.Health.Status == kargoapi.HealthStateUnknown { + // If Stage health evaluated to Unknown, we'll treat it as an error so + // that Stage health will be re-assessed with a progressive backoff. + return status, + errors.New("Stage health evaluated to Unknown") // nolint: staticcheck + } + return status, nil }, }, { diff --git a/internal/health/checker/builtin/argocd.go b/internal/health/checker/builtin/argocd.go index aa9e8e29ed..0e7657759e 100644 --- a/internal/health/checker/builtin/argocd.go +++ b/internal/health/checker/builtin/argocd.go @@ -19,6 +19,8 @@ import ( const applicationStatusesKey = "applicationStatuses" +var appHealthCooldownDuration = 10 * time.Second + // ArgoCDHealthInput is the input for a health check associated with the the // argocd-update step. type ArgoCDHealthInput struct { @@ -161,9 +163,9 @@ var healthErrorConditions = []argocd.ApplicationConditionType{ // getApplicationHealth assesses the health of an Argo CD Application by looking // at its conditions, health status, and sync status. Based on these, it returns -// an overall health state, the Argo CD Application's health status, and its sync -// status. If it can not (fully) assess the health of the Argo CD Application, it -// returns an error with a message explaining why. +// an overall health state and the Argo CD Application's health status. If it +// can not (fully) assess the health of the Argo CD Application, it returns an +// error with a message explaining why. func (a *argocdChecker) getApplicationHealth( ctx context.Context, appKey client.ObjectKey, @@ -200,6 +202,58 @@ func (a *argocdChecker) getApplicationHealth( // Reflect the health and sync status of the Argo CD Application. appStatus.ApplicationStatus = app.Status + if appStatus.OperationState != nil && appStatus.OperationState.Phase != argocd.OperationSucceeded { + // This App is in a transitional state. By Kargo Standards, the Stage cannot + // be Healthy. + return kargoapi.HealthStateUnknown, + appStatus, + fmt.Errorf( + "last operation of Argo CD Application %q in namespace %q has "+ + "status %q; Application health status not trusted", + appKey.Name, + appKey.Namespace, + appStatus.OperationState.Phase, + ) + } + + // Argo CD has separate reconciliation loops for operations (like syncing) and + // assessing App health. This means that in the moments immediately following + // a completed operation, App health may reflect state from PRIOR to the + // operation. If that status indicates the App is in a Healthy state, but + // would have indicated otherwise had it not been stale, it creates the + // possibility of an overly-optimistic assessment of Stage health. If that + // occurs following a Promotion, it can prompt a verification process to kick + // of prematurely. + // + // To work around this, we will not immediately trust App health if the most + // recently completed operation completed fewer than ten seconds ago. In such + // a case, we will deem Stage health Unknown and return an error. This will + // cause the Stage to be queued for re-reconciliation with a progressive + // backoff. This will continue until the App's health status is considered + // reliable. + // + // TODO(krancour): This workaround can be revisited if/when + // https://github.com/argoproj/argo-cd/pull/21120 is merged, as (for newer + // versions of Argo CD, at least) it will allow us to accurately determine + // whether an App's health was last assessed before or after its most recent + // operation completed. + if app.Status.OperationState != nil && app.Status.OperationState.FinishedAt != nil { + if time.Since(app.Status.OperationState.FinishedAt.Time) < appHealthCooldownDuration { + return kargoapi.HealthStateUnknown, + appStatus, + fmt.Errorf( + "last operation of Argo CD Application %q in namespace %q completed "+ + "less than %s ago; Application health status not trusted", + appKey.Name, + appKey.Namespace, + appHealthCooldownDuration, + ) + } + } + + // If we get to here, we assume the Argo CD Application's health state is + // reliable. + // Check for any error conditions. If these are found, the application is // considered unhealthy as they may indicate a problem which can result in // e.g. the health status result to become unreliable. @@ -218,49 +272,6 @@ func (a *argocdChecker) getApplicationHealth( return kargoapi.HealthStateUnhealthy, appStatus, errors.Join(issues...) } - // Argo CD has separate reconciliation loops for operations (like syncing) and - // assessing Application health. This means there is possibly a period in the - // moments after an Application sync has completed that its health status is - // stale. If the stale status indicates the Application is in a Healthy state, - // but would have indicated otherwise had the Health status not been stale, it - // creates the possibility of the Stage prematurely being considered Healthy, - // which can in-turn prompt a verification process to kick of prematurely. To - // work around this, we will not trust Application health if the operation - // state is non-nil and indicates the operation completed fewer than ten - // seconds ago. In such a case, we will pause until ten seconds have passed - // since operation completion. Note: This workaround assumes no/minimal clock - // drift between the Kargo controller and the Argo CD Application controller. - // - // TODO(krancour): This workaround can be revisited if/when - // https://github.com/argoproj/argo-cd/pull/21120 is merged, as (for newer - // versions of Argo CD) it will allow us to accurately determine whether an - // Application's health was last assessed before or after the most recently - // completed operation. - if app.Status.OperationState != nil && !app.Status.OperationState.FinishedAt.IsZero() { - coolDownPeriod := 10 * time.Second - timeDiff := time.Since(app.Status.OperationState.FinishedAt.Time) - if timeDiff < coolDownPeriod { - // Wait the remaining interval for the coolDownPeriod to have elapsed. - time.Sleep(coolDownPeriod - timeDiff) - // Re-fetch the application to get the latest state. - if err := a.argocdClient.Get(ctx, appKey, app); err != nil { - if apierrors.IsNotFound(err) { - err = fmt.Errorf( - "unable to find Argo CD Application %q in namespace %q", - appKey.Name, appKey.Namespace, - ) - } else { - err = fmt.Errorf( - "error finding Argo CD Application %q in namespace %q: %w", - appKey.Name, appKey.Namespace, err, - ) - } - return kargoapi.HealthStateUnknown, appStatus, err - } - } - } - - // We can now assume the Argo CD Application's health state is reliable. stageHealth, err := a.stageHealthForAppHealth(app) if err != nil || stageHealth != kargoapi.HealthStateHealthy { // If there was an error or the App is not Healthy, we're done. @@ -350,8 +361,9 @@ func (a *argocdChecker) stageHealthForAppSync( return kargoapi.HealthStateHealthy, nil } -// stageHealthForAppHealth returns the v1alpha1.HealthState for an Argo CD -// Application based on its health status. +// stageHealthForAppHealth assesses how the specified Argo CD Application's +// health affects Stage heathy. All results apart from Healthy will also include +// an error. func (a *argocdChecker) stageHealthForAppHealth( app *argocd.Application, ) (kargoapi.HealthState, error) { diff --git a/internal/health/checker/builtin/argocd_test.go b/internal/health/checker/builtin/argocd_test.go index 0687749874..0ab16394e1 100644 --- a/internal/health/checker/builtin/argocd_test.go +++ b/internal/health/checker/builtin/argocd_test.go @@ -56,6 +56,12 @@ func Test_argocdUpdater_check(t *testing.T) { Sources: []argocd.ApplicationSource{{}}, }, Status: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Health: argocd.HealthStatus{ Status: argocd.HealthStatusHealthy, }, @@ -63,9 +69,6 @@ func Test_argocdUpdater_check(t *testing.T) { Status: argocd.SyncStatusCodeSynced, Revisions: []string{"fake-version"}, }, - OperationState: &argocd.OperationState{ - FinishedAt: ptr.To(metav1.Now()), - }, }, }, &argocd.Application{ @@ -74,14 +77,16 @@ func Test_argocdUpdater_check(t *testing.T) { Name: testAppName2, }, Status: argocd.ApplicationStatus{ - Conditions: []argocd.ApplicationCondition{ - { - Type: argocd.ApplicationConditionComparisonError, - }, - { - Type: argocd.ApplicationConditionInvalidSpecError, + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), }, }, + Conditions: []argocd.ApplicationCondition{ + {Type: argocd.ApplicationConditionComparisonError}, + {Type: argocd.ApplicationConditionInvalidSpecError}, + }, }, }, ). @@ -110,6 +115,12 @@ func Test_argocdUpdater_check(t *testing.T) { Sources: []argocd.ApplicationSource{{}}, }, Status: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Health: argocd.HealthStatus{ Status: argocd.HealthStatusHealthy, }, @@ -117,9 +128,6 @@ func Test_argocdUpdater_check(t *testing.T) { Status: argocd.SyncStatusCodeSynced, Revisions: []string{"fake-version"}, }, - OperationState: &argocd.OperationState{ - FinishedAt: ptr.To(metav1.Now()), - }, }, }, &argocd.Application{ @@ -131,6 +139,12 @@ func Test_argocdUpdater_check(t *testing.T) { Sources: []argocd.ApplicationSource{{}}, }, Status: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Health: argocd.HealthStatus{ Status: argocd.HealthStatusHealthy, }, @@ -138,9 +152,6 @@ func Test_argocdUpdater_check(t *testing.T) { Status: argocd.SyncStatusCodeSynced, Revisions: []string{"fake-commit"}, }, - OperationState: &argocd.OperationState{ - FinishedAt: ptr.To(metav1.Now()), - }, }, }, ). @@ -266,9 +277,64 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { require.Equal(t, argocd.SyncStatusCodeUnknown, appStatus.Sync.Status) }, }, + { + name: "Application has an in-progress operation", + appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{Phase: argocd.OperationRunning}, + Health: argocd.HealthStatus{Status: argocd.HealthStatusHealthy}, + }, + assertions: func( + t *testing.T, + stageHealth kargoapi.HealthState, + appStatus ArgoCDAppStatus, + err error, + ) { + require.Error(t, err) + require.ErrorContains(t, err, "last operation of Argo CD Application") + require.ErrorContains(t, err, string(argocd.OperationRunning)) + require.ErrorContains(t, err, "Application health status not trusted") + require.Equal(t, kargoapi.HealthStateUnknown, stageHealth) + require.Equal(t, testApp.Namespace, appStatus.Namespace) + require.Equal(t, testApp.Name, appStatus.Name) + require.Equal(t, argocd.HealthStatusHealthy, appStatus.Health.Status) + }, + }, + { + name: "Application's last operation completed recently", + appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1*appHealthCooldownDuration + time.Second), + }, + }, + Health: argocd.HealthStatus{Status: argocd.HealthStatusHealthy}, + }, + assertions: func( + t *testing.T, + stageHealth kargoapi.HealthState, + appStatus ArgoCDAppStatus, + err error, + ) { + require.Error(t, err) + require.ErrorContains(t, err, "last operation of Argo CD Application") + require.ErrorContains(t, err, "completed less than") + require.ErrorContains(t, err, "Application health status not trusted") + require.Equal(t, kargoapi.HealthStateUnknown, stageHealth) + require.Equal(t, testApp.Namespace, appStatus.Namespace) + require.Equal(t, testApp.Name, appStatus.Name) + require.Equal(t, argocd.HealthStatusHealthy, appStatus.Health.Status) + }, + }, { name: "Application has error conditions", appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Conditions: []argocd.ApplicationCondition{ { Type: argocd.ApplicationConditionComparisonError, @@ -307,8 +373,47 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { }, }, { - name: "no error conditions and no desired revisions", + name: "Application is not Healthy", + appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, + Health: argocd.HealthStatus{ + Status: argocd.HealthStatusDegraded, + Message: "fake-message", + }, + Sync: argocd.SyncStatus{ + Status: argocd.SyncStatusCodeSynced, + }, + }, + assertions: func( + t *testing.T, + stageHealth kargoapi.HealthState, + appStatus ArgoCDAppStatus, + err error, + ) { + require.ErrorContains(t, err, "Argo CD Application") + require.ErrorContains(t, err, "has health state") + require.ErrorContains(t, err, string(argocd.HealthStatusDegraded)) + require.Equal(t, kargoapi.HealthStateUnhealthy, stageHealth) + require.Equal(t, testApp.Namespace, appStatus.Namespace) + require.Equal(t, testApp.Name, appStatus.Name) + require.Equal(t, argocd.HealthStatusDegraded, appStatus.Health.Status) + require.Equal(t, argocd.SyncStatusCodeSynced, appStatus.Sync.Status) + }, + }, + { + name: "Application is Healthy and check has no desired revisions", appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Health: argocd.HealthStatus{ Status: argocd.HealthStatusHealthy, Message: "fake-message", @@ -332,8 +437,14 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { }, }, { - name: "no error conditions, but revisions out of sync", + name: "Application is Healthy, but not synced to desired revisions", appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Health: argocd.HealthStatus{ Status: argocd.HealthStatusHealthy, }, @@ -341,9 +452,6 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { Status: argocd.SyncStatusCodeSynced, Revisions: []string{"fake-version", "wrong-fake-commit", "another-fake-commit"}, }, - OperationState: &argocd.OperationState{ - FinishedAt: ptr.To(metav1.Now()), - }, }, desiredRevisions: []string{"fake-version", "fake-commit", "another-fake-commit"}, assertions: func( @@ -363,8 +471,14 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { }, }, { - name: "no error conditions and revisions in sync", + name: "Application is Healthy and synced to desired revisions", appStatus: argocd.ApplicationStatus{ + OperationState: &argocd.OperationState{ + Phase: argocd.OperationSucceeded, + FinishedAt: &metav1.Time{ + Time: time.Now().Add(-1 * appHealthCooldownDuration), + }, + }, Health: argocd.HealthStatus{ Status: argocd.HealthStatusHealthy, }, @@ -372,9 +486,6 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { Status: argocd.SyncStatusCodeSynced, Revisions: []string{"fake-version", "fake-commit", "another-fake-commit"}, }, - OperationState: &argocd.OperationState{ - FinishedAt: &metav1.Time{Time: metav1.Now().Add(-10 * time.Second)}, - }, }, desiredRevisions: []string{"fake-version", "fake-commit", "another-fake-commit"}, assertions: func( @@ -415,60 +526,6 @@ func Test_argocdUpdater_getApplicationHealth(t *testing.T) { testCase.assertions(t, stageHealth, appStatus, err) }) } - - t.Run("waits for operation cooldown", func(t *testing.T) { - app := testApp.DeepCopy() - app.Status = argocd.ApplicationStatus{ - Health: argocd.HealthStatus{ - Status: argocd.HealthStatusProgressing, - }, - Sync: argocd.SyncStatus{ - Status: argocd.SyncStatusCodeSynced, - Revisions: []string{"fake-version", "fake-commit", "another-fake-commit"}, - }, - OperationState: &argocd.OperationState{ - FinishedAt: ptr.To(metav1.Now()), - }, - } - var count int - runner := &argocdChecker{ - argocdClient: fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ - Get: func( - _ context.Context, - _ client.WithWatch, - _ client.ObjectKey, - obj client.Object, - _ ...client.GetOption, - ) error { - count++ - - appCopy := app.DeepCopy() - if count > 1 { - appCopy.Status.Health.Status = argocd.HealthStatusHealthy - } - - *obj.(*argocd.Application) = *appCopy // nolint: forcetypeassert - return nil - }, - }).Build(), - } - _, _, err := runner.getApplicationHealth( - context.Background(), - client.ObjectKey{ - Namespace: testApp.Namespace, - Name: testApp.Name, - }, - nil, - ) - elapsed := time.Since(app.Status.OperationState.FinishedAt.Time) - require.NoError(t, err) - // We wait for 10 seconds after the sync operation has finished. As such, - // the elapsed time should be greater than 8 seconds, but less than 12 - // seconds. To ensure we do not introduce flakes in the tests. - require.Greater(t, elapsed, 8*time.Second) - require.Less(t, elapsed, 12*time.Second) - require.Equal(t, 2, count) - }) } func Test_argocdUpdater_stageHealthForAppSync(t *testing.T) {