diff --git a/applicationset/controllers/applicationset_controller.go b/applicationset/controllers/applicationset_controller.go index 150797b2fb901..253f2ec6685ec 100644 --- a/applicationset/controllers/applicationset_controller.go +++ b/applicationset/controllers/applicationset_controller.go @@ -92,6 +92,7 @@ type ApplicationSetReconciler struct { GlobalPreservedAnnotations []string GlobalPreservedLabels []string Metrics *metrics.ApplicationsetMetrics + MaxResourcesStatusCount int } // +kubebuilder:rbac:groups=argoproj.io,resources=applicationsets,verbs=get;list;watch;create;update;patch;delete @@ -1310,6 +1311,11 @@ func (r *ApplicationSetReconciler) updateResourcesStatus(ctx context.Context, lo sort.Slice(statuses, func(i, j int) bool { return statuses[i].Name < statuses[j].Name }) + + if r.MaxResourcesStatusCount > 0 && len(statuses) > r.MaxResourcesStatusCount { + logCtx.Warnf("Truncating ApplicationSet %s resource status from %d to max allowed %d entries", appset.Name, len(statuses), r.MaxResourcesStatusCount) + statuses = statuses[:r.MaxResourcesStatusCount] + } appset.Status.Resources = statuses // DefaultRetry will retry 5 times with a backoff factor of 1, jitter of 0.1 and a duration of 10ms err := retry.RetryOnConflict(retry.DefaultRetry, func() error { diff --git a/applicationset/controllers/applicationset_controller_test.go b/applicationset/controllers/applicationset_controller_test.go index 13c809ce65e1e..cfcb1780df51d 100644 --- a/applicationset/controllers/applicationset_controller_test.go +++ b/applicationset/controllers/applicationset_controller_test.go @@ -6117,10 +6117,11 @@ func TestUpdateResourceStatus(t *testing.T) { require.NoError(t, err) for _, cc := range []struct { - name string - appSet v1alpha1.ApplicationSet - apps []v1alpha1.Application - expectedResources []v1alpha1.ResourceStatus + name string + appSet v1alpha1.ApplicationSet + apps []v1alpha1.Application + expectedResources []v1alpha1.ResourceStatus + maxResourcesStatusCount int }{ { name: "handles an empty application list", @@ -6284,6 +6285,73 @@ func TestUpdateResourceStatus(t *testing.T) { apps: []v1alpha1.Application{}, expectedResources: nil, }, + { + name: "truncates resources status list to", + appSet: v1alpha1.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "argocd", + }, + Status: v1alpha1.ApplicationSetStatus{ + Resources: []v1alpha1.ResourceStatus{ + { + Name: "app1", + Status: v1alpha1.SyncStatusCodeOutOfSync, + Health: &v1alpha1.HealthStatus{ + Status: health.HealthStatusProgressing, + Message: "this is progressing", + }, + }, + { + Name: "app2", + Status: v1alpha1.SyncStatusCodeOutOfSync, + Health: &v1alpha1.HealthStatus{ + Status: health.HealthStatusProgressing, + Message: "this is progressing", + }, + }, + }, + }, + }, + apps: []v1alpha1.Application{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "app1", + }, + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Status: v1alpha1.SyncStatusCodeSynced, + }, + Health: v1alpha1.AppHealthStatus{ + Status: health.HealthStatusHealthy, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "app2", + }, + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Status: v1alpha1.SyncStatusCodeSynced, + }, + Health: v1alpha1.AppHealthStatus{ + Status: health.HealthStatusHealthy, + }, + }, + }, + }, + expectedResources: []v1alpha1.ResourceStatus{ + { + Name: "app1", + Status: v1alpha1.SyncStatusCodeSynced, + Health: &v1alpha1.HealthStatus{ + Status: health.HealthStatusHealthy, + }, + }, + }, + maxResourcesStatusCount: 1, + }, } { t.Run(cc.name, func(t *testing.T) { kubeclientset := kubefake.NewSimpleClientset([]runtime.Object{}...) @@ -6294,13 +6362,14 @@ func TestUpdateResourceStatus(t *testing.T) { argodb := db.NewDB("argocd", settings.NewSettingsManager(t.Context(), kubeclientset, "argocd"), kubeclientset) r := ApplicationSetReconciler{ - Client: client, - Scheme: scheme, - Recorder: record.NewFakeRecorder(1), - Generators: map[string]generators.Generator{}, - ArgoDB: argodb, - KubeClientset: kubeclientset, - Metrics: metrics, + Client: client, + Scheme: scheme, + Recorder: record.NewFakeRecorder(1), + Generators: map[string]generators.Generator{}, + ArgoDB: argodb, + KubeClientset: kubeclientset, + Metrics: metrics, + MaxResourcesStatusCount: cc.maxResourcesStatusCount, } err := r.updateResourcesStatus(t.Context(), log.NewEntry(log.StandardLogger()), &cc.appSet, cc.apps) diff --git a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go index dbe6515bf2eda..c221597196b9a 100644 --- a/cmd/argocd-applicationset-controller/commands/applicationset_controller.go +++ b/cmd/argocd-applicationset-controller/commands/applicationset_controller.go @@ -79,6 +79,7 @@ func NewCommand() *cobra.Command { enableScmProviders bool webhookParallelism int tokenRefStrictMode bool + maxResourcesStatusCount int ) scheme := runtime.NewScheme() _ = clientgoscheme.AddToScheme(scheme) @@ -231,6 +232,7 @@ func NewCommand() *cobra.Command { GlobalPreservedAnnotations: globalPreservedAnnotations, GlobalPreservedLabels: globalPreservedLabels, Metrics: &metrics, + MaxResourcesStatusCount: maxResourcesStatusCount, }).SetupWithManager(mgr, enableProgressiveSyncs, maxConcurrentReconciliations); err != nil { log.Error(err, "unable to create controller", "controller", "ApplicationSet") os.Exit(1) @@ -275,6 +277,7 @@ func NewCommand() *cobra.Command { command.Flags().IntVar(&webhookParallelism, "webhook-parallelism-limit", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_WEBHOOK_PARALLELISM_LIMIT", 50, 1, 1000), "Number of webhook requests processed concurrently") command.Flags().StringSliceVar(&metricsAplicationsetLabels, "metrics-applicationset-labels", []string{}, "List of Application labels that will be added to the argocd_applicationset_labels metric") command.Flags().BoolVar(&enableGitHubAPIMetrics, "enable-github-api-metrics", env.ParseBoolFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_ENABLE_GITHUB_API_METRICS", false), "Enable GitHub API metrics for generators that use the GitHub API") + command.Flags().IntVar(&maxResourcesStatusCount, "max-resources-status-count", env.ParseNumFromEnv("ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT", 0, 0, math.MaxInt), "Max number of resources stored in appset status.") return &command } diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml index b6a7bceb95e7c..abbd499606503 100644 --- a/docs/operator-manual/argocd-cmd-params-cm.yaml +++ b/docs/operator-manual/argocd-cmd-params-cm.yaml @@ -290,6 +290,8 @@ data: applicationsetcontroller.global.preserved.labels: "acme.com/label1,acme.com/label2" # Enable GitHub API metrics for generators that use GitHub API applicationsetcontroller.enable.github.api.metrics: "true" + # The maximum number of resources stored in the status of an ApplicationSet. This is a safeguard to prevent the status from growing too large. + applicationsetcontroller.status.max.resources.count: "5000" ## Argo CD Notifications Controller Properties # Set the logging level. One of: debug|info|warn|error (default "info") diff --git a/docs/operator-manual/server-commands/argocd-applicationset-controller.md b/docs/operator-manual/server-commands/argocd-applicationset-controller.md index b29efc7909815..40c97c4f0ba71 100644 --- a/docs/operator-manual/server-commands/argocd-applicationset-controller.md +++ b/docs/operator-manual/server-commands/argocd-applicationset-controller.md @@ -37,6 +37,7 @@ argocd-applicationset-controller [flags] --kubeconfig string Path to a kube config. Only required if out-of-cluster --logformat string Set the logging format. One of: json|text (default "json") --loglevel string Set the logging level. One of: debug|info|warn|error (default "info") + --max-resources-status-count int Max number of resources stored in appset status. --metrics-addr string The address the metric endpoint binds to. (default ":8080") --metrics-applicationset-labels strings List of Application labels that will be added to the argocd_applicationset_labels metric -n, --namespace string If present, the namespace scope for this CLI request diff --git a/manifests/base/applicationset-controller/argocd-applicationset-controller-deployment.yaml b/manifests/base/applicationset-controller/argocd-applicationset-controller-deployment.yaml index 860bcf7481cc0..cbad6a4014610 100644 --- a/manifests/base/applicationset-controller/argocd-applicationset-controller-deployment.yaml +++ b/manifests/base/applicationset-controller/argocd-applicationset-controller-deployment.yaml @@ -187,6 +187,12 @@ spec: name: argocd-cmd-params-cm key: applicationsetcontroller.requeue.after optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: applicationsetcontroller.status.max.resources.count + optional: true volumeMounts: - mountPath: /app/config/ssh name: ssh-known-hosts diff --git a/manifests/core-install-with-hydrator.yaml b/manifests/core-install-with-hydrator.yaml index 9719f494c63d3..69497219146e7 100644 --- a/manifests/core-install-with-hydrator.yaml +++ b/manifests/core-install-with-hydrator.yaml @@ -24699,6 +24699,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml index 6d0a5a0ef1b08..82a9727617ad9 100644 --- a/manifests/core-install.yaml +++ b/manifests/core-install.yaml @@ -24667,6 +24667,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/ha/install-with-hydrator.yaml b/manifests/ha/install-with-hydrator.yaml index 4f80db098ad1c..386bd9028a953 100644 --- a/manifests/ha/install-with-hydrator.yaml +++ b/manifests/ha/install-with-hydrator.yaml @@ -26065,6 +26065,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml index 130ed9ecf7b4b..d0c1422f40442 100644 --- a/manifests/ha/install.yaml +++ b/manifests/ha/install.yaml @@ -26035,6 +26035,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/ha/namespace-install-with-hydrator.yaml b/manifests/ha/namespace-install-with-hydrator.yaml index 6182c8f6db4d6..1ffb88dce4cfd 100644 --- a/manifests/ha/namespace-install-with-hydrator.yaml +++ b/manifests/ha/namespace-install-with-hydrator.yaml @@ -1868,6 +1868,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml index 9bf3d28aa90cb..176329918884d 100644 --- a/manifests/ha/namespace-install.yaml +++ b/manifests/ha/namespace-install.yaml @@ -1838,6 +1838,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/install-with-hydrator.yaml b/manifests/install-with-hydrator.yaml index 415551e9d4b71..290dbbfdd39dd 100644 --- a/manifests/install-with-hydrator.yaml +++ b/manifests/install-with-hydrator.yaml @@ -25159,6 +25159,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/install.yaml b/manifests/install.yaml index 62a5738b4d08f..cfe663200acbf 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -25127,6 +25127,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/namespace-install-with-hydrator.yaml b/manifests/namespace-install-with-hydrator.yaml index 80866b9eb2b12..e2ddf95b7eae7 100644 --- a/manifests/namespace-install-with-hydrator.yaml +++ b/manifests/namespace-install-with-hydrator.yaml @@ -962,6 +962,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml index 568e2e989834e..f33dcc704c2ae 100644 --- a/manifests/namespace-install.yaml +++ b/manifests/namespace-install.yaml @@ -930,6 +930,12 @@ spec: key: applicationsetcontroller.requeue.after name: argocd-cmd-params-cm optional: true + - name: ARGOCD_APPLICATIONSET_CONTROLLER_MAX_RESOURCES_STATUS_COUNT + valueFrom: + configMapKeyRef: + key: applicationsetcontroller.status.max.resources.count + name: argocd-cmd-params-cm + optional: true image: quay.io/argoproj/argocd:v3.1.6 imagePullPolicy: Always name: argocd-applicationset-controller