feat(http-add-on): replace kube-rbac-proxy with native secure metrics (#807)

The kube-rbac-proxy sidecar is being discontinued and its images will
become unavailable. This change uses controller-runtime's built-in
metrics authentication instead, following kubebuilder recommendations.

Changes:
- Remove kube-rbac-proxy sidecar container from operator deployment
- Add operator.metrics config (secure, auth, certDir) to values.yaml
- Replace proxy-role ClusterRole with system:auth-delegator binding
- Remove kube-rbac-proxy image and securityContext references

Signed-off-by: Vincent Link <vlink@redhat.com>

Author: linkvt

http-add-on/README.md

@@ -87,8 +87,6 @@ their default values.
 | `additionalLabels` | object | `{}` | Additional labels to be applied to installed resources. Note that not all resources will receive these labels. |
 | `crds.install` | bool | `true` | Whether to install the `HTTPScaledObject` [`CustomResourceDefinition`](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) |
 | `images.interceptor` | string | `"ghcr.io/kedacore/http-add-on-interceptor"` | Image name for the interceptor image component |
-| `images.kubeRbacProxy.name` | string | `"gcr.io/kubebuilder/kube-rbac-proxy"` | Image name for the Kube RBAC Proxy image component |
-| `images.kubeRbacProxy.tag` | string | `"v0.13.0"` | Image tag for the Kube RBAC Proxy image component |
 | `images.operator` | string | `"ghcr.io/kedacore/http-add-on-operator"` | Image name for the operator image component |
 | `images.scaler` | string | `"ghcr.io/kedacore/http-add-on-scaler"` | Image name for the scaler image component |
 | `images.tag` | string | `""` | Image tag for the http add on. This tag is applied to the images listed in `images.operator`, `images.interceptor`, and `images.scaler`. Optional, given app version of Helm chart is used by default |
@@ -97,7 +95,6 @@ their default values.
 | `logging.interceptor.stackTracesEnabled` | bool | `false` | Display stack traces in the logs |
 | `logging.interceptor.timeEncoding` | string | `"rfc3339"` | Logging time encoding for KEDA http-add-on Interceptor. allowed values are `epoch`, `millis`, `nano`, `iso8601`, `rfc3339` or `rfc3339nano` |
 | `logging.operator.format` | string | `"console"` | Logging format for KEDA http-add-on operator. allowed values: `json` or `console` |
-| `logging.operator.kubeRbacProxy.level` | int | `10` | Logging level for KEDA http-add-on operator rbac proxy allowed values: `0` for info, `4` for debug, or an integer value greater than 0 |
 | `logging.operator.level` | string | `"info"` | Logging level for KEDA http-add-on operator. allowed values: `debug`, `info`, `error`, or an integer value greater than 0, specified as string |
 | `logging.operator.stackTracesEnabled` | bool | `false` | Display stack traces in the logs |
 | `logging.operator.timeEncoding` | string | `"rfc3339"` | Logging time encoding for KEDA http-add-on operator. allowed values are `epoch`, `millis`, `nano`, `iso8601`, `rfc3339` or `rfc3339nano` |
@@ -122,13 +119,14 @@ their default values.
 | `operator.affinity` | object | `{}` | Affinity for pod scheduling ([docs](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/)) |
 | `operator.extraEnvs` | object | `{}` | Extra environment variables to set (key-value map with "ENV name":"value") |
 | `operator.imagePullSecrets` | list | `[]` | The image pull secrets for the operator component |
-| `operator.kubeRbacProxy.resources.limits` | object | `{"cpu":"300m","memory":"200Mi"}` | The CPU/memory resource limit for the operator component's kube rbac proxy |
-| `operator.kubeRbacProxy.resources.requests` | object | `{"cpu":"10m","memory":"20Mi"}` | The CPU/memory resource request for the operator component's kube rbac proxy |
+| `operator.metrics.auth` | bool | `true` | Enable authentication and authorization for the metrics endpoint |
+| `operator.metrics.certDir` | string | `""` | Directory containing TLS certificates (tls.crt/tls.key). If empty, self-signed certs are generated. |
+| `operator.metrics.secure` | bool | `true` | Enable HTTPS for the metrics endpoint |
 | `operator.nodeSelector` | object | `{}` | Node selector for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/)) |
 | `operator.podAnnotations` | object | `{}` | Annotations to be added to the operator pods |
 | `operator.port` | int | `8443` | The port for the operator main server to run on |
 | `operator.pullPolicy` | string | `"Always"` | The image pull policy for the operator component |
-| `operator.replicas` | int | `1` | Number of replicas, oerator k8s resources will not be installed if this is set to 0 |
+| `operator.replicas` | int | `1` | Number of replicas, operator k8s resources will not be installed if this is set to 0 |
 | `operator.resources.limits` | object | `{"cpu":0.5,"memory":"64Mi"}` | The CPU/memory resource limit for the operator component |
 | `operator.resources.requests` | object | `{"cpu":"250m","memory":"20Mi"}` | The CPU/memory resource request for the operator component |
 | `operator.tolerations` | list | `[]` | Tolerations for pod scheduling ([docs](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)) |
@@ -184,8 +182,8 @@ their default values.
 | `interceptor.replicas.max` | int | `50` | The maximum number of interceptor replicas that should ever be running |
 | `interceptor.replicas.min` | int | `3` | The minimum number of interceptor replicas that should ever be running |
 | `interceptor.replicas.waitTimeout` | string | `"20s"` | The maximum time the interceptor should wait for an HTTP request to reach a backend before it is considered a failure |
-| `interceptor.resources.limits` | object | `{"cpu":0.5,"memory":"64Mi"}` | The CPU/memory resource limit for the operator component |
-| `interceptor.resources.requests` | object | `{"cpu":"250m","memory":"20Mi"}` | The CPU/memory resource request for the operator component |
+| `interceptor.resources.limits` | object | `{"cpu":0.5,"memory":"64Mi"}` | The CPU/memory resource limit for the interceptor component |
+| `interceptor.resources.requests` | object | `{"cpu":"250m","memory":"20Mi"}` | The CPU/memory resource request for the interceptor component |
 | `interceptor.responseHeaderTimeout` | string | `"500ms"` | How long the interceptor will wait between forwarding a request to a backend and receiving response headers back before failing the request |
 | `interceptor.scaledObject.pollingInterval` | int | `1` | The interval (in milliseconds) that KEDA should poll the external scaler to fetch scaling metrics about the interceptor |
 | `interceptor.tcpConnectTimeout` | string | `"500ms"` | How long the interceptor waits to establish TCP connections with backends before failing a request. |

http-add-on/templates/operator/deployment.yaml

@@ -40,23 +40,16 @@ spec:
       {{- end }}
       containers:
       - args:
-        - --secure-listen-address=0.0.0.0:{{ .Values.operator.port | default 8443 }}
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        - --v={{ .Values.logging.operator.kubeRbacProxy.level }}
-        image: "{{ .Values.images.kubeRbacProxy.name }}:{{ .Values.images.kubeRbacProxy.tag }}"
-        name: kube-rbac-proxy
-        resources:
-          {{- toYaml .Values.operator.kubeRbacProxy.resources | nindent 10 }}
-        {{- if .Values.securityContext.kuberbacproxy }}
-        securityContext:
-        {{- toYaml .Values.securityContext.kuberbacproxy | nindent 10 }}
-        {{- else }}
-        securityContext:
-        {{- toYaml .Values.securityContext | nindent 10 }}
+        - --metrics-bind-address=:{{ .Values.operator.port }}
+        {{- if .Values.operator.metrics.secure }}
+        - --metrics-secure=true
+        {{- end }}
+        {{- if .Values.operator.metrics.auth }}
+        - --metrics-auth=true
+        {{- end }}
+        {{- if .Values.operator.metrics.certDir }}
+        - --metrics-cert-dir={{ .Values.operator.metrics.certDir }}
         {{- end }}
-      - args:
-        - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
         - --zap-log-level={{ .Values.logging.operator.level }}
         - --zap-time-encoding={{ .Values.logging.operator.timeEncoding }}
@@ -85,7 +78,7 @@ spec:
         {{- end }}
         ports:
         - name: metrics
-          containerPort: 8080
+          containerPort: {{ .Values.operator.port }}
         - name: probes
           containerPort: 8081
         livenessProbe:

http-add-on/templates/operator/rbac.yml

@@ -83,28 +83,6 @@ rules:
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
-metadata:
-  labels:
-    httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
-    keda.sh/addon: {{ .Chart.Name }}
-    app: {{ .Chart.Name }}
-    name: {{ .Chart.Name }}-proxy-role
-    app.kubernetes.io/component: operator
-    app.kubernetes.io/name: http-add-on
-    {{- include "keda-http-add-on.labels" . | indent 4 }}
-  name: {{ .Chart.Name }}-proxy-role
-rules:
-- apiGroups: ["authentication.k8s.io"]
-  resources:
-  - tokenreviews
-  verbs: ["create"]
-- apiGroups: ["authorization.k8s.io"]
-  resources:
-  - subjectaccessreviews
-  verbs: ["create"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
 metadata:
   labels:
     httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
@@ -163,6 +141,7 @@ subjects:
 - kind: ServiceAccount
   name: {{ .Chart.Name }}
   namespace: {{ .Release.Namespace }}
+{{- if .Values.operator.metrics.auth }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -171,17 +150,18 @@ metadata:
     httpscaledobjects.http.keda.sh/version: {{ .Values.images.tag | default .Chart.AppVersion }}
     keda.sh/addon: {{ .Chart.Name }}
     app: {{ .Chart.Name }}
-    name: {{ .Chart.Name }}-rolebinding
+    name: {{ .Chart.Name }}-auth-delegator
     app.kubernetes.io/component: operator
     app.kubernetes.io/name: http-add-on
     {{- include "keda-http-add-on.labels" . | indent 4 }}
-  name: {{ .Chart.Name }}-proxy-rolebinding
+  name: {{ .Chart.Name }}-auth-delegator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: {{ .Chart.Name }}-proxy-role
+  name: system:auth-delegator
 subjects:
 - kind: ServiceAccount
   name: {{ .Chart.Name }}
   namespace: {{ .Release.Namespace }}
 {{- end }}
+{{- end }}

http-add-on/values.yaml

@@ -18,11 +18,6 @@ logging:
     timeEncoding: rfc3339
     # -- Display stack traces in the logs
     stackTracesEnabled: false
-
-    kubeRbacProxy:
-      # -- Logging level for KEDA http-add-on operator rbac proxy
-      # allowed values: `0` for info, `4` for debug, or an integer value greater than 0
-      level: 10
   scaler:
     # -- Logging level for KEDA http-add-on Scaler.
     # allowed values: `debug`, `info`, `error`, or an integer value greater than 0, specified as string
@@ -53,7 +48,7 @@ logging:
 operator:
   # -- Extra environment variables to set (key-value map with "ENV name":"value")
   extraEnvs: {}
-  # -- Number of replicas, oerator k8s resources will not be installed if this is set to 0
+  # -- Number of replicas, operator k8s resources will not be installed if this is set to 0
   replicas: 1
   # -- The image pull secrets for the operator component
   imagePullSecrets: []
@@ -84,16 +79,15 @@ operator:
   # -- Annotations to be added to the operator pods
   podAnnotations: {}
 
-  kubeRbacProxy:
-    resources:
-      # -- The CPU/memory resource limit for the operator component's kube rbac proxy
-      limits:
-        cpu: 300m
-        memory: 200Mi
-      # -- The CPU/memory resource request for the operator component's kube rbac proxy
-      requests:
-        cpu: 10m
-        memory: 20Mi
+  # Operator metrics endpoint configuration
+  # NOTE: Consider defaulting to secure: false, auth: false for v1 release
+  metrics:
+    # -- Enable HTTPS for the metrics endpoint
+    secure: true
+    # -- Enable authentication and authorization for the metrics endpoint
+    auth: true
+    # -- Directory containing TLS certificates (tls.crt/tls.key). If empty, self-signed certs are generated.
+    certDir: ""
 
 scaler:
   # -- Extra environment variables to set (key-value map with "ENV name":"value")
@@ -197,11 +191,11 @@ interceptor:
   topologySpreadConstraints: []
   # interceptor pod resource limits
   resources:
-    # -- The CPU/memory resource limit for the operator component
+    # -- The CPU/memory resource limit for the interceptor component
     limits:
       cpu: 0.5
       memory: 64Mi
-    # -- The CPU/memory resource request for the operator component
+    # -- The CPU/memory resource request for the interceptor component
     requests:
       cpu: 250m
       memory: 20Mi
@@ -246,12 +240,6 @@ images:
   interceptor: ghcr.io/kedacore/http-add-on-interceptor
   # -- Image name for the scaler image component
   scaler: ghcr.io/kedacore/http-add-on-scaler
-  # the kube-rbac-proxy image to use
-  kubeRbacProxy:
-    # -- Image name for the Kube RBAC Proxy image component
-    name: gcr.io/kubebuilder/kube-rbac-proxy
-    # -- Image tag for the Kube RBAC Proxy image component
-    tag: v0.13.0
 
 rbac:
   # -- Install aggregate roles for edit and view
@@ -276,14 +264,6 @@ securityContext:
     # readOnlyRootFilesystem: true
     # seccompProfile:
     #   type: RuntimeDefault
-  # kuberbacproxy:
-    # capabilities:
-    #   drop:
-    #   - ALL
-    # allowPrivilegeEscalation: false
-    # readOnlyRootFilesystem: true
-    # seccompProfile:
-    #   type: RuntimeDefault
   # scaler:
     # capabilities:
     #   drop: