refactor: enable metrics manifests

Makefile

@@ -62,6 +62,10 @@ CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen
 IMG_PREFIX ?= controller
 IMG_TAG ?= latest
 
+# ENABLE_METRICS: If set to true, includes Prometheus Service and ServiceMonitor resources.
+ENABLE_METRICS ?= false
+# ENABLE_TLS: If set to true (and ENABLE_METRICS is true), configures metrics to use HTTPS with CertManager.
+ENABLE_TLS ?= false
 
 # Default value for ignore-not-found flag in undeploy target
 ignore-not-found ?= true
@@ -208,14 +212,13 @@ docker-buildx-reporter: ## Build and push docker image for the reporter for cros
 	- $(CONTAINER_TOOL) buildx rm reporter-builder
 
 .PHONY: build-installer
-build-installer: manifests generate $(KUSTOMIZE) ## Generate CRDs and deployment manifests for release.
+build-installer: build-manifests-temp ## Generate CRDs and deployment manifests for release.
 	mkdir -p dist
 	# Generate CRDs only
 	$(KUSTOMIZE) build config/crd > dist/crds.yaml
 	@echo "Generated dist/crds.yaml"
 	# Generate controller deployment without CRDs
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG}
-	$(KUSTOMIZE) build config/default > dist/install.yaml
+	cp $(BUILD_DIR)/manifests.yaml dist/install.yaml
 	@echo "Generated dist/install.yaml with image ${IMG_PREFIX}:${IMG_TAG}"
 	@echo "NOTE: Install crds.yaml first, then install.yaml. Deployment runs on any available node by default."
 
@@ -229,6 +232,32 @@ ifndef ignore-not-found
   ignore-not-found = false
 endif
 
+# Temporary directory for building manifests
+BUILD_DIR := $(ROOT_DIR)/bin/build
+
+# Internal target to build manifests in a temporary directory to keep the source config clean.
+# This prevents 'kustomize edit' from modifying your local git state.
+# Features (Metrics, TLS) are enabled by adding Kustomize Components to the temporary copy.
+# TODO: we can do better for prometheus metrics ports that are added by manager_prometheus_metrics.yaml
+.PHONY: build-manifests-temp
+build-manifests-temp: manifests $(KUSTOMIZE)
+	@mkdir -p $(BUILD_DIR)
+	@rm -rf $(BUILD_DIR)/config
+	@cp -r config $(BUILD_DIR)/
+	@cd $(BUILD_DIR)/config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG}
+	@if [ "$(ENABLE_METRICS)" = "true" ]; then \
+		cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../prometheus; \
+		if [ "$(ENABLE_TLS)" = "true" ]; then \
+			cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../certmanager && \
+			$(KUSTOMIZE) edit add component ../prometheus/tls; \
+		else \
+			cd $(BUILD_DIR)/config/prometheus && $(KUSTOMIZE) edit add patch --path manager_prometheus_metrics.yaml --kind Deployment --name controller-manager; \
+		fi; \
+	fi
+	@$(KUSTOMIZE) build $(BUILD_DIR)/config/default > $(BUILD_DIR)/manifests.yaml
+	@rm -rf $(BUILD_DIR)/config
+
+
 .PHONY: install
 install: manifests $(KUSTOMIZE) ## Install CRDs into the K8s cluster specified in ~/.kube/config.
 	@out="$$( $(KUSTOMIZE) build config/crd 2>/dev/null || true )"; \
@@ -240,13 +269,12 @@ uninstall: manifests $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specifi
 	if [ -n "$$out" ]; then echo "$$out" | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
 
 .PHONY: deploy
-deploy: manifests $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG}
-	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
+deploy: build-manifests-temp ## Deploy controller to the K8s cluster. Use ENABLE_METRICS=true and ENABLE_TLS=true to enable features.
+	$(KUBECTL) apply -f $(BUILD_DIR)/manifests.yaml
 
 .PHONY: undeploy
-undeploy: $(KUSTOMIZE) ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+undeploy: build-manifests-temp ## Undeploy controller from the K8s cluster. Use ENABLE_METRICS=true and ENABLE_TLS=true if they were enabled during deploy.
+	$(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f $(BUILD_DIR)/manifests.yaml
 
 ## --------------------------------------
 ## Testing

cmd/main.go

@@ -19,12 +19,14 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	"go.uber.org/zap/zapcore"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -33,6 +35,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	nodereadinessiov1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1"
@@ -60,8 +63,15 @@ func main() {
 	var enableLeaderElection bool
 	var probeAddr string
 	var enableWebhook bool
+	var metricsSecure bool
+	var metricsCertDir string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
+	flag.BoolVar(&metricsSecure, "metrics-secure", false,
+		"If set, the metrics endpoint is served securely via HTTPS. "+
+			"Requires certificate and key.")
+	flag.StringVar(&metricsCertDir, "metrics-cert-dir", "",
+		"The directory where the certificates for metrics are located.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
@@ -80,7 +90,15 @@ func main() {
 	ctrl.Log.Info(fmt.Sprintf("version: %s", info.GetVersionString()))
 
 	metricsServerOptions := metricsserver.Options{
-		BindAddress: metricsAddr,
+		BindAddress:   metricsAddr,
+		CertDir:       metricsCertDir,
+		SecureServing: metricsSecure,
+		FilterProvider: func() func(c *rest.Config, httpClient *http.Client) (metricsserver.Filter, error) {
+			if metricsSecure {
+				return filters.WithAuthenticationAndAuthorization
+			}
+			return nil
+		}(),
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{

config/certmanager/certificate.yaml

@@ -0,0 +1,24 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: metrics-certs
+  namespace: system
+spec:
+  commonName: nrr-metrics
+  dnsNames:
+  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
+  - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: metrics-server-cert

config/certmanager/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+resources:
+- certificate.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/certmanager/kustomizeconfig.yaml

@@ -0,0 +1,19 @@
+nameReference:
+- kind: Issuer
+  group: cert-manager.io
+  fieldSpecs:
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/issuerRef/name
+
+- kind: Secret
+  version: v1
+  fieldSpecs:
+  - kind: Certificate
+    group: cert-manager.io
+    path: spec/secretName
+
+varReference:
+- kind: Certificate
+  group: cert-manager.io
+  path: spec/dnsNames

config/default/kustomization.yaml

@@ -17,21 +17,3 @@ namePrefix: nrr-
 resources:
 - ../rbac
 - ../manager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-#- ../prometheus
-# [METRICS] Expose the controller manager metrics service.
-# - metrics_service.yaml
-# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy.
-# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics.
-# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will
-# be able to communicate with the Webhook Server.
-#- ../network-policy
-
-# Uncomment the patches line if you enable Metrics
-# patches:
-# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
-# More info: https://book.kubebuilder.io/reference/metrics
-# - path: manager_metrics_patch.yaml
-#   target:
-#     kind: Deployment
-

config/prometheus/kustomization.yaml

@@ -1,11 +1,22 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
 resources:
 - monitor.yaml
+- metrics_service.yaml
 
-# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
-# to securely reference certificates created and managed by cert-manager.
-# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
-# to mount the "metrics-server-cert" secret in the Manager Deployment.
-#patches:
-#  - path: monitor_tls_patch.yaml
-#    target:
-#      kind: ServiceMonitor
+patches:
+# Bind metrics to port 8080 for HTTP. 
+# This matches the Service and ServiceMonitor configuration in this directory.
+# - path: manager_prometheus_metrics.yaml
+#   target:
+#     kind: Deployment
+#     name: controller-manager
+
+# By default, metrics are disabled in the manager (default : "0").
+# This component adds the Service and ServiceMonitor for Prometheus,
+# and applies the patch to bind the manager to port :8080(it is done in Makefile for now).
+
+# Patches for TLS are in the 'tls' component which will:
+# 1. Overlay the HTTPS args (:8443) and security flags
+# 2. Add ServiceMonitor TLS config
+# 3. Mount CertManager secrets

config/prometheus/manager_prometheus_metrics.yaml

@@ -0,0 +1,4 @@
+# This patch adds the args to allow exposing the metrics endpoint using HTTP
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-bind-address=:8080

config/default/metrics_service.yaml → config/prometheus/metrics_service.yaml

@@ -9,10 +9,10 @@ metadata:
   namespace: system
 spec:
   ports:
-  - name: https
-    port: 8443
+  - name: http
+    port: 8080
     protocol: TCP
-    targetPort: 8443
+    targetPort: 8080
   selector:
     control-plane: controller-manager
     app.kubernetes.io/name: nrrcontroller

config/prometheus/monitor.yaml

@@ -11,16 +11,16 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: https # Ensure this is the name of the port that exposes HTTPS metrics
-      scheme: https
+      port: http # Ensure this is the name of the port that exposes HTTP metrics
+      scheme: http
       bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
       tlsConfig:
         # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
         # certificate verification, exposing the system to potential man-in-the-middle attacks.
         # For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
         # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
         # which securely references the certificate from the 'metrics-server-cert' secret.
-        insecureSkipVerify: true
+        insecureSkipVerify: false
   selector:
     matchLabels:
       control-plane: controller-manager

config/prometheus/tls/cert_metrics_manager_patch.yaml

@@ -0,0 +1,18 @@
+# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs.
+
+# Add the volumeMount for the metrics-server certs
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    mountPath: /tmp/k8s-metrics-server/metrics-certs
+    name: metrics-certs
+    readOnly: true
+
+# Add the metrics-server certs volume configuration
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: metrics-certs
+    secret:
+      secretName: metrics-server-cert
+      optional: false

config/prometheus/tls/kustomization.yaml

@@ -0,0 +1,80 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+patches:
+# Enable HTTPS args in Deployment
+- path: manager_prometheus_metrics_tls.yaml
+  target:
+    kind: Deployment
+# Configure ServiceMonitor for TLS
+- path: monitor_tls_patch.yaml
+  target:
+    kind: ServiceMonitor
+# Mount CertManager secrets in Deployment
+- path: cert_metrics_manager_patch.yaml
+  target:
+    kind: Deployment
+# Switch Service to 8443 for TLS
+- path: metrics_service_tls_patch.yaml
+  target:
+    kind: Service
+
+replacements:
+  - source:
+      kind: Service
+      version: v1
+      name: controller-manager-metrics-service
+      fieldPath: metadata.name
+    targets:
+      - select:
+          kind: Certificate
+          group: cert-manager.io
+          version: v1
+          name: metrics-certs
+        fieldPaths:
+          - spec.dnsNames.0
+          - spec.dnsNames.1
+        options:
+          delimiter: '.'
+          index: 0
+          create: true
+      - select:
+          kind: ServiceMonitor
+          group: monitoring.coreos.com
+          version: v1
+          name: controller-manager-metrics-monitor
+        fieldPaths:
+          - spec.endpoints.0.tlsConfig.serverName
+        options:
+          delimiter: '.'
+          index: 0
+          create: true
+  - source:
+      kind: Service
+      version: v1
+      name: controller-manager-metrics-service
+      fieldPath: metadata.namespace
+    targets:
+      - select:
+          kind: Certificate
+          group: cert-manager.io
+          version: v1
+          name: metrics-certs
+        fieldPaths:
+          - spec.dnsNames.0
+          - spec.dnsNames.1
+        options:
+          delimiter: '.'
+          index: 1
+          create: true
+      - select:
+          kind: ServiceMonitor
+          group: monitoring.coreos.com
+          version: v1
+          name: controller-manager-metrics-monitor
+        fieldPaths:
+          - spec.endpoints.0.tlsConfig.serverName
+        options:
+          delimiter: '.'
+          index: 1
+          create: true

config/prometheus/tls/manager_prometheus_metrics_tls.yaml

@@ -0,0 +1,10 @@
+# This patch adds the args to allow exposing the metrics endpoint using HTTPS
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-bind-address=:8443
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-secure
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --metrics-cert-dir=/tmp/k8s-metrics-server/metrics-certs