Skip to content

Commit e388c40

Browse files
AvineshTripathiAvineshTripathi
committed
feat(config): refactor metrics to use Kustomize components and dynamic build directory
Signed-off-by: AvineshTripathi <[email protected]>
1 parent 50a3a97 commit e388c40

12 files changed

Lines changed: 170 additions & 118 deletions

‎Makefile‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen
6262
IMG_PREFIX ?= controller
6363
IMG_TAG ?= latest
6464

65+
ENABLE_METRICS ?= false
66+
ENABLE_TLS ?= false
6567

6668
# Default value for ignore-not-found flag in undeploy target
6769
ignore-not-found ?= true
@@ -208,10 +210,9 @@ docker-buildx-reporter: ## Build and push docker image for the reporter for cros
208210
- $(CONTAINER_TOOL) buildx rm reporter-builder
209211

210212
.PHONY: build-installer
211-
build-installer: manifests generate $(KUSTOMIZE) ## Generate a consolidated YAML with CRDs and deployment.
213+
build-installer: ## Generate a consolidated YAML with CRDs and deployment.
212214
mkdir -p dist
213-
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG}
214-
$(KUSTOMIZE) build config/default > dist/install.yaml
215+
$(MAKE) -s build-manifests-temp > dist/install.yaml
215216

216217
## --------------------------------------
217218
## Deployment
@@ -223,6 +224,32 @@ ifndef ignore-not-found
223224
ignore-not-found = false
224225
endif
225226

227+
# Temporary directory for building manifests
228+
BUILD_DIR := $(ROOT_DIR)/bin/build
229+
230+
# Internal target to build manifests in a temporary directory to keep the source config clean.
231+
# This prevents 'kustomize edit' from modifying your local git state.
232+
# Features (Metrics, TLS) are enabled by adding Kustomize Components to the temporary copy.
233+
# TODO: we can do better for prometheus metrics ports that are added by manager_prometheus_metrics.yaml
234+
.PHONY: build-manifests-temp
235+
build-manifests-temp: manifests $(KUSTOMIZE)
236+
@mkdir -p $(BUILD_DIR)
237+
@rm -rf $(BUILD_DIR)/config
238+
@cp -r config $(BUILD_DIR)/
239+
@cd $(BUILD_DIR)/config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG}
240+
@if [ "$(ENABLE_METRICS)" = "true" ]; then \
241+
cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../prometheus; \
242+
if [ "$(ENABLE_TLS)" = "true" ]; then \
243+
cd $(BUILD_DIR)/config/default && $(KUSTOMIZE) edit add component ../certmanager && \
244+
$(KUSTOMIZE) edit add component ../prometheus/tls; \
245+
else \
246+
cd $(BUILD_DIR)/config/prometheus && $(KUSTOMIZE) edit add patch --path manager_prometheus_metrics.yaml --kind Deployment --name controller-manager; \
247+
fi; \
248+
fi
249+
@$(KUSTOMIZE) build $(BUILD_DIR)/config/default
250+
@rm -rf $(BUILD_DIR)/config
251+
252+
226253
.PHONY: install
227254
install: manifests $(KUSTOMIZE) ## Install CRDs into the K8s cluster specified in ~/.kube/config.
228255
@out="$$( $(KUSTOMIZE) build config/crd 2>/dev/null || true )"; \
@@ -234,13 +261,17 @@ uninstall: manifests $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specifi
234261
if [ -n "$$out" ]; then echo "$$out" | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
235262

236263
.PHONY: deploy
237-
deploy: manifests $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config.
238-
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_PREFIX}:${IMG_TAG}
239-
$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
264+
deploy: ## Deploy controller to the K8s cluster. Use ENABLE_METRICS=true and ENABLE_TLS=true to enable features.
265+
$(MAKE) -s build-manifests-temp | $(KUBECTL) apply -f -
240266

241267
.PHONY: undeploy
242-
undeploy: $(KUSTOMIZE) ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
243-
$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
268+
undeploy: ## Undeploy controller from the K8s cluster. Use ENABLE_METRICS=true and ENABLE_TLS=true if they were enabled during deploy.
269+
$(MAKE) -s build-manifests-temp | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
270+
271+
.PHONY: debug-deploy
272+
debug-deploy: ## Build and save manifests to debug_manifests.yaml for inspection. Use ENABLE_METRICS=true and ENABLE_TLS=true to enable features.
273+
$(MAKE) -s build-manifests-temp > debug_manifests.yaml
274+
@echo "Manifests generated in debug_manifests.yaml"
244275

245276
## --------------------------------------
246277
## Testing
@@ -345,7 +376,6 @@ docs-serve: ## Serve mdBook locally.
345376
.PHONY: crd-ref-docs
346377
crd-ref-docs:
347378
crd-ref-docs \
348-
--source-path=${PWD}/api/v1alpha1/ \
349379
--config=crd-ref-docs.yaml \
350380
--renderer=markdown \
351381
--output-path=${PWD}/docs/book/src/reference/api-spec.md

‎config/certmanager/kustomization.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
apiVersion: kustomize.config.k8s.io/v1alpha1
2+
kind: Component
13
resources:
24
- certificate.yaml
35

‎config/default/kustomization.yaml‎

Lines changed: 0 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -17,97 +17,3 @@ namePrefix: nrr-
1717
resources:
1818
- ../rbac
1919
- ../manager
20-
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
21-
- ../prometheus
22-
# [METRICS] Expose the controller manager metrics service.
23-
- metrics_service.yaml
24-
# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy.
25-
# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics.
26-
# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will
27-
# be able to communicate with the Webhook Server.
28-
#- ../network-policy
29-
- ../certmanager
30-
31-
# Uncomment the patches line if you enable Metrics
32-
patches:
33-
# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
34-
# More info: https://book.kubebuilder.io/reference/metrics
35-
- path: manager_metrics_patch.yaml
36-
target:
37-
kind: Deployment
38-
39-
# Uncomment the patches line if you enable Metrics and CertManager
40-
# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
41-
# This patch will protect the metrics with certManager self-signed certs.
42-
- path: cert_metrics_manager_patch.yaml
43-
target:
44-
kind: Deployment
45-
46-
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
47-
# crd/kustomization.yaml
48-
# - path: manager_webhook_patch.yaml
49-
# target:
50-
# kind: Deployment
51-
52-
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
53-
# Uncomment the following replacements to add the cert-manager CA injection annotations
54-
replacements:
55-
- source: # Uncomment the following block to enable certificates for metrics
56-
kind: Service
57-
version: v1
58-
name: controller-manager-metrics-service
59-
fieldPath: metadata.name
60-
targets:
61-
- select:
62-
kind: Certificate
63-
group: cert-manager.io
64-
version: v1
65-
name: metrics-certs
66-
fieldPaths:
67-
- spec.dnsNames.0
68-
- spec.dnsNames.1
69-
options:
70-
delimiter: '.'
71-
index: 0
72-
create: true
73-
- select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor
74-
kind: ServiceMonitor
75-
group: monitoring.coreos.com
76-
version: v1
77-
name: controller-manager-metrics-monitor
78-
fieldPaths:
79-
- spec.endpoints.0.tlsConfig.serverName
80-
options:
81-
delimiter: '.'
82-
index: 0
83-
create: true
84-
85-
- source:
86-
kind: Service
87-
version: v1
88-
name: controller-manager-metrics-service
89-
fieldPath: metadata.namespace
90-
targets:
91-
- select:
92-
kind: Certificate
93-
group: cert-manager.io
94-
version: v1
95-
name: metrics-certs
96-
fieldPaths:
97-
- spec.dnsNames.0
98-
- spec.dnsNames.1
99-
options:
100-
delimiter: '.'
101-
index: 1
102-
create: true
103-
- select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor
104-
kind: ServiceMonitor
105-
group: monitoring.coreos.com
106-
version: v1
107-
name: controller-manager-metrics-monitor
108-
fieldPaths:
109-
- spec.endpoints.0.tlsConfig.serverName
110-
options:
111-
delimiter: '.'
112-
index: 1
113-
create: true

‎config/prometheus/kustomization.yaml‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,22 @@
1+
apiVersion: kustomize.config.k8s.io/v1alpha1
2+
kind: Component
13
resources:
24
- monitor.yaml
5+
- metrics_service.yaml
36

4-
# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
5-
# to securely reference certificates created and managed by cert-manager.
6-
# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
7-
# to mount the "metrics-server-cert" secret in the Manager Deployment.
87
patches:
9-
- path: monitor_tls_patch.yaml
10-
target:
11-
kind: ServiceMonitor
8+
# Bind metrics to port 8080 for HTTP.
9+
# This matches the Service and ServiceMonitor configuration in this directory.
10+
# - path: manager_prometheus_metrics.yaml
11+
# target:
12+
# kind: Deployment
13+
# name: controller-manager
14+
15+
# By default, metrics are disabled in the manager (default : "0").
16+
# This component adds the Service and ServiceMonitor for Prometheus,
17+
# and applies the patch to bind the manager to port :8080(it is done in Makefile for now).
18+
19+
# Patches for TLS are in the 'tls' component which will:
20+
# 1. Overlay the HTTPS args (:8443) and security flags
21+
# 2. Add ServiceMonitor TLS config
22+
# 3. Mount CertManager secrets

‎config/prometheus/manager_prometheus_metrics.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# This patch adds the args to allow exposing the metrics endpoint using HTTP
2+
- op: add
3+
path: /spec/template/spec/containers/0/args/-
4+
value: --metrics-bind-address=:8080

config/default/metrics_service.yaml renamed to config/prometheus/metrics_service.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ metadata:
99
namespace: system
1010
spec:
1111
ports:
12-
- name: https
13-
port: 8443
12+
- name: http
13+
port: 8080
1414
protocol: TCP
15-
targetPort: 8443
15+
targetPort: 8080
1616
selector:
1717
control-plane: controller-manager
1818
app.kubernetes.io/name: nrrcontroller

‎config/prometheus/monitor.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ metadata:
1111
spec:
1212
endpoints:
1313
- path: /metrics
14-
port: https # Ensure this is the name of the port that exposes HTTPS metrics
15-
scheme: https
14+
port: http # Ensure this is the name of the port that exposes HTTP metrics
15+
scheme: http
1616
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
1717
tlsConfig:
1818
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables

config/default/cert_metrics_manager_patch.yaml renamed to config/prometheus/tls/cert_metrics_manager_patch.yaml

File renamed without changes.

‎config/prometheus/tls/kustomization.yaml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
apiVersion: kustomize.config.k8s.io/v1alpha1
2+
kind: Component
3+
4+
patches:
5+
# Enable HTTPS args in Deployment
6+
- path: manager_prometheus_metrics_tls.yaml
7+
target:
8+
kind: Deployment
9+
# Configure ServiceMonitor for TLS
10+
- path: monitor_tls_patch.yaml
11+
target:
12+
kind: ServiceMonitor
13+
# Mount CertManager secrets in Deployment
14+
- path: cert_metrics_manager_patch.yaml
15+
target:
16+
kind: Deployment
17+
# Switch Service to 8443 for TLS
18+
- path: metrics_service_tls_patch.yaml
19+
target:
20+
kind: Service
21+
22+
replacements:
23+
- source:
24+
kind: Service
25+
version: v1
26+
name: controller-manager-metrics-service
27+
fieldPath: metadata.name
28+
targets:
29+
- select:
30+
kind: Certificate
31+
group: cert-manager.io
32+
version: v1
33+
name: metrics-certs
34+
fieldPaths:
35+
- spec.dnsNames.0
36+
- spec.dnsNames.1
37+
options:
38+
delimiter: '.'
39+
index: 0
40+
create: true
41+
- select:
42+
kind: ServiceMonitor
43+
group: monitoring.coreos.com
44+
version: v1
45+
name: controller-manager-metrics-monitor
46+
fieldPaths:
47+
- spec.endpoints.0.tlsConfig.serverName
48+
options:
49+
delimiter: '.'
50+
index: 0
51+
create: true
52+
- source:
53+
kind: Service
54+
version: v1
55+
name: controller-manager-metrics-service
56+
fieldPath: metadata.namespace
57+
targets:
58+
- select:
59+
kind: Certificate
60+
group: cert-manager.io
61+
version: v1
62+
name: metrics-certs
63+
fieldPaths:
64+
- spec.dnsNames.0
65+
- spec.dnsNames.1
66+
options:
67+
delimiter: '.'
68+
index: 1
69+
create: true
70+
- select:
71+
kind: ServiceMonitor
72+
group: monitoring.coreos.com
73+
version: v1
74+
name: controller-manager-metrics-monitor
75+
fieldPaths:
76+
- spec.endpoints.0.tlsConfig.serverName
77+
options:
78+
delimiter: '.'
79+
index: 1
80+
create: true

config/default/manager_metrics_patch.yaml renamed to config/prometheus/tls/manager_prometheus_metrics_tls.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
# This patch adds the args to allow exposing the metrics endpoint using HTTPS
22
- op: add
3-
path: /spec/template/spec/containers/0/args/0
3+
path: /spec/template/spec/containers/0/args/-
44
value: --metrics-bind-address=:8443
55
- op: add
6-
path: /spec/template/spec/containers/0/args/1
6+
path: /spec/template/spec/containers/0/args/-
77
value: --metrics-secure
88
- op: add
9-
path: /spec/template/spec/containers/0/args/2
9+
path: /spec/template/spec/containers/0/args/-
1010
value: --metrics-cert-dir=/tmp/k8s-metrics-server/metrics-certs

0 commit comments

Comments
 (0)