feat(t8s-cluster/management-cluster): switch to hcp

charts/t8s-cluster/ci/autoscaling-values.yaml

@@ -0,0 +1,6 @@
+nodePools:
+  autoscaling:
+    flavor: standard.2.4096
+    replicas:
+      min: 1
+      max: 3

charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl

@@ -35,6 +35,7 @@ openstack
   {{- toYaml $args -}}
 {{- end -}}
 
+{{/* this can be split up with containerd >=2.0.0 */}}
 {{- define "t8s-cluster.clusterClass.containerdConfig.plugins" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
 [plugins]
@@ -129,12 +130,6 @@ server = {{ printf "https://%s" .registry | quote }}
 
 {{- define "t8s-cluster.clusterClass.args.shared" -}}
   {{- $args := include "t8s-cluster.clusterClass.args.base" (dict) | fromYaml -}}
-  {{- $args = mustMerge (dict
-      "authorization-always-allow-paths" (list "/healthz" "/readyz" "/livez" "/metrics" | join ",")
-      "bind-address" "0.0.0.0"
-    )
-    $args
-  -}}
   {{- toYaml $args -}}
 {{- end -}}
 
@@ -170,10 +165,6 @@ server = {{ printf "https://%s" .registry | quote }}
 {{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.authenticationConfigFileName" (dict) -}}
 {{- end -}}
 
-{{- define "t8s-cluster.clusterClass.apiServer.auditConfigPath" -}}
-{{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.auditConfigFileName" (dict) -}}
-{{- end -}}
-
 {{- define "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" -}}
 {{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict) -}}
 {{- end -}}
@@ -186,10 +177,6 @@ server = {{ printf "https://%s" .registry | quote }}
 authentication-config.yaml
 {{- end -}}
 
-{{- define "t8s-cluster.clusterClass.apiServer.auditConfigFileName" -}}
-audit-config.yaml
-{{- end -}}
-
 {{- define "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" -}}
 event-rate-limit-config.yaml
 {{- end -}}
@@ -202,8 +189,6 @@ admission-control-config.yaml
   {{- $args := include "t8s-cluster.clusterClass.args.base" (dict "context" .context) | fromYaml -}}
   {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}}
   {{- $args = set $args "authentication-config" (include "t8s-cluster.clusterClass.apiServer.authenticationConfigPath" (dict)) -}}
-  {{- $args = set $args "audit-policy-file" (include "t8s-cluster.clusterClass.apiServer.auditConfigPath" (dict)) -}}
-  {{- $args = set $args "audit-log-path" "-" -}}
   {{- $args = set $args "admission-control-config-file" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict)) -}}
   {{- $args = set $args "enable-admission-plugins" (include "t8s-cluster.clusterClass.apiServer.admissionPlugins" (dict) | fromYamlArray | join ",") -}}
   {{- $args = set $args "event-ttl" "4h" -}}
@@ -217,10 +202,6 @@ admission-control-config.yaml
         "path" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict))
         "fileName" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigFileName" (dict))
       )
-      (include "t8s-cluster.clusterClass.apiServer.auditConfigFileName" (dict)) (dict
-        "path" (include "t8s-cluster.clusterClass.apiServer.auditConfigPath" (dict))
-        "fileName" (include "t8s-cluster.clusterClass.apiServer.auditConfigFileName" (dict))
-      )
       (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict)) (dict
         "path" (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" (dict))
         "fileName" (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict))

charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml

@@ -1,12 +1,12 @@
 {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate" -}}
   {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}}
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
-kind: {{ .Values.controlPlane.hosted | ternary "K0sWorkerConfigTemplate" "KubeadmConfigTemplate" }}
+kind: KubeadmConfigTemplate
 metadata:
   name: {{ printf "%s-%s-compute-plane" .Release.Name (.gpu | ternary "gpu" "standard") }}
   namespace: {{ .Release.Namespace }}
   labels: {{- include "common.labels.standard" . | nindent 4 }}
 spec:
   template:
-    spec: {{- include (printf "t8s-cluster.clusterClass.bootstrapConfigTemplate.%s.spec" (.Values.controlPlane.hosted | ternary "k0smotron" "kubeadm")) (dict "gpu" .gpu "context" .context) | nindent 6 }}
+    spec: {{- include "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" (dict "gpu" .gpu "context" .context) | nindent 6 }}
 {{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml

@@ -29,10 +29,10 @@ spec:
         name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }}
     {{- end }}
     ref:
-      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+      apiVersion: controlplane.cluster.x-k8s.io/{{ .Values.controlPlane.hosted | ternary "v1alpha1" "v1beta1" }}
       {{- if .Values.controlPlane.hosted }}
-      kind: K0smotronControlPlaneTemplate
-      name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" .) }}
+      kind: HostedControlPlaneTemplate
+      name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" .) }}
       {{- else }}
       kind: KubeadmControlPlaneTemplate
       name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}
@@ -147,7 +147,8 @@ spec:
             - status: 'False'
               timeout: 300s
               type: Ready
-        nodeDrainTimeout: "8m"
+        nodeDrainTimeout: 8m
+        nodeDeletionTimeout: 15m
         strategy:
           type: RollingUpdate
           rollingUpdate:
@@ -164,7 +165,7 @@ spec:
           bootstrap:
             ref:
               apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
-              kind: {{ $.Values.controlPlane.hosted | ternary "K0sWorkerConfigTemplate" "KubeadmConfigTemplate" }}
+              kind: KubeadmConfigTemplate
               name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }}
           infrastructure:
             ref:

charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl → charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_helpers.tpl

@@ -1,5 +1,5 @@
-{{- define "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" -}}
+{{- define "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" -}}
   {{/* the full context is needed for .Files.Get */}}
-  {{- $inputs := dict "spec" (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" $) -}}
+  {{- $inputs := dict "spec" (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.spec" $) -}}
   {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}}
 {{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml

@@ -0,0 +1,71 @@
+{{/*
+KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades.
+Here we are generating a hash suffix.
+This function needs the whole `$` context to be able to use `.Files.Get`
+*/}}
+{{- define "t8s-cluster.clusterClass.hostedControlPlaneTemplate.spec" -}}
+deployment:
+  apiServer:
+    mounts:
+      config:
+        path: {{ include "t8s-cluster.clusterClass.configPath" (dict) }}
+        configMap:
+          name: {{ printf "%s-apiserver" $.Release.Name }}
+          {{- $items := list -}}
+          {{- range $name, $file := mustMerge (include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml) (include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml) -}}
+            {{- $items = append $items (dict "key" $name "path" (get $file "fileName" | required (printf "missing fileName for %s" $file))) -}}
+          {{- end }}
+          items: {{- toYaml $items | nindent 10 }}
+    args: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 6 }}
+    resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }}
+    audit:
+      webhook:
+        targets:
+          - server: https://k8s.master.wazuh.teuto.net/{{ .Release.Namespace }}/{{ .Release.Name }}
+            authentication:
+              secretName: wazuh-audit-webhook
+              secretNamespace: capi-hosted-control-plane-system
+      policy:
+        # Long-running requests like watches will not generate an audit event in RequestReceived.
+        omitStages:
+          - RequestReceived
+        rules:
+          - level: None
+            users:
+              - system:kube-controller-manager
+              - system:kube-scheduler
+              - system:apiserver
+          - level: None
+            resources:
+              - group: coordination.k8s.io
+                resources:
+                  - leases
+              - group: ""
+                resources:
+                  - events
+          - level: Metadata
+            verbs: [] # All verbs
+            resources:
+              - group: ""
+                resources:
+                  - secrets
+          - level: Metadata
+            verbs:
+              - create
+              - update
+              - patch
+              - delete
+              - deletecollection
+            resources: [] # All resources
+  controllerManager:
+    args: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 6 }}
+    resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }}
+    replicas: 1
+  scheduler:
+    args: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 6 }}
+    resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }}
+    replicas: 1
+gateway:
+  namespace: capi-hosted-control-plane-system
+  name: controlplane
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/hostedControlPlaneTemplate.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.controlPlane.hosted -}}
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: HostedControlPlaneTemplate
+metadata:
+  name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" .) }}
+  namespace: {{ $.Release.Namespace }}
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+spec:
+  template:
+    spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.spec" . | default (dict) | fromYaml | toYaml | nindent 6 }}
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml

@@ -4,7 +4,7 @@ Here we are generating a hash suffix.
 */}}
 {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.spec" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
-flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" }}
+flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" | required "flavor is required" }}
 image:
   filter:
     name: placeholder

charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl

@@ -1,7 +1,7 @@
 {{- define "t8s-cluster.patches.kubelet.imagePulls" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $values := dict -}}
-  {{- if and (semverCompare ">=1.27.0" (include "t8s-cluster.k8s-version" .context)) (gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1) -}}
+  {{- if gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1 -}}
     {{- $values = mustMerge $values (dict "serializeImagePulls" false "maxParallelImagePulls" .Values.global.kubeletExtraConfig.maxParallelImagePulls) -}}
   {{- end -}}
   {{- toYaml $values -}}

charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin/cinder-csi-plugin.yaml

@@ -46,11 +46,6 @@ spec:
       name: cloud-config
     csi:
       plugin:
-        # TODO: keep this until https://github.com/k0sproject/k0s/issues/1842 is addressed
-        {{- if .Values.controlPlane.hosted }}
-        nodePlugin:
-          kubeletDir: /var/lib/k0s/kubelet
-        {{- end }}
         controllerPlugin:
           extraArgs:
             - --with-topology=false

charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml

@@ -55,22 +55,16 @@ spec:
               - op: replace
                 path: /kind
                 value: Deployment
-              {{- if gt (.Values.version.minor | int) 28 }}
               - op: remove
                 path: /spec/template/spec/nodeSelector
-              {{- end }}
   {{- end }}
   values:
     priorityClassName: system-cluster-critical
-  {{- if le (.Values.version.minor | int) 28 }}
+  {{- if .Values.controlPlane.hosted }}
     tolerations:
+      - key: node.cluster.x-k8s.io/uninitialized
+        effect: NoSchedule
       - key: node.cloudprovider.kubernetes.io/uninitialized
         value: "true"
         effect: NoSchedule
-    {{- if not .Values.controlPlane.hosted }}
-      - key: node-role.kubernetes.io/control-plane
-        effect: NoSchedule
-    nodeSelector:
-      node-role.kubernetes.io/control-plane: ""
-    {{- end }}
   {{- end }}