diff --git a/charts/t8s-cluster/ci/autoscaling-values.yaml b/charts/t8s-cluster/ci/autoscaling-values.yaml new file mode 100644 index 0000000000..1c4aa633fa --- /dev/null +++ b/charts/t8s-cluster/ci/autoscaling-values.yaml @@ -0,0 +1,6 @@ +nodePools: + autoscaling: + flavor: standard.2.4096 + replicas: + min: 1 + max: 3 diff --git a/charts/t8s-cluster/files/audit-config.yaml b/charts/t8s-cluster/files/audit-config.yaml deleted file mode 100644 index 4ae55a2ef5..0000000000 --- a/charts/t8s-cluster/files/audit-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: audit.k8s.io/v1 -kind: Policy -# Long-running requests like watches will not generate an audit event in RequestReceived. -omitStages: - - "RequestReceived" -rules: - - level: Metadata - verbs: [] # All verbs - resources: [] # All resources diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl index 1ac843eb93..d4971a7a6e 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl @@ -35,6 +35,7 @@ openstack {{- toYaml $args -}} {{- end -}} +{{/* this can be split up with containerd >=2.0.0 */}} {{- define "t8s-cluster.clusterClass.containerdConfig.plugins" -}} {{- $_ := mustMerge . (pick .context "Values") -}} [plugins] @@ -129,12 +130,6 @@ server = {{ printf "https://%s" .registry | quote }} {{- define "t8s-cluster.clusterClass.args.shared" -}} {{- $args := include "t8s-cluster.clusterClass.args.base" (dict) | fromYaml -}} - {{- $args = mustMerge (dict - "authorization-always-allow-paths" (list "/healthz" "/readyz" "/livez" "/metrics" | join ",") - "bind-address" "0.0.0.0" - ) - $args - -}} {{- toYaml $args -}} {{- end -}} @@ -170,10 +165,6 @@ server = {{ printf "https://%s" .registry | quote }} {{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.authenticationConfigFileName" (dict) -}} {{- end -}} -{{- define "t8s-cluster.clusterClass.apiServer.auditConfigPath" -}} -{{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.auditConfigFileName" (dict) -}} -{{- end -}} - {{- define "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" -}} {{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict) -}} {{- end -}} @@ -186,10 +177,6 @@ server = {{ printf "https://%s" .registry | quote }} authentication-config.yaml {{- end -}} -{{- define "t8s-cluster.clusterClass.apiServer.auditConfigFileName" -}} -audit-config.yaml -{{- end -}} - {{- define "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" -}} event-rate-limit-config.yaml {{- end -}} @@ -202,8 +189,6 @@ admission-control-config.yaml {{- $args := include "t8s-cluster.clusterClass.args.base" (dict "context" .context) | fromYaml -}} {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}} {{- $args = set $args "authentication-config" (include "t8s-cluster.clusterClass.apiServer.authenticationConfigPath" (dict)) -}} - {{- $args = set $args "audit-policy-file" (include "t8s-cluster.clusterClass.apiServer.auditConfigPath" (dict)) -}} - {{- $args = set $args "audit-log-path" "-" -}} {{- $args = set $args "admission-control-config-file" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict)) -}} {{- $args = set $args "enable-admission-plugins" (include "t8s-cluster.clusterClass.apiServer.admissionPlugins" (dict) | fromYamlArray | join ",") -}} {{- $args = set $args "event-ttl" "4h" -}} @@ -217,10 +202,6 @@ admission-control-config.yaml "path" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict)) "fileName" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigFileName" (dict)) ) - (include "t8s-cluster.clusterClass.apiServer.auditConfigFileName" (dict)) (dict - "path" (include "t8s-cluster.clusterClass.apiServer.auditConfigPath" (dict)) - "fileName" (include "t8s-cluster.clusterClass.apiServer.auditConfigFileName" (dict)) - ) (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict)) (dict "path" (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" (dict)) "fileName" (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict)) diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml index b113f0c504..636fb5b70a 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml @@ -1,12 +1,12 @@ {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate" -}} {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: {{ .Values.controlPlane.hosted | ternary "K0sWorkerConfigTemplate" "KubeadmConfigTemplate" }} +kind: KubeadmConfigTemplate metadata: name: {{ printf "%s-%s-compute-plane" .Release.Name (.gpu | ternary "gpu" "standard") }} namespace: {{ .Release.Namespace }} labels: {{- include "common.labels.standard" . | nindent 4 }} spec: template: - spec: {{- include (printf "t8s-cluster.clusterClass.bootstrapConfigTemplate.%s.spec" (.Values.controlPlane.hosted | ternary "k0smotron" "kubeadm")) (dict "gpu" .gpu "context" .context) | nindent 6 }} + spec: {{- include "t8s-cluster.clusterClass.bootstrapConfigTemplate.kubeadm.spec" (dict "gpu" .gpu "context" .context) | nindent 6 }} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_k0smotronConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_k0smotronConfigTemplateSpec.yaml deleted file mode 100644 index f987219cf1..0000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_k0smotronConfigTemplateSpec.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate.k0smotron.spec" -}} - {{- $_ := mustMerge . (pick .context "Values") -}} - {{- $kubeletExtraArgs := list -}} - {{- range $key, $value := include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | fromYaml -}} - {{- $kubeletExtraArgs = append $kubeletExtraArgs (printf "--%s=%s" $key $value) -}} - {{- end -}} -args: - - --cri-socket=remote:/run/containerd/containerd.sock - {{- if le (.Values.version.minor | int) 28 }} - - --enable-cloud-provider - {{- end }} - - {{ printf `--kubelet-extra-args="%s"` ($kubeletExtraArgs | join " ") }} -files: {{- include "t8s-cluster.clusterClass.configTemplate.files" (dict "context" .context "gpu" .gpu "excludePatches" true) | nindent 2 }} - {{/* this service isn't needed by k0s */}} - {{- $preStartCommands := list - "systemctl disable --now kubelet.service" - }} - {{- $preStartCommands = concat $preStartCommands - (include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" .context) | fromYamlArray) - (include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | fromYamlArray) - }} -preStartCommands: {{- $preStartCommands | default (list) | toYaml | nindent 2 }} -{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml index 1cb3629fb5..ed5d7af805 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml @@ -29,10 +29,10 @@ spec: name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }} {{- end }} ref: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + apiVersion: controlplane.cluster.x-k8s.io/{{ .Values.controlPlane.hosted | ternary "v1alpha1" "v1beta1" }} {{- if .Values.controlPlane.hosted }} - kind: K0smotronControlPlaneTemplate - name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" .) }} + kind: HostedControlPlaneTemplate + name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" .) }} {{- else }} kind: KubeadmControlPlaneTemplate name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} @@ -147,7 +147,8 @@ spec: - status: 'False' timeout: 300s type: Ready - nodeDrainTimeout: "8m" + nodeDrainTimeout: 8m + nodeDeletionTimeout: 15m strategy: type: RollingUpdate rollingUpdate: @@ -164,7 +165,7 @@ spec: bootstrap: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: {{ $.Values.controlPlane.hosted | ternary "K0sWorkerConfigTemplate" "KubeadmConfigTemplate" }} + kind: KubeadmConfigTemplate name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }} infrastructure: ref: diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_helpers.tpl similarity index 61% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl rename to charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_helpers.tpl index 51b65b60d2..f6b300d1c5 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_helpers.tpl @@ -1,5 +1,5 @@ -{{- define "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" -}} +{{- define "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" -}} {{/* the full context is needed for .Files.Get */}} - {{- $inputs := dict "spec" (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" $) -}} + {{- $inputs := dict "spec" (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.spec" $) -}} {{- mustToJson $inputs | toString | quote | sha1sum | trunc 8 -}} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml new file mode 100644 index 0000000000..35152c847e --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml @@ -0,0 +1,71 @@ +{{/* +KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades. +Here we are generating a hash suffix. +This function needs the whole `$` context to be able to use `.Files.Get` +*/}} +{{- define "t8s-cluster.clusterClass.hostedControlPlaneTemplate.spec" -}} +deployment: + apiServer: + mounts: + config: + path: {{ include "t8s-cluster.clusterClass.configPath" (dict) }} + configMap: + name: {{ printf "%s-apiserver" $.Release.Name }} + {{- $items := list -}} + {{- range $name, $file := mustMerge (include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml) (include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml) -}} + {{- $items = append $items (dict "key" $name "path" (get $file "fileName" | required (printf "missing fileName for %s" $file))) -}} + {{- end }} + items: {{- toYaml $items | nindent 10 }} + args: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 6 }} + resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }} + audit: + webhook: + targets: + - server: https://k8s.master.wazuh.teuto.net/{{ .Release.Namespace }}/{{ .Release.Name }} + authentication: + secretName: wazuh-audit-webhook + secretNamespace: capi-hosted-control-plane-system + policy: + # Long-running requests like watches will not generate an audit event in RequestReceived. + omitStages: + - RequestReceived + rules: + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:apiserver + - level: None + resources: + - group: coordination.k8s.io + resources: + - leases + - group: "" + resources: + - events + - level: Metadata + verbs: [] # All verbs + resources: + - group: "" + resources: + - secrets + - level: Metadata + verbs: + - create + - update + - patch + - delete + - deletecollection + resources: [] # All resources + controllerManager: + args: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 6 }} + resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }} + replicas: 1 + scheduler: + args: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 6 }} + resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }} + replicas: 1 +gateway: + namespace: capi-hosted-control-plane-system + name: controlplane +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneConfigMap.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/hostedControlPlaneConfigMap.yaml similarity index 100% rename from charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneConfigMap.yaml rename to charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/hostedControlPlaneConfigMap.yaml diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/hostedControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/hostedControlPlaneTemplate.yaml new file mode 100644 index 0000000000..27a673a3ed --- /dev/null +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/hostedControlPlaneTemplate.yaml @@ -0,0 +1,11 @@ +{{- if .Values.controlPlane.hosted -}} +apiVersion: controlplane.cluster.x-k8s.io/v1alpha1 +kind: HostedControlPlaneTemplate +metadata: + name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" .) }} + namespace: {{ $.Release.Namespace }} + labels: {{- include "common.labels.standard" $ | nindent 4 }} +spec: + template: + spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.spec" . | default (dict) | fromYaml | toYaml | nindent 6 }} +{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml deleted file mode 100644 index c50027a66d..0000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{/* -KubeadmControlPlaneTemplate is immutable. We need to create new versions during upgrades. -Here we are generating a hash suffix. -This function needs the whole `$` context to be able to use `.Files.Get` -*/}} -{{- define "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" -}} -replicas: {{ $.Values.controlPlane.singleNode | ternary 1 3 }} -resources: {{- include "common.resources" .Values.controlPlane | nindent 2 }} -service: - type: LoadBalancer - apiPort: 6443 - konnectivityPort: 8132 -controllerPlaneFlags: - - --disable-components=metrics-server -mounts: - - configMap: - name: {{ printf "%s-apiserver" $.Release.Name }} - {{- $items := list -}} - {{- range $name, $file := mustMerge (include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml) (include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml) -}} - {{- $items = append $items (dict "key" $name "path" (get $file "fileName" | required (printf "missing fileName for %s" $file))) -}} - {{- end }} - items: {{- toYaml $items | nindent 6 }} - path: {{ include "t8s-cluster.clusterClass.configPath" (dict) }} -k0sConfig: - apiVersion: k0s.k0sproject.io/v1beta1 - kind: ClusterConfig - spec: - network: - provider: custom - api: - extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 8 }} - controllerManager: - extraArgs: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 8 }} - scheduler: - extraArgs: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 8 }} - telemetry: - enabled: false - workerProfiles: - - name: default - {{- $values := dict "cgroupDriver" "systemd" }} - {{ $values = mustMerge $values (include "t8s-cluster.kubelet.options" (dict) | fromYaml) }} - {{ $values = mustMerge $values (include "t8s-cluster.patches.kubelet.imagePulls" (dict "context" .) | fromYaml) }} - values: {{- toYaml $values | nindent 10 }} - featureGates: {{- range $featureGate, $components := include "t8s-cluster.featureGates" (dict) | fromYaml }} - - name: {{ $featureGate }} - enabled: true - components: {{- toYaml $components | nindent 10 }} - {{- end }} -etcd: - persistence: - size: 5Gi -{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml deleted file mode 100644 index 73037a18d3..0000000000 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.controlPlane.hosted -}} -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: K0smotronControlPlaneTemplate -metadata: - name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.specHash" .) }} - namespace: {{ $.Release.Namespace }} - labels: {{- include "common.labels.standard" $ | nindent 4 }} -spec: - template: - metadata: - annotations: - checksum/configmap: {{ include "common.utils.checksumTemplate" (dict "path" "/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneConfigMap.yaml" "context" $) }} - spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" . | nindent 6 }} -{{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml index 349d996c6b..54d3d0acf2 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml @@ -4,7 +4,7 @@ Here we are generating a hash suffix. */}} {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.spec" -}} {{- $_ := mustMerge . (pick .context "Values") -}} -flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" }} +flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" | required "flavor is required" }} image: filter: name: placeholder diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl index 8cffeaf210..d1a0463354 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/patches/_kubelet.tpl @@ -1,7 +1,7 @@ {{- define "t8s-cluster.patches.kubelet.imagePulls" -}} {{- $_ := mustMerge . (pick .context "Values") -}} {{- $values := dict -}} - {{- if and (semverCompare ">=1.27.0" (include "t8s-cluster.k8s-version" .context)) (gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1) -}} + {{- if gt (int .Values.global.kubeletExtraConfig.maxParallelImagePulls) 1 -}} {{- $values = mustMerge $values (dict "serializeImagePulls" false "maxParallelImagePulls" .Values.global.kubeletExtraConfig.maxParallelImagePulls) -}} {{- end -}} {{- toYaml $values -}} diff --git a/charts/t8s-cluster/templates/management-cluster/etcd-defrag.yaml b/charts/t8s-cluster/templates/management-cluster/etcd-defrag.yaml deleted file mode 100644 index ae78f50777..0000000000 --- a/charts/t8s-cluster/templates/management-cluster/etcd-defrag.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if .Values.controlPlane.hosted -}} - {{- include "t8s-cluster.etcd-defrag" (dict "context" $ "hosted" true) | nindent 0 -}} -{{- end -}} diff --git a/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin/cinder-csi-plugin.yaml b/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin/cinder-csi-plugin.yaml index adbc2c5948..f8babe0988 100644 --- a/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin/cinder-csi-plugin.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/cinder-csi-plugin/cinder-csi-plugin.yaml @@ -46,11 +46,6 @@ spec: name: cloud-config csi: plugin: - # TODO: keep this until https://github.com/k0sproject/k0s/issues/1842 is addressed - {{- if .Values.controlPlane.hosted }} - nodePlugin: - kubeletDir: /var/lib/k0s/kubelet - {{- end }} controllerPlugin: extraArgs: - --with-topology=false diff --git a/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml b/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml index 5dab7f53f6..e3765a73f8 100644 --- a/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml @@ -55,22 +55,16 @@ spec: - op: replace path: /kind value: Deployment - {{- if gt (.Values.version.minor | int) 28 }} - op: remove path: /spec/template/spec/nodeSelector - {{- end }} {{- end }} values: priorityClassName: system-cluster-critical - {{- if le (.Values.version.minor | int) 28 }} + {{- if .Values.controlPlane.hosted }} tolerations: + - key: node.cluster.x-k8s.io/uninitialized + effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - {{- if not .Values.controlPlane.hosted }} - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - nodeSelector: - node-role.kubernetes.io/control-plane: "" - {{- end }} {{- end }} diff --git a/charts/t8s-cluster/templates/workload-cluster/tests/cni-cilium.yaml b/charts/t8s-cluster/templates/workload-cluster/tests/cni-cilium.yaml new file mode 100644 index 0000000000..9705821f91 --- /dev/null +++ b/charts/t8s-cluster/templates/workload-cluster/tests/cni-cilium.yaml @@ -0,0 +1,73 @@ +{{- if and (eq (include "t8s-cluster.cni" .) "cilium") .Release.IsUpgrade false -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-cni" .Release.Name }} + namespace: {{ .Release.Namespace}} + labels: {{- include "common.labels.standard" . | nindent 4 }} + annotations: + helm.sh/hook: test +spec: + template: + spec: + securityContext: + runAsNonRoot: true + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + automountServiceAccountToken: false + enableServiceLinks: false + initContainers: + - name: test-kubeconfig + image: {{ include "common.images.image" (dict "imageRoot" .Values.global.kubectl.image "global" .Values.global) }} + imagePullPolicy: {{ include "common.images.pullPolicy" .Values.global.kubectl.image }} + securityContext: &securityContext + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + volumeMounts: &kubeconfigMount + - mountPath: /tmp/workload-kubeconfig + name: workload-kubeconfig + subPath: value + env: &kubeconfigEnv + - name: KUBECONFIG + value: /tmp/workload-kubeconfig + command: + - kubectl + - get + - pods + - -A + - name: connectivity-test + image: {{ include "common.images.image" (dict "imageRoot" .Values.global.cilium.image "global" .Values.global) }} + imagePullPolicy: {{ include "common.images.pullPolicy" .Values.global.cilium.image }} + securityContext: *securityContext + volumeMounts: *kubeconfigMount + env: *kubeconfigEnv + command: + - cilium + - connectivity + - test + - --test-namespace=cilium-test + containers: + - name: delete-namespace + image: {{ include "common.images.image" (dict "imageRoot" .Values.global.kubectl.image "global" .Values.global) }} + imagePullPolicy: {{ include "common.images.pullPolicy" .Values.global.kubectl.image }} + securityContext: *securityContext + volumeMounts: *kubeconfigMount + env: *kubeconfigEnv + command: + - kubectl + - delete + - namespace + - cilium-test-1 + restartPolicy: Never + volumes: + - name: workload-kubeconfig + secret: + secretName: {{ .Release.Name }}-kubeconfig +{{- end -}} diff --git a/charts/t8s-cluster/values.schema.json b/charts/t8s-cluster/values.schema.json index 685e09e6b1..43b61c2bb9 100644 --- a/charts/t8s-cluster/values.schema.json +++ b/charts/t8s-cluster/values.schema.json @@ -179,9 +179,6 @@ "$ref": "#/$defs/resourceRequirements" } }, - "required": [ - "flavor" - ], "additionalProperties": false }, "cloud": {