feat(t8s-cluster/cilium): enable kubeProxy replacement

Author: cwrau

charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl

@@ -145,9 +145,13 @@ server = {{ printf "https://%s" .registry | quote }}
 {{- end }}
 
 {{- define "t8s-cluster.clusterClass.args.controllerManager" -}}
+  {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $args := include "t8s-cluster.clusterClass.args.shared" (dict) | fromYaml -}}
   {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}}
   {{- $args = set $args "terminated-pod-gc-threshold" "100" -}}
+  {{- if .Values.controlPlane.hosted -}}
+    {{- $args = set $args "allocate-node-cidrs" "true" -}}
+  {{- end }}
   {{- toYaml $args -}}
 {{- end }}
 

charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml

@@ -65,6 +65,8 @@ deployment:
     args: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 6 }}
     resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }}
     replicas: 1
+kubeProxy:
+  disabled: {{ .Values.controlPlane.hosted }}
 gateway:
   namespace: capi-hosted-control-plane-system
   name: controlplane

charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml

@@ -52,6 +52,10 @@ identityRef:
     {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "VXLAN" (dict "port" 8472 "protocol" "udp") -}}
     {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (http)" (dict "port" 4240) -}}
     {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (ping)" (dict "protocol" "icmp") -}}
+    {{- if .Values.controlPlane.hosted -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "allow pod-pod native routing (ingress)" (dict "remoteIPPrefix" "10.0.0.0/16" "protocol" nil) -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "allow node-pod native routing (ingress)" (dict "remoteIPPrefix" "10.6.0.0/16" "protocol" nil) -}}
+    {{- end -}}
   {{- end }}
   {{- range $name, $securityGroupRule := $cniSecurityGroupRules -}}
     {{- $securityGroupRules = set $securityGroupRules (printf "%s %s" $cni $name) $securityGroupRule -}}
@@ -97,17 +101,18 @@ identityRef:
       "description" ($securityGroupRule.description | default $name)
       "direction" "ingress"
       "etherType" "IPv4"
-      "protocol" ($securityGroupRule.protocol | default "tcp")
+      "protocol" (hasKey $securityGroupRule "protocol" | ternary $securityGroupRule.protocol "tcp")
     -}}
-    {{- if hasKey $securityGroupRule "remoteGroupID" -}}
-      {{- $_securityGroupRule = set $_securityGroupRule "remoteGroupID" $securityGroupRule.remoteGroupID -}}
-    {{- end -}}
     {{- if or (hasKey $securityGroupRule "port") (and (hasKey $securityGroupRule "portMin") (hasKey $securityGroupRule "portMax")) -}}
       {{- $_securityGroupRule = set $_securityGroupRule "portRangeMin" ($securityGroupRule.portMin | default $securityGroupRule.port) -}}
       {{- $_securityGroupRule = set $_securityGroupRule "portRangeMax" ($securityGroupRule.portMax | default $securityGroupRule.port) -}}
     {{- end -}}
-    {{- if hasKey $securityGroupRule "remoteManagedGroups" -}}
+    {{- if hasKey $securityGroupRule "remoteGroupID" -}}
+      {{- $_securityGroupRule = set $_securityGroupRule "remoteGroupID" $securityGroupRule.remoteGroupID -}}
+    {{- else if hasKey $securityGroupRule "remoteManagedGroups" -}}
       {{- $_securityGroupRule = set $_securityGroupRule "remoteManagedGroups" $securityGroupRule.remoteManagedGroups -}}
+    {{- else if hasKey $securityGroupRule "remoteIPPrefix" -}}
+      {{- $_securityGroupRule = set $_securityGroupRule "remoteIPPrefix" $securityGroupRule.remoteIPPrefix -}}
     {{- else -}}
       {{- $_securityGroupRule = set $_securityGroupRule "remoteManagedGroups" $allRemoteManagedGroups -}}
     {{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml

@@ -5,6 +5,11 @@ Here we are generating a hash suffix.
 {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.spec" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
 flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" | required "flavor is required" }}
+  {{- if .Values.controlPlane.hosted }}
+ports:
+  - allowedAddressPairs:
+      - ipAddress: 10.0.0.0/16
+  {{- end }}
 image:
   filter:
     name: placeholder

charts/t8s-cluster/templates/workload-cluster/cni-cilium.yaml

@@ -27,19 +27,45 @@ spec:
   values:
     nodePort:
       enabled: true
-    # enable eBPF based routing instead of iptables
+    {{- if .Values.controlPlane.hosted }}
     bpf:
-      masquerade: false # disable for now as this creates routing problems
+      masquerade: true
+      tproxy: true
+      enableTCX: true
+    endpointRoutes:
+      enabled: true
+    bandwidthManager:
+      enabled: true
     egressGateway:
-      enabled: false # disable for now as this depends on bpf.masquerade
-    # enable eBPF bases host routing
-    # currently not really possible with CAPI, as they don't support disabling the built-in kube-proxy
-    # kubeProxyReplacement: strict
+      enabled: true
+    # currently not really possible with kubeadmcontrolplane, as they don't support disabling the built-in kube-proxy
+    kubeProxyReplacement: true
+    ipam:
+      mode: kubernetes
+    routingMode: native
+    autoDirectNodeRoutes: true
+    directRoutingSkipUnreachable: true
+    localRedirectPolicies:
+      enabled: true
+    k8s:
+      requireIPv4PodCIDR: true
+      {{- $gateway := lookup "gateway.networking.k8s.io/v1" "Gateway" "capi-hosted-control-plane-system" "controlplane" -}}
+      {{- if not $gateway -}}
+        {{- fail "Hosted control plane Gateway 'controlplane' in namespace 'capi-hosted-control-plane-system' not found" -}}
+      {{- else }}
+    k8sServiceHost: {{ printf "%s.%s.%s" .Release.Name .Release.Namespace (replace "*." "" (index $gateway.spec.listeners 0).hostname) }}
+      {{- end }}
+    k8sServicePort: 443
+    ipv4NativeRoutingCIDR: 10.0.0.0/16 # default net, see hosted control plane controller
+    {{- end }}
 
     rollOutCiliumPods: true
     encryption:
       enabled: false
       nodeEncryption: false
+    envoy:
+      prometheus:
+        enabled: true
     hubble:
       metrics:
         enabled:
@@ -57,6 +83,12 @@ spec:
       ui:
         rollOutPods: true
         enabled: true
+      export:
+        static:
+          enabled: true
+          filePath: stdout
+          allowList:
+            - '{"verdict":["DROPPED","ERROR"]}'
     operator:
       rollOutPods: true
       prometheus: