feat(t8s-cluster/cilium): enable kubeProxy replacement

charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl

@@ -145,9 +145,13 @@ server = {{ printf "https://%s" .registry | quote }}
 {{- end }}
 
 {{- define "t8s-cluster.clusterClass.args.controllerManager" -}}
+  {{- $_ := mustMerge . (pick .context "Values") -}}
   {{- $args := include "t8s-cluster.clusterClass.args.shared" (dict) | fromYaml -}}
   {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}}
   {{- $args = set $args "terminated-pod-gc-threshold" "100" -}}
+  {{- if .Values.controlPlane.hosted -}}
+    {{- $args = set $args "allocate-node-cidrs" "true" -}}
+  {{- end }}
   {{- toYaml $args -}}
 {{- end }}
 

charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml

@@ -65,6 +65,8 @@ deployment:
     args: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 6 }}
     resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }}
     replicas: 1
+kubeProxy:
+  disabled: {{ .Values.controlPlane.hosted }}
 gateway:
   namespace: capi-hosted-control-plane-system
   name: controlplane

charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml

@@ -34,39 +34,49 @@ identityRef:
   name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" (dict "context" .context) }}
   cloudName: {{ include "t8s-cluster.clusterClass.cloudName" (dict) }}
 
-  {{- $securityGroupRules := dict -}}
-
   {{- $cni := include "t8s-cluster.cni" .context -}}
+
+  {{- $allowNativeRouting := and .Values.controlPlane.hosted (eq $cni "cilium") -}}
+
+  {{- $securityGroupRules := dict -}}
   {{- $controlPlaneSecurityGroup := "controlplane" -}}
   {{- $computePlaneSecurityGroup := "worker" -}}
   {{- $allRemoteManagedGroups := list $computePlaneSecurityGroup -}}
   {{- if not .Values.controlPlane.hosted -}}
     {{ $allRemoteManagedGroups = append $allRemoteManagedGroups $controlPlaneSecurityGroup -}}
   {{- end -}}
 
-  {{- $cniSecurityGroupRules := dict -}}
-  {{- if eq $cni "calico" }}
-    {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "BGP" (dict "port" 179) -}}
-    {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "IP-in-IP" (dict "protocol" "4") -}}
-  {{- else if eq $cni "cilium" -}}
-    {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "VXLAN" (dict "port" 8472 "protocol" "udp") -}}
-    {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (http)" (dict "port" 4240) -}}
-    {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (ping)" (dict "protocol" "icmp") -}}
+  {{- if not $allowNativeRouting -}}
+    {{- $cniSecurityGroupRules := dict -}}
+    {{- if eq $cni "calico" }}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "BGP" (dict "port" 179) -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "IP-in-IP" (dict "protocol" "4") -}}
+    {{- else if eq $cni "cilium" -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "VXLAN" (dict "port" 8472 "protocol" "udp") -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (http)" (dict "port" 4240) -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (ping)" (dict "protocol" "icmp") -}}
+    {{- if .Values.controlPlane.hosted -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "allow pod-pod native routing (ingress)" (dict "remoteIPPrefix" "10.0.0.0/16" "protocol" nil) -}}
+      {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "allow node-pod native routing (ingress)" (dict "remoteIPPrefix" "10.6.0.0/16" "protocol" nil) -}}
+    {{- end -}}
   {{- end }}
-  {{- range $name, $securityGroupRule := $cniSecurityGroupRules -}}
-    {{- $securityGroupRules = set $securityGroupRules (printf "%s %s" $cni $name) $securityGroupRule -}}
-  {{- end -}}
+    {{- range $name, $securityGroupRule := $cniSecurityGroupRules -}}
+      {{- $securityGroupRules = set $securityGroupRules (printf "%s %s" $cni $name) $securityGroupRule -}}
+    {{- end -}}
 
-  {{- $metricsSecurityGroupRules := dict -}}
-  {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "prometheus node-exporter" (dict "port" 9100 "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
-  {{- if not .Values.controlPlane.hosted -}}
-    {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "etcd" (dict "port" 2381 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
-    {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-proxy" (dict "port" 10249 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
-    {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-controller-manager" (dict "port" 10257 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
-    {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-scheduler" (dict "port" 10259 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
-  {{- end }}
-  {{- range $name, $securityGroupRule := $metricsSecurityGroupRules -}}
-    {{- $securityGroupRules = set $securityGroupRules (printf "metrics (%s)" $name) $securityGroupRule -}}
+    {{- $metricsSecurityGroupRules := dict -}}
+    {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "prometheus node-exporter" (dict "port" 9100 "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
+    {{- if not .Values.controlPlane.hosted -}}
+      {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "etcd" (dict "port" 2381 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
+      {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-proxy" (dict "port" 10249 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
+      {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-controller-manager" (dict "port" 10257 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
+      {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-scheduler" (dict "port" 10259 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}}
+    {{- end }}
+    {{- range $name, $securityGroupRule := $metricsSecurityGroupRules -}}
+      {{- $securityGroupRules = set $securityGroupRules (printf "metrics (%s)" $name) $securityGroupRule -}}
+    {{- end -}}
+  {{- else -}}
+    {{- $securityGroupRules = set $securityGroupRules "cilium allow pod-pod native routing" (dict "remoteIPPrefix" "10.0.0.0/16" "protocol" nil) -}}
   {{- end -}}
 
   {{- range $name, $securityGroupRule := .Values.controlPlane.additionalSecurityGroupRules -}}
@@ -97,17 +107,18 @@ identityRef:
       "description" ($securityGroupRule.description | default $name)
       "direction" "ingress"
       "etherType" "IPv4"
-      "protocol" ($securityGroupRule.protocol | default "tcp")
+      "protocol" (hasKey $securityGroupRule "protocol" | ternary $securityGroupRule.protocol "tcp")
     -}}
-    {{- if hasKey $securityGroupRule "remoteGroupID" -}}
-      {{- $_securityGroupRule = set $_securityGroupRule "remoteGroupID" $securityGroupRule.remoteGroupID -}}
-    {{- end -}}
     {{- if or (hasKey $securityGroupRule "port") (and (hasKey $securityGroupRule "portMin") (hasKey $securityGroupRule "portMax")) -}}
       {{- $_securityGroupRule = set $_securityGroupRule "portRangeMin" ($securityGroupRule.portMin | default $securityGroupRule.port) -}}
       {{- $_securityGroupRule = set $_securityGroupRule "portRangeMax" ($securityGroupRule.portMax | default $securityGroupRule.port) -}}
     {{- end -}}
-    {{- if hasKey $securityGroupRule "remoteManagedGroups" -}}
+    {{- if hasKey $securityGroupRule "remoteGroupID" -}}
+      {{- $_securityGroupRule = set $_securityGroupRule "remoteGroupID" $securityGroupRule.remoteGroupID -}}
+    {{- else if hasKey $securityGroupRule "remoteManagedGroups" -}}
       {{- $_securityGroupRule = set $_securityGroupRule "remoteManagedGroups" $securityGroupRule.remoteManagedGroups -}}
+    {{- else if hasKey $securityGroupRule "remoteIPPrefix" -}}
+      {{- $_securityGroupRule = set $_securityGroupRule "remoteIPPrefix" $securityGroupRule.remoteIPPrefix -}}
     {{- else -}}
       {{- $_securityGroupRule = set $_securityGroupRule "remoteManagedGroups" $allRemoteManagedGroups -}}
     {{- end -}}
@@ -124,6 +135,8 @@ managedSecurityGroups:
   allNodesSecurityGroupRules: {{- toYaml $allNodesSecurityGroupRules | nindent 4 }}
   controlPlaneNodesSecurityGroupRules: {{- toYaml $controlPlaneNodesSecurityGroupRules | nindent 4 }}
   workerNodesSecurityGroupRules: {{- toYaml $workerNodesSecurityGroupRules | nindent 4 }}
+  {{/* for native routing we'd have to allow ingress from the pods and the nodes; which is just everything... */}}
+  allowAllInClusterTraffic: {{ $allowNativeRouting }}
 managedSubnets:
   - cidr: 10.6.0.0/24
 {{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml

@@ -5,6 +5,11 @@ Here we are generating a hash suffix.
 {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.spec" -}}
   {{- $_ := mustMerge . (pick .context "Values") -}}
 flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" | required "flavor is required" }}
+  {{- if .Values.controlPlane.hosted }}
+ports:
+  - allowedAddressPairs:
+      - ipAddress: 10.0.0.0/16
+  {{- end }}
 image:
   filter:
     name: placeholder

charts/t8s-cluster/templates/workload-cluster/cni-cilium.yaml

@@ -27,19 +27,45 @@ spec:
   values:
     nodePort:
       enabled: true
-    # enable eBPF based routing instead of iptables
+    {{- if .Values.controlPlane.hosted }}
     bpf:
-      masquerade: false # disable for now as this creates routing problems
+      masquerade: true
+      tproxy: true
+      enableTCX: true
+    endpointRoutes:
+      enabled: true
+    bandwidthManager:
+      enabled: true
     egressGateway:
-      enabled: false # disable for now as this depends on bpf.masquerade
-    # enable eBPF bases host routing
-    # currently not really possible with CAPI, as they don't support disabling the built-in kube-proxy
-    # kubeProxyReplacement: strict
+      enabled: true
+    # currently not really possible with kubeadmcontrolplane, as they don't support disabling the built-in kube-proxy
+    kubeProxyReplacement: true
+    ipam:
+      mode: kubernetes
+    routingMode: native
+    autoDirectNodeRoutes: true
+    directRoutingSkipUnreachable: true
+    localRedirectPolicies:
+      enabled: true
+    k8s:
+      requireIPv4PodCIDR: true
+      {{- $gateway := lookup "gateway.networking.k8s.io/v1" "Gateway" "capi-hosted-control-plane-system" "controlplane" -}}
+      {{- if not $gateway -}}
+        {{- fail "Hosted control plane Gateway 'controlplane' in namespace 'capi-hosted-control-plane-system' not found" -}}
+      {{- else }}
+    k8sServiceHost: {{ printf "%s.%s.%s" .Release.Name .Release.Namespace (replace "*." "" (index $gateway.spec.listeners 0).hostname) }}
+      {{- end }}
+    k8sServicePort: 443
+    ipv4NativeRoutingCIDR: 10.0.0.0/16 # default net, see hosted control plane controller
+    {{- end }}
 
     rollOutCiliumPods: true
     encryption:
       enabled: false
       nodeEncryption: false
+    envoy:
+      prometheus:
+        enabled: true
     hubble:
       metrics:
         enabled:
@@ -57,6 +83,12 @@ spec:
       ui:
         rollOutPods: true
         enabled: true
+      export:
+        static:
+          enabled: true
+          filePath: stdout
+          allowList:
+            - '{"verdict":["DROPPED","ERROR"]}'
     operator:
       rollOutPods: true
       prometheus: