diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index 311ce538..bbd68b00 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -31,14 +31,15 @@ import ( ) const ( - ProvisionerName = "eks" - defaultLaunchConfigurationRetention = 2 - OverrideDefaultLabelsAnnotation = "instancemgr.keikoproj.io/default-labels" - IRSAEnabledAnnotation = "instancemgr.keikoproj.io/irsa-enabled" - OsFamilyAnnotation = "instancemgr.keikoproj.io/os-family" - ClusterAutoscalerEnabledAnnotation = "instancemgr.keikoproj.io/cluster-autoscaler-enabled" - CustomNetworkingEnabledAnnotation = "instancemgr.keikoproj.io/custom-networking-enabled" - CustomNetworkingHostPodsAnnotation = "instancemgr.keikoproj.io/custom-networking-host-pods" + ProvisionerName = "eks" + defaultLaunchConfigurationRetention = 2 + OverrideDefaultLabelsAnnotation = "instancemgr.keikoproj.io/default-labels" + IRSAEnabledAnnotation = "instancemgr.keikoproj.io/irsa-enabled" + OsFamilyAnnotation = "instancemgr.keikoproj.io/os-family" + ClusterAutoscalerEnabledAnnotation = "instancemgr.keikoproj.io/cluster-autoscaler-enabled" + CustomNetworkingEnabledAnnotation = "instancemgr.keikoproj.io/custom-networking-enabled" + CustomNetworkingHostPodsAnnotation = "instancemgr.keikoproj.io/custom-networking-host-pods" + CustomNetworkingPrefixAssignmentEnabledAnnotation = "instancemgr.keikoproj.io/custom-networking-prefix-assignment-enabled" OsFamilyWindows = "windows" OsFamilyBottleRocket = "bottlerocket" diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go index 15c87d5b..ea524e78 100644 --- a/controllers/provisioners/eks/helpers.go +++ b/controllers/provisioners/eks/helpers.go @@ -502,8 +502,17 @@ func (ctx *EksInstanceGroupContext) GetComputedBootstrapOptions() *v1alpha1.Boot } instanceTypeNetworkInfo := awsprovider.GetInstanceTypeNetworkInfo(state.GetInstanceTypeInfo(), configuration.InstanceType) - maxPods := (*instanceTypeNetworkInfo.MaximumNetworkInterfaces-1)* - (*instanceTypeNetworkInfo.Ipv4AddressesPerInterface-1) + hostNetworkPods + var prefixAssignmentEnabled = instanceGroup.GetAnnotations()[CustomNetworkingPrefixAssignmentEnabledAnnotation] == "true" + var maxPods int64 = 0 + + var enis = aws.Int64Value(instanceTypeNetworkInfo.MaximumNetworkInterfaces)-1 //Primary interface is not used for pod networking when custom networking is enabled + var ipsPerInterface int64 = 1 + if prefixAssignmentEnabled { + ipsPerInterface = 16 //Number of ips in a /28 block + } + + maxPods = enis * ((aws.Int64Value(instanceTypeNetworkInfo.Ipv4AddressesPerInterface)-1) * ipsPerInterface)+ hostNetworkPods + if configuration.BootstrapOptions == nil { return &v1alpha1.BootstrapOptions{ MaxPods: maxPods, diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go index 5f16ed0e..ec2d826e 100644 --- a/controllers/provisioners/eks/helpers_test.go +++ b/controllers/provisioners/eks/helpers_test.go @@ -314,6 +314,16 @@ func TestCustomNetworkingMaxPods(t *testing.T) { bootstrapOptions: nil, expectedMaxPods: "--max-pods=20", }, + { + annotations: map[string]string{ + ClusterAutoscalerEnabledAnnotation: "true", + CustomNetworkingPrefixAssignmentEnabledAnnotation: "true", + CustomNetworkingHostPodsAnnotation: "2", + CustomNetworkingEnabledAnnotation: "true", + }, + bootstrapOptions: nil, + expectedMaxPods: "--max-pods=290", + }, { annotations: map[string]string{ ClusterAutoscalerEnabledAnnotation: "true", diff --git a/docs/EKS.md b/docs/EKS.md index d69dad4d..9fac3ce2 100644 --- a/docs/EKS.md +++ b/docs/EKS.md @@ -655,4 +655,5 @@ The following operators are supported: |instancemgr.keikoproj.io/os-family|InstanceGroup|either "windows", "bottlerocket", or "amazonlinux2" (default)|this is required if you are running a windows or bottlerocket based AMI, by default the controller will try to bootstrap an amazonlinux2 AMI| |instancemgr.keikoproj.io/default-labels|InstanceGroup|comma-seprarated key-value string e.g. "label1=value1,label2=value2"|allows overriding the default node labels added by the controller, by default the role label is added depending on the cluster version| |instancemgr.keikoproj.io/custom-networking-enabled|InstanceGroup|"true"|setting this annotation to true will automatically calculate the correct setting for max pods and pass it to the kubelet| +|instancemgr.keikoproj.io/custom-networking-prefix-assignment-enabled|InstanceGroup|"true"|setting this annotation to true will change the max pod calculations to reflect the pod density supported by vpc prefix assignment. Supported in AWS VPC CNI versions 1.9.0 and above - see [AWS VPC CNI 1.9.0](https://github.com/aws/amazon-vpc-cni-k8s/releases/tag/v1.9.0) for more information.| |instancemgr.keikoproj.io/custom-networking-host-pods|InstanceGroup|"2"|setting this annotation increases the number of max pods on nodes with custom networking, due to the fact that hostNetwork pods do not use an additional IP address |