feat: add support for specifying container runtime (#323)

* feat: add support for specifying container runtime

Signed-off-by: Jonah Back <[email protected]>

* document bootstrap options

Signed-off-by: Jonah Back <[email protected]>

* Validate container runtime being used

Signed-off-by: Jonah Back <[email protected]>

Author: backjo

api/v1alpha1/instancegroup_types.go

@@ -78,11 +78,15 @@ const (
 	DedicatedPlacementTenancyType = "dedicated"
 )
 
+type ContainerRuntime string
 type ScalingConfigurationType string
 
 const (
 	LaunchConfiguration ScalingConfigurationType = "LaunchConfiguration"
 	LaunchTemplate      ScalingConfigurationType = "LaunchTemplate"
+
+	DockerRuntime     ContainerRuntime = "dockerd"
+	ContainerDRuntime ContainerRuntime = "containerd"
 )
 
 var (
@@ -105,6 +109,7 @@ var (
 	}
 	DefaultCRDStrategyMaxRetries = 3
 
+	AllowedContainerRuntimes            = []ContainerRuntime{ContainerDRuntime, DockerRuntime}
 	AllowedFileSystemTypes              = []string{FileSystemTypeXFS, FileSystemTypeEXT4}
 	AllowedMixedPolicyStrategies        = []string{LaunchTemplateStrategyCapacityOptimized, LaunchTemplateStrategyLowestPrice}
 	AllowedInstancePools                = []string{SubFamilyFlexibleInstancePool}
@@ -187,7 +192,8 @@ type EKSManagedSpec struct {
 }
 
 type BootstrapOptions struct {
-	MaxPods int64 `json:"maxPods,omitempty"`
+	MaxPods          int64            `json:"maxPods,omitempty"`
+	ContainerRuntime ContainerRuntime `json:"containerRuntime,omitempty"`
 }
 
 type WarmPoolSpec struct {
@@ -474,6 +480,15 @@ func (s *EKSSpec) HasWarmPool() bool {
 	return false
 }
 
+func contains(s []ContainerRuntime, e ContainerRuntime) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *EKSConfiguration) Validate() error {
 	if common.StringEmpty(c.EksClusterName) {
 		return errors.Errorf("validation failed, 'clusterName' is a required parameter")
@@ -506,6 +521,13 @@ func (c *EKSConfiguration) Validate() error {
 		c.SuspendedProcesses = processes
 	}
 
+
+	if c.BootstrapOptions != nil {
+		if c.BootstrapOptions.ContainerRuntime != "" && !contains(AllowedContainerRuntimes, c.BootstrapOptions.ContainerRuntime) {
+			return errors.Errorf("validation failed, 'bootstrapOptions.containerRuntime' must be one of %+v", AllowedContainerRuntimes)
+		}
+	}
+
 	hooks := []LifecycleHookSpec{}
 	for _, h := range c.LifecycleHooks {
 		if h.HeartbeatTimeout == 0 {

api/v1alpha1/instancegroup_types_test.go

@@ -109,6 +109,26 @@ func TestInstanceGroupSpecValidate(t *testing.T) {
 			},
 			want: "validation failed, 'LicenseSpecifications[0]' must be a valid IAM role ARN",
 		},
+		{
+			name: "eks with invalid container runtime",
+			args: args{
+				instancegroup: MockInstanceGroup("eks", "rollingUpdate", &EKSSpec{
+					MaxSize: 1,
+					MinSize: 1,
+					Type:    "LaunchTemplate",
+					EKSConfiguration: &EKSConfiguration{
+						BootstrapOptions: &BootstrapOptions{ContainerRuntime: "foo"},
+						EksClusterName:        "my-eks-cluster",
+						NodeSecurityGroups:    []string{"sg-123456789"},
+						Image:                 "ami-12345",
+						InstanceType:          "m5.large",
+						KeyPairName:           "thisShouldBeOptional",
+						Subnets:               []string{"subnet-1111111", "subnet-222222"},
+					},
+				}, nil, nil),
+			},
+			want: "validation failed, 'bootstrapOptions.containerRuntime' must be one of [containerd dockerd]",
+		},
 		{
 			name: "eks with valid Placement",
 			args: args{
@@ -283,7 +303,7 @@ func TestInstanceGroupSpecValidate(t *testing.T) {
 						},
 						Volumes: []NodeVolume{
 							{
-								Type:       "gp2",
+								Type: "gp2",
 								Iops: 1000,
 							},
 						},

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -75,6 +75,8 @@ spec:
                         type: string
                       bootstrapOptions:
                         properties:
+                          containerRuntime:
+                            type: string
                           maxPods:
                             format: int64
                             type: integer

controllers/providers/aws/eks_test.go

@@ -8,12 +8,12 @@ import (
 
 func TestClusterDns(t *testing.T) {
 	var (
-		g       = gomega.NewGomegaWithT(t)
+		g = gomega.NewGomegaWithT(t)
 	)
 
 	awsWorker := AwsWorker{}
 	cidr := "172.16.0.0/12"
 	ip := awsWorker.GetDNSClusterIP(&eks.Cluster{KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{ServiceIpv4Cidr: &cidr}})
 	g.Expect(ip).To(gomega.Equal("172.16.0.10"))
 
-}
+}

controllers/provisioners/eks/helpers.go

@@ -539,6 +539,10 @@ func (ctx *EksInstanceGroupContext) GetBootstrapArgs() string {
 		if bootstrapOptions != nil && bootstrapOptions.MaxPods > 0 {
 			sb.WriteString("--use-max-pods false ")
 		}
+
+		if bootstrapOptions != nil && bootstrapOptions.ContainerRuntime != "" {
+			sb.WriteString(fmt.Sprintf("--container-runtime %v ", bootstrapOptions.ContainerRuntime))
+		}
 		if state.Cluster != nil {
 			sb.WriteString(fmt.Sprintf("--b64-cluster-ca %v ", aws.StringValue(state.Cluster.CertificateAuthority.Data)))
 			sb.WriteString(fmt.Sprintf("--apiserver-endpoint %v ", aws.StringValue(state.Cluster.Endpoint)))

controllers/provisioners/eks/helpers_test.go

@@ -96,7 +96,8 @@ func TestGetBasicUserDataAmazonLinux2(t *testing.T) {
 	ctx := MockContext(ig, k, w)
 
 	configuration.BootstrapOptions = &v1alpha1.BootstrapOptions{
-		MaxPods: 4,
+		MaxPods:          4,
+		ContainerRuntime: "containerd",
 	}
 	configuration.Labels = map[string]string{
 		"foo": "bar",
@@ -159,7 +160,7 @@ if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
 	fi
 fi
 set -o xtrace
-/etc/eks/bootstrap.sh foo --use-max-pods false --b64-cluster-ca dGVzdA== --apiserver-endpoint foo.amazonaws.com --dns-cluster-ip 172.20.0.10 --kubelet-extra-args '--node-labels=foo=bar,instancemgr.keikoproj.io/image=ami-123456789012,node.kubernetes.io/role=instance-group-1 --register-with-taints=foo=bar:NoSchedule --eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2 --max-pods=4'
+/etc/eks/bootstrap.sh foo --use-max-pods false --container-runtime containerd --b64-cluster-ca dGVzdA== --apiserver-endpoint foo.amazonaws.com --dns-cluster-ip 172.20.0.10 --kubelet-extra-args '--node-labels=foo=bar,instancemgr.keikoproj.io/image=ami-123456789012,node.kubernetes.io/role=instance-group-1 --register-with-taints=foo=bar:NoSchedule --eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2 --max-pods=4'
 set +o xtrace
 bar`
 	userData := ctx.GetBasicUserData("foo", args, kubeletArgs, userDataPayload, mounts)

docs/EKS.md

@@ -58,6 +58,11 @@ spec:
       # All (will suspend all above processes)
       suspendProcesses: <[]string> : must match scaling process names to suspend
 
+      bootstrapOptions:
+        containerRuntime: <string> : one of "dockerd" or "containerd". Specifies which container runtime to use. Currently only available on Amazon Linux 2
+        maxPods: <int> : maximum number of pods that can be run per-node in this IG.
+                 
+
       bootstrapArguments: <string> : additional flags to pass to boostrap.sh script
       spotPrice: <string> : must be a decimal number represnting a minimal spot price