diff --git a/integration/tests/crud/creategetdelete_test.go b/integration/tests/crud/creategetdelete_test.go index ca53b19971..22fa5dba79 100644 --- a/integration/tests/crud/creategetdelete_test.go +++ b/integration/tests/crud/creategetdelete_test.go @@ -69,6 +69,8 @@ const ( deleteNg = "ng-delete" taintsNg1 = "ng-taints-1" taintsNg2 = "ng-taints-2" + maxPodsMNG1 = "mng-max-pods-1" + maxPodsMNG2 = "mng-max-pods-2" scaleSingleNg = "ng-scale-single" scaleMultipleNg = "ng-scale-multiple" @@ -872,6 +874,8 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { clientset := makeClientset() nodeListN1 := tests.ListNodes(clientset, taintsNg1) nodeListN2 := tests.ListNodes(clientset, taintsNg2) + mngNodeListN1 := tests.ListNodes(clientset, maxPodsMNG1) + mngNodeListN2 := tests.ListNodes(clientset, maxPodsMNG2) tests.AssertNodeTaints(nodeListN1, []corev1.Taint{ { @@ -896,12 +900,22 @@ var _ = Describe("(Integration) Create, Get, Scale & Delete", func() { }, }) - By("asserting that maxPods is set correctly") + By("asserting that maxPods is set correctly for AL2 nodegroup") expectedMaxPods := 123 for _, node := range nodeListN1.Items { maxPods, _ := node.Status.Allocatable.Pods().AsInt64() Expect(maxPods).To(Equal(int64(expectedMaxPods))) } + + By("asserting that maxPods is set correctly for AL2023 nodegroups") + for _, node := range mngNodeListN1.Items { + maxPods, _ := node.Status.Allocatable.Pods().AsInt64() + Expect(maxPods).To(Equal(int64(expectedMaxPods))) + } + for _, node := range mngNodeListN2.Items { + maxPods, _ := node.Status.Allocatable.Pods().AsInt64() + Expect(maxPods).To(Equal(int64(expectedMaxPods))) + } }) It("should be able to create a new GPU nodegroup", func() { diff --git a/integration/tests/crud/testdata/taints-max-pods.yaml b/integration/tests/crud/testdata/taints-max-pods.yaml index 7881cd83ac..4fc0efbf66 100644 --- a/integration/tests/crud/testdata/taints-max-pods.yaml +++ b/integration/tests/crud/testdata/taints-max-pods.yaml @@ -6,16 +6,30 @@ metadata: name: nodeGroups: -- name: ng-taints-1 - taints: - key1: val1:NoSchedule - key2: :NoExecute - maxPodsPerNode: 123 -- name: ng-taints-2 - volumeSize: 35 - taints: - - key: key1 - value: value1 - effect: NoSchedule - - key: key2 - effect: NoExecute + - name: ng-taints-1 + taints: + key1: val1:NoSchedule + key2: :NoExecute + maxPodsPerNode: 123 + - name: ng-taints-2 + volumeSize: 35 + taints: + - key: key1 + value: value1 + effect: NoSchedule + - key: key2 + effect: NoExecute + +managedNodeGroups: + - name: mng-max-pods-1 + desiredCapacity: 1 + maxPodsPerNode: 123 + - name: mng-max-pods-2 + desiredCapacity: 1 + overrideBootstrapCommand: | + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + config: + maxPods: 123 diff --git a/pkg/apis/eksctl.io/v1alpha5/validation.go b/pkg/apis/eksctl.io/v1alpha5/validation.go index 1a50f128bc..e0297bbcfa 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation.go @@ -759,19 +759,6 @@ func validateNodeGroupBase(np NodePool, path string, controlPlaneOnOutposts bool } } - if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 { - fieldNotSupported := func(field string) error { - return &unsupportedFieldError{ - ng: ng, - path: path, - field: field, - } - } - if ng.OverrideBootstrapCommand != nil { - return fieldNotSupported("overrideBootstrapCommand") - } - } - if ng.CapacityReservation != nil { if ng.CapacityReservation.CapacityReservationPreference != nil { if ng.CapacityReservation.CapacityReservationTarget != nil { @@ -1276,10 +1263,6 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error { } } - if ng.AMIFamily == NodeImageFamilyAmazonLinux2023 && ng.MaxPodsPerNode > 0 { - return fmt.Errorf("eksctl does not support configuring maxPodsPerNode EKS-managed nodes based on %s", NodeImageFamilyAmazonLinux2023) - } - if ng.AMIFamily == NodeImageFamilyBottlerocket { fieldNotSupported := func(field string) error { return &unsupportedFieldError{ @@ -1362,9 +1345,6 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error { if ng.OverrideBootstrapCommand == nil && ng.AMIFamily != NodeImageFamilyAmazonLinux2023 { return fmt.Errorf("%[1]s.overrideBootstrapCommand is required when using a custom AMI based on %s (%[1]s.ami)", path, ng.AMIFamily) } - if ng.OverrideBootstrapCommand != nil && ng.AMIFamily == NodeImageFamilyAmazonLinux2023 { - return fmt.Errorf("%[1]s.overrideBootstrapCommand is not supported when using a custom AMI based on %s (%[1]s.ami)", path, ng.AMIFamily) - } notSupportedWithCustomAMIErr := func(field string) error { return fmt.Errorf("%s.%s is not supported when using a custom AMI (%s.ami)", path, field, path) } @@ -1378,7 +1358,7 @@ func ValidateManagedNodeGroup(index int, ng *ManagedNodeGroup) error { return notSupportedWithCustomAMIErr("releaseVersion") } - case ng.OverrideBootstrapCommand != nil: + case ng.OverrideBootstrapCommand != nil && ng.AMIFamily != NodeImageFamilyAmazonLinux2023: return fmt.Errorf("%s.overrideBootstrapCommand can only be set when a custom AMI (%s.ami) is specified", path, path) } diff --git a/pkg/apis/eksctl.io/v1alpha5/validation_test.go b/pkg/apis/eksctl.io/v1alpha5/validation_test.go index 8f917766c0..65376de648 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation_test.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation_test.go @@ -2178,33 +2178,6 @@ var _ = Describe("ClusterConfig validation", func() { }) }) - Describe("AmazonLinux2023 node groups", func() { - It("returns an error when setting maxPodsPerNode for managed nodegroups", func() { - ng := api.NewManagedNodeGroup() - ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 - ng.MaxPodsPerNode = 5 - err := api.ValidateManagedNodeGroup(0, ng) - Expect(err).To(MatchError(ContainSubstring("eksctl does not support configuring maxPodsPerNode EKS-managed nodes"))) - }) - It("returns an error when setting overrideBootstrapCommand for self-managed nodegroups", func() { - cfg := api.NewClusterConfig() - ng := cfg.NewNodeGroup() - ng.Name = "node-group" - ng.AMI = "ami-1234" - ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 - ng.OverrideBootstrapCommand = aws.String("echo 'rubarb'") - Expect(api.ValidateNodeGroup(0, ng, cfg)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) - }) - It("returns an error when setting overrideBootstrapCommand for EKS-managed nodegroups", func() { - ng := api.NewManagedNodeGroup() - ng.Name = "node-group" - ng.AMI = "ami-1234" - ng.AMIFamily = api.NodeImageFamilyAmazonLinux2023 - ng.OverrideBootstrapCommand = aws.String("echo 'rubarb'") - Expect(api.ValidateManagedNodeGroup(0, ng)).To(MatchError(ContainSubstring(fmt.Sprintf("overrideBootstrapCommand is not supported for %s nodegroups", api.NodeImageFamilyAmazonLinux2023)))) - }) - }) - Describe("Windows node groups", func() { It("returns an error with unsupported fields", func() { doc := api.InlineDocument{ diff --git a/pkg/nodebootstrap/al2023.go b/pkg/nodebootstrap/al2023.go index 2fcc28e8b2..8d95c838e6 100644 --- a/pkg/nodebootstrap/al2023.go +++ b/pkg/nodebootstrap/al2023.go @@ -5,7 +5,8 @@ import ( "encoding/base64" "encoding/json" "fmt" - "strconv" + + "sigs.k8s.io/yaml" nodeadmapi "github.com/awslabs/amazon-eks-ami/nodeadm/api" nodeadm "github.com/awslabs/amazon-eks-ami/nodeadm/api/v1alpha1" @@ -22,8 +23,9 @@ type AL2023 struct { nodePool api.NodePool clusterDNS string - scripts []string - cloudboot []string + scripts []string + cloudboot []string + nodeConfigs []*nodeadm.NodeConfig UserDataMimeBoundary string } @@ -54,27 +56,55 @@ func newAL2023Bootstrapper(cfg *api.ClusterConfig, np api.NodePool, clusterDNS s } func (m *AL2023) UserData() (string, error) { - nodeConfig, err := m.createNodeConfig() + ng := m.nodePool.BaseNodeGroup() + + minimalNodeConfig, err := m.createMinimalNodeConfig() if err != nil { - return "", fmt.Errorf("generating node config: %w", err) + return "", fmt.Errorf("generating minimal node config: %w", err) + } + if minimalNodeConfig != nil { + m.nodeConfigs = append(m.nodeConfigs, minimalNodeConfig) + } + + if ng.MaxPodsPerNode > 0 { + nodeConfig := &nodeadm.NodeConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), + }, + } + kubeletConfig, err := ToKubeletConfig(api.InlineDocument{"maxPods": ng.MaxPodsPerNode}) + if err != nil { + return "", err + } + nodeConfig.Spec.Kubelet.Config = kubeletConfig + m.nodeConfigs = append(m.nodeConfigs, nodeConfig) } for _, command := range m.nodePool.BaseNodeGroup().PreBootstrapCommands { m.scripts = append(m.scripts, "#!/bin/bash\n"+command) } - if len(m.scripts) == 0 && len(m.cloudboot) == 0 && nodeConfig == nil { + if ng.OverrideBootstrapCommand != nil { + nodeConfig, err := stringToNodeConfig(*ng.OverrideBootstrapCommand) + if err != nil { + return "", err + } + m.nodeConfigs = append(m.nodeConfigs, nodeConfig) + } + + if len(m.scripts) == 0 && len(m.cloudboot) == 0 && len(m.nodeConfigs) == 0 { return "", nil } var buf bytes.Buffer - if err := createMimeMessage(&buf, m.scripts, m.cloudboot, nodeConfig, m.UserDataMimeBoundary); err != nil { + if err := createMimeMessage(&buf, m.scripts, m.cloudboot, m.nodeConfigs, m.UserDataMimeBoundary); err != nil { return "", err } return base64.StdEncoding.EncodeToString(buf.Bytes()), nil } -func (m *AL2023) createNodeConfig() (*nodeadm.NodeConfig, error) { +func (m *AL2023) createMinimalNodeConfig() (*nodeadm.NodeConfig, error) { kubeletConfig := api.InlineDocument{} switch nodeGroup := m.nodePool.(type) { case *api.ManagedNodeGroup: @@ -89,9 +119,6 @@ func (m *AL2023) createNodeConfig() (*nodeadm.NodeConfig, error) { kubeletConfig["clusterDNS"] = []string{m.clusterDNS} ng := m.nodePool.BaseNodeGroup() - if ng.MaxPodsPerNode > 0 { - kubeletConfig["maxPods"] = strconv.Itoa(ng.MaxPodsPerNode) - } nodeKubeletConfig, err := ToKubeletConfig(kubeletConfig) if err != nil { return nil, err @@ -135,3 +162,12 @@ func ToKubeletConfig(kubeletExtraConfig api.InlineDocument) (map[string]runtime. } return kubeletConfig, nil } + +func stringToNodeConfig(overrideBootstrapCommand string) (*nodeadm.NodeConfig, error) { + var config nodeadm.NodeConfig + err := yaml.Unmarshal([]byte(overrideBootstrapCommand), &config) + if err != nil { + return nil, fmt.Errorf("unmarshalling \"overrideBootstrapCommand\" into \"nodeadm.NodeConfig\": %w", err) + } + return &config, nil +} diff --git a/pkg/nodebootstrap/al2023_test.go b/pkg/nodebootstrap/al2023_test.go index 56568adaaa..493f266d99 100644 --- a/pkg/nodebootstrap/al2023_test.go +++ b/pkg/nodebootstrap/al2023_test.go @@ -3,6 +3,7 @@ package nodebootstrap_test import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "io" "mime/multipart" @@ -106,9 +107,9 @@ var _ = DescribeTable("Managed AL2023", func(e al2023Entry) { }), ) -type al2023KubeletEntry struct { - updateNodeGroup func(*api.NodeGroup) - expectedNodeConfig nodeadm.NodeConfig +type al2023OverrideNodeConfigEntry struct { + updateNodeGroup func(*api.NodeGroup) + expectedNodeConfigs []nodeadm.NodeConfig } func mustToKubeletConfig(kubeletExtraConfig api.InlineDocument) map[string]runtime.RawExtension { @@ -119,7 +120,7 @@ func mustToKubeletConfig(kubeletExtraConfig api.InlineDocument) map[string]runti return kubeletConfig } -var _ = DescribeTable("AL2023 kubeletExtraConfig", func(e al2023KubeletEntry) { +var _ = DescribeTable("AL2023 override node config", func(e al2023OverrideNodeConfigEntry) { cfg, dns := makeDefaultClusterSettings() ng := api.NewNodeGroup() if e.updateNodeGroup != nil { @@ -134,7 +135,7 @@ var _ = DescribeTable("AL2023 kubeletExtraConfig", func(e al2023KubeletEntry) { decoded, err := base64.StdEncoding.DecodeString(userData) Expect(err).NotTo(HaveOccurred()) reader := multipart.NewReader(bytes.NewReader(decoded), al2023BS.UserDataMimeBoundary) - foundNodeConfig := false + nodeConfigCounter := 0 for { part, err := reader.NextPart() if errors.Is(err, io.EOF) { @@ -144,17 +145,18 @@ var _ = DescribeTable("AL2023 kubeletExtraConfig", func(e al2023KubeletEntry) { if part.Header.Get("Content-Type") != "application/node.eks.aws" { continue } - foundNodeConfig = true var nodeConfigBuf bytes.Buffer _, err = io.Copy(&nodeConfigBuf, part) Expect(err).NotTo(HaveOccurred()) var nodeConfig nodeadm.NodeConfig Expect(yaml.Unmarshal(nodeConfigBuf.Bytes(), &nodeConfig)).To(Succeed()) - Expect(nodeConfig).To(Equal(e.expectedNodeConfig)) + Expect(nodeConfigCounter).To(BeNumerically("<", len(e.expectedNodeConfigs))) + Expect(nodeConfig).To(Equal(e.expectedNodeConfigs[nodeConfigCounter])) + nodeConfigCounter++ } - Expect(foundNodeConfig).To(BeTrue(), "expected to find NodeConfig in user data") + Expect(nodeConfigCounter).To(BeNumerically("==", len(e.expectedNodeConfigs))) }, - Entry("nodegroup with maxPods and taints", al2023KubeletEntry{ + Entry("nodegroup with maxPods and taints", al2023OverrideNodeConfigEntry{ updateNodeGroup: func(ng *api.NodeGroup) { ng.MaxPodsPerNode = 11 ng.Labels = map[string]string{"alpha.eksctl.io/nodegroup-name": "al2023-mng-test"} @@ -167,33 +169,47 @@ var _ = DescribeTable("AL2023 kubeletExtraConfig", func(e al2023KubeletEntry) { } }, - expectedNodeConfig: nodeadm.NodeConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: nodeadmapi.KindNodeConfig, - APIVersion: nodeadm.GroupVersion.String(), + expectedNodeConfigs: []nodeadm.NodeConfig{ + { + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), + }, + Spec: nodeadm.NodeConfigSpec{ + Cluster: nodeadm.ClusterDetails{ + APIServerEndpoint: "https://test.xxx.us-west-2.eks.amazonaws.com", + CertificateAuthority: []byte("test CA"), + CIDR: "10.100.0.0/16", + Name: "al2023-test", + }, + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "clusterDNS": []string{"10.100.0.10"}, + }), + Flags: []string{ + "--node-labels=alpha.eksctl.io/nodegroup-name=al2023-mng-test", + "--register-with-taints=special=true:NoSchedule", + }, + }, + }, }, - Spec: nodeadm.NodeConfigSpec{ - Cluster: nodeadm.ClusterDetails{ - APIServerEndpoint: "https://test.xxx.us-west-2.eks.amazonaws.com", - CertificateAuthority: []byte("test CA"), - CIDR: "10.100.0.0/16", - Name: "al2023-test", + { + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), }, - Kubelet: nodeadm.KubeletOptions{ - Config: mustToKubeletConfig(map[string]interface{}{ - "clusterDNS": []string{"10.100.0.10"}, - "maxPods": "11", - }), - Flags: []string{ - "--node-labels=alpha.eksctl.io/nodegroup-name=al2023-mng-test", - "--register-with-taints=special=true:NoSchedule", + Spec: nodeadm.NodeConfigSpec{ + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "maxPods": 11, + }), }, }, }, }, }), - Entry("nodegroup with maxPods, taints and kubeletExtraConfig", al2023KubeletEntry{ + Entry("nodegroup with maxPods, taints and kubeletExtraConfig", al2023OverrideNodeConfigEntry{ updateNodeGroup: func(ng *api.NodeGroup) { ng.MaxPodsPerNode = 11 ng.Labels = map[string]string{"alpha.eksctl.io/nodegroup-name": "al2023-mng-test"} @@ -213,31 +229,121 @@ var _ = DescribeTable("AL2023 kubeletExtraConfig", func(e al2023KubeletEntry) { } }, - expectedNodeConfig: nodeadm.NodeConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: nodeadmapi.KindNodeConfig, - APIVersion: nodeadm.GroupVersion.String(), + expectedNodeConfigs: []nodeadm.NodeConfig{ + { + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), + }, + Spec: nodeadm.NodeConfigSpec{ + Cluster: nodeadm.ClusterDetails{ + APIServerEndpoint: "https://test.xxx.us-west-2.eks.amazonaws.com", + CertificateAuthority: []byte("test CA"), + CIDR: "10.100.0.0/16", + Name: "al2023-test", + }, + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "clusterDNS": []string{"10.100.0.10"}, + "shutdownGracePeriod": "5m", + "kubeReserved": map[string]interface{}{ + "cpu": "500m", + "memory": "250Mi", + }, + }), + Flags: []string{ + "--node-labels=alpha.eksctl.io/nodegroup-name=al2023-mng-test", + "--register-with-taints=special=true:NoSchedule", + }, + }, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), + }, + Spec: nodeadm.NodeConfigSpec{ + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "maxPods": 11, + }), + }, + }, + }, + }, + }), + + Entry("nodegroup with overrideBootstrapCommand", al2023OverrideNodeConfigEntry{ + updateNodeGroup: func(ng *api.NodeGroup) { + nodeConfig := nodeadm.NodeConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), + }, + Spec: nodeadm.NodeConfigSpec{ + Instance: nodeadm.InstanceOptions{ + LocalStorage: nodeadm.LocalStorageOptions{ + Strategy: nodeadm.LocalStorageRAID0, + }, + }, + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "shutdownGracePeriod": "5m", + "featureGates": map[string]interface{}{ + "DisableKubeletCloudCredentialProviders": true, + }, + }), + }, + }, + } + jsonNodeConfig, err := json.Marshal(nodeConfig) + Expect(err).NotTo(HaveOccurred()) + ng.OverrideBootstrapCommand = aws.String(string(jsonNodeConfig)) + ng.Labels = map[string]string{"alpha.eksctl.io/nodegroup-name": "al2023-mng-test"} + + }, + expectedNodeConfigs: []nodeadm.NodeConfig{ + { + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), + }, + Spec: nodeadm.NodeConfigSpec{ + Cluster: nodeadm.ClusterDetails{ + APIServerEndpoint: "https://test.xxx.us-west-2.eks.amazonaws.com", + CertificateAuthority: []byte("test CA"), + CIDR: "10.100.0.0/16", + Name: "al2023-test", + }, + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "clusterDNS": []string{"10.100.0.10"}, + }), + Flags: []string{ + "--node-labels=alpha.eksctl.io/nodegroup-name=al2023-mng-test", + }, + }, + }, }, - Spec: nodeadm.NodeConfigSpec{ - Cluster: nodeadm.ClusterDetails{ - APIServerEndpoint: "https://test.xxx.us-west-2.eks.amazonaws.com", - CertificateAuthority: []byte("test CA"), - CIDR: "10.100.0.0/16", - Name: "al2023-test", + { + TypeMeta: metav1.TypeMeta{ + Kind: nodeadmapi.KindNodeConfig, + APIVersion: nodeadm.GroupVersion.String(), }, - Kubelet: nodeadm.KubeletOptions{ - Config: mustToKubeletConfig(map[string]interface{}{ - "clusterDNS": []string{"10.100.0.10"}, - "maxPods": "11", - "shutdownGracePeriod": "5m", - "kubeReserved": map[string]interface{}{ - "cpu": "500m", - "memory": "250Mi", + Spec: nodeadm.NodeConfigSpec{ + Instance: nodeadm.InstanceOptions{ + LocalStorage: nodeadm.LocalStorageOptions{ + Strategy: nodeadm.LocalStorageRAID0, }, - }), - Flags: []string{ - "--node-labels=alpha.eksctl.io/nodegroup-name=al2023-mng-test", - "--register-with-taints=special=true:NoSchedule", + }, + Kubelet: nodeadm.KubeletOptions{ + Config: mustToKubeletConfig(map[string]interface{}{ + "shutdownGracePeriod": "5m", + "featureGates": map[string]interface{}{ + "DisableKubeletCloudCredentialProviders": true, + }, + }), }, }, }, diff --git a/pkg/nodebootstrap/managed_al2.go b/pkg/nodebootstrap/managed_al2.go index dd635791d3..f868116377 100644 --- a/pkg/nodebootstrap/managed_al2.go +++ b/pkg/nodebootstrap/managed_al2.go @@ -102,7 +102,7 @@ set -ex return script } -func createMimeMessage(writer io.Writer, scripts, cloudboots []string, nodeConfig *nodeadm.NodeConfig, mimeBoundary string) error { +func createMimeMessage(writer io.Writer, scripts, cloudboots []string, nodeConfigs []*nodeadm.NodeConfig, mimeBoundary string) error { mw := multipart.NewWriter(writer) if mimeBoundary != "" { if err := mw.SetBoundary(mimeBoundary); err != nil { @@ -139,8 +139,8 @@ func createMimeMessage(writer io.Writer, scripts, cloudboots []string, nodeConfi } } - if nodeConfig != nil { - yamlData, err := yaml.Marshal(nodeConfig) + for _, nc := range nodeConfigs { + yamlData, err := yaml.Marshal(nc) if err != nil { return fmt.Errorf("error marshalling node configuration: %w", err) } diff --git a/userdocs/mkdocs.yml b/userdocs/mkdocs.yml index 44534c5ef8..dd60b40596 100644 --- a/userdocs/mkdocs.yml +++ b/userdocs/mkdocs.yml @@ -162,6 +162,7 @@ nav: - usage/nodegroups.md - usage/nodegroup-unmanaged.md - usage/nodegroup-managed.md + - usage/node-bootstrapping.md - usage/launch-template-support.md - usage/nodegroup-with-custom-subnet.md - usage/nodegroup-customize-dns.md diff --git a/userdocs/src/usage/node-bootstrapping.md b/userdocs/src/usage/node-bootstrapping.md new file mode 100644 index 0000000000..3fd063bb8c --- /dev/null +++ b/userdocs/src/usage/node-bootstrapping.md @@ -0,0 +1,57 @@ +# Node bootstrapping + +## AmazonLinux2023 + +AL2023 introduced a new node initialization process [nodeadm](https://awslabs.github.io/amazon-eks-ami/nodeadm/) that uses a YAML configuration schema, dropping the use of `/etc/eks/bootstrap.sh` script. + +### Default settings + +For self-managed nodes and EKS-managed nodes based on custom AMIs, `eksctl` creates a default, minimal, `NodeConfig` and automatically injects it into the nodegroups's launch template userdata. i.e. + +```yaml +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=// + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://XXXX.us-west-2.eks.amazonaws.com + certificateAuthority: XXXX + cidr: 10.100.0.0/16 + name: my-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + flags: + - --node-labels=alpha.eksctl.io/cluster-name=my-cluster,alpha.eksctl.io/nodegroup-name=my-nodegroup + - --register-with-taints=special=true:NoSchedule + +--//-- +``` + +For EKS-managed nodes based on native AMIs, the default `NodeConfig` is being added by EKS MNG under the hood, appended directly to the EC2's userdata. Thus, in this scenario, `eksctl` does not need to include it within the launch template. + +### Configuring the bootstrapping process + +To set advanced properties of `NodeConfig`, or simply override the default values, eksctl allows you to specify a custom `NodeConfig` via `nodeGroup.overrideBootstrapCommand` or `managedNodeGroup.overrideBootstrapCommand` e.g. + +```yaml +managedNodeGroups: + - name: mng-1 + amiFamily: AmazonLinux2023 + ami: ami-0253856dd7ab7dbc8 + overrideBootstrapCommand: | + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + instance: + localStorage: + strategy: RAID0 +``` + +This custom config will be prepended to the userdata by eksctl, and merged by `nodeadm` with the default config. Read more about `nodeadm`'s capability of merging multiple configuration objects [here](https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/examples/#merging-multiple-configuration-objects).