Add al2023 support (#491)

* added support for al2023

Signed-off-by: eramirez8 <[email protected]>

* trim quotes from override al os family

Signed-off-by: eramirez8 <[email protected]>

---------

Signed-off-by: eramirez8 <[email protected]>
Co-authored-by: eramirez8 <[email protected]>

Author: estela-ramirez

.gitignore

@@ -8,3 +8,4 @@ vendor/*
 .DS_Store
 *.idea
 .windsurfrules
+.qodo

api/instancemgr/v1alpha1/instancegroup_types.go

@@ -53,8 +53,9 @@ const (
 	ReconcileErr    ReconcileState = "Error"
 
 	// Userdata bootstrap stages
-	PreBootstrapStage  = "PreBootstrap"
-	PostBootstrapStage = "PostBootstrap"
+	PreBootstrapStage   = "PreBootstrap"
+	PostBootstrapStage  = "PostBootstrap"
+	NodeConfigYamlStage = "NodeConfigYaml"
 
 	LifecycleStateNormal      = "normal"
 	LifecycleStateSpot        = "spot"
@@ -840,6 +841,7 @@ func (c *EKSConfiguration) GetManagedPolicies() []string {
 func (c *EKSConfiguration) GetUserData() []UserDataStage {
 	return c.UserData
 }
+
 func (c *EKSConfiguration) SetManagedPolicies(policies []string) {
 	c.ManagedPolicies = policies
 }

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -253,6 +253,8 @@ spec:
                           - stage
                           type: object
                         type: array
+                      amazonLinuxOsFamily:
+                        type: string
                       volumes:
                         items:
                           properties:

controllers/instancegroup_controller.go

@@ -56,6 +56,7 @@ type InstanceGroupReconciler struct {
 	Metrics                     *common.MetricsCollector
 	DisableWinClusterInjection  bool
 	DefaultScalingConfiguration *v1alpha1.ScalingConfigurationType
+	AmazonLinuxOsFamily         string
 }
 
 type InstanceGroupAuthenticator struct {
@@ -143,6 +144,7 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque
 		ConfigRetention:            r.ConfigRetention,
 		Metrics:                    r.Metrics,
 		DisableWinClusterInjection: r.DisableWinClusterInjection,
+		AmazonLinuxOsFamily:        r.AmazonLinuxOsFamily,
 	}
 
 	var (

controllers/provisioners/config.go

@@ -19,14 +19,13 @@ import (
 	"fmt"
 	"strings"
 
-	"k8s.io/apimachinery/pkg/labels"
-
 	"github.com/ghodss/yaml"
 	"github.com/keikoproj/instance-manager/api/instancemgr/v1alpha1"
 	"github.com/keikoproj/instance-manager/controllers/common"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 

controllers/provisioners/eks/create.go

@@ -46,7 +46,6 @@ func (ctx *EksInstanceGroupContext) Create() error {
 		placement       = configuration.GetPlacement()
 		metadataOptions = configuration.GetMetadataOptions()
 	)
-
 	ctx.SetState(v1alpha1.ReconcileModifying)
 
 	// no need to create a role if one is already provided

controllers/provisioners/eks/eks.go

@@ -17,12 +17,12 @@ package eks
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/go-logr/logr"
-
 	"github.com/keikoproj/instance-manager/api/instancemgr/v1alpha1"
 	"github.com/keikoproj/instance-manager/controllers/common"
 	awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws"
@@ -41,9 +41,10 @@ const (
 	CustomNetworkingHostPodsAnnotation                = "instancemgr.keikoproj.io/custom-networking-host-pods"
 	CustomNetworkingPrefixAssignmentEnabledAnnotation = "instancemgr.keikoproj.io/custom-networking-prefix-assignment-enabled"
 
-	OsFamilyWindows      = "windows"
-	OsFamilyBottleRocket = "bottlerocket"
-	OsFamilyAmazonLinux2 = "amazonlinux2"
+	OsFamilyWindows         = "windows"
+	OsFamilyBottleRocket    = "bottlerocket"
+	OsFamilyAmazonLinux2    = "amazonlinux2"
+	OsFamilyAmazonLinux2023 = "amazonlinux2023"
 )
 
 var (
@@ -54,7 +55,7 @@ var (
 	InstanceMgrLifecycleLabel = "instancemgr.keikoproj.io/lifecycle"
 	InstanceMgrImageLabel     = "instancemgr.keikoproj.io/image"
 
-	AllowedOsFamilies      = []string{OsFamilyWindows, OsFamilyBottleRocket, OsFamilyAmazonLinux2}
+	AllowedOsFamilies      = []string{OsFamilyWindows, OsFamilyBottleRocket, OsFamilyAmazonLinux2, OsFamilyAmazonLinux2023}
 	DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
 	CNIManagedPolicy       = "AmazonEKS_CNI_Policy"
 	SupportedArchitectures = []string{"x86_64", "arm64"}
@@ -78,6 +79,7 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext {
 		ConfigRetention:            p.ConfigRetention,
 		Metrics:                    p.Metrics,
 		DisableWinClusterInjection: p.DisableWinClusterInjection,
+		AmazonLinuxOsFamily:        p.AmazonLinuxOsFamily,
 	}
 
 	ctx.SetState(v1alpha1.ReconcileInit)
@@ -99,11 +101,13 @@ type EksInstanceGroupContext struct {
 	ResourcePrefix             string
 	Metrics                    *common.MetricsCollector
 	DisableWinClusterInjection bool
+	AmazonLinuxOsFamily        string
 }
 
 type UserDataPayload struct {
-	PreBootstrap  []string
-	PostBootstrap []string
+	PreBootstrap   []string
+	PostBootstrap  []string
+	NodeConfigYaml string
 }
 
 type MountOpts struct {
@@ -125,6 +129,8 @@ type EKSUserData struct {
 	PostBootstrap    []string
 	MountOptions     []MountOpts
 	MaxPods          int64
+	ClusterIP        string
+	NodeConfigYaml   string
 }
 
 func (ctx *EksInstanceGroupContext) GetInstanceGroup() *v1alpha1.InstanceGroup {
@@ -139,14 +145,16 @@ func (ctx *EksInstanceGroupContext) GetOsFamily() string {
 		instanceGroup = ctx.GetInstanceGroup()
 		annotations   = instanceGroup.GetAnnotations()
 	)
+	overrideAmazonLinuxFamily := strings.Trim(ctx.AmazonLinuxOsFamily, "\" ")
 
 	if v, exists := annotations[OsFamilyAnnotation]; exists {
 		if common.ContainsEqualFold(AllowedOsFamilies, v) {
 			return annotations[OsFamilyAnnotation]
 		}
 		ctx.Log.Info("used unsupported annotation value '%v=%v', will default to 'amazonlinux2', allowed values: %+v", OsFamilyAnnotation, v, AllowedOsFamilies)
+	} else if common.ContainsEqualFold(AllowedOsFamilies, overrideAmazonLinuxFamily) {
+		return overrideAmazonLinuxFamily
 	}
-
 	return OsFamilyAmazonLinux2
 }
 

controllers/provisioners/eks/helpers.go

@@ -121,6 +121,8 @@ func (ctx *EksInstanceGroupContext) GetBasicUserData(clusterName, args string, k
 		nodeLabels       = ctx.GetComputedLabels()
 		nodeTaints       = configuration.GetTaints()
 		bootstrapOptions = ctx.GetComputedBootstrapOptions()
+		cluster          = state.GetCluster()
+		clusterIP        = ctx.AwsWorker.GetDNSClusterIP(cluster)
 	)
 	var maxPods int64
 
@@ -193,6 +195,59 @@ set -o xtrace
 /etc/eks/bootstrap.sh {{ .ClusterName }} {{ .Arguments }}
 set +o xtrace
 {{range $post := .PostBootstrap}}{{$post}}{{end}}`
+	case OsFamilyAmazonLinux2023:
+		UserDataTemplate = `MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+echo "IG manager using AL2023 amis"
+{{range $pre := .PreBootstrap}}{{$pre}}{{end}}
+{{- range .MountOptions}}
+mkfs.{{ .FileSystem | ToLower }} {{ .Device }}
+mkdir {{ .Mount }}
+mount {{ .Device }} {{ .Mount }}
+mount
+{{- if .Persistance}}
+echo "{{ .Device}}    {{ .Mount }}    {{ .FileSystem | ToLower }}    defaults    0    2" >> /etc/fstab
+{{- end}}
+{{- end}}
+if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
+	TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+	INSTANCE_ID=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
+	REGION=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
+	LIFECYCLE=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
+	if [[ $LIFECYCLE == *"Warmed"* ]]; then
+		rm /var/lib/cloud/instances/$INSTANCE_ID/sem/config_scripts_user
+		exit 0
+	fi
+fi
+--BOUNDARY
+Content-Type: application/node.eks.aws
+
+{{ .NodeConfigYaml }}
+
+--BOUNDARY
+Content-Type: application/node.eks.aws
+
+---
+apiVersion: node.eks.aws/v1alpha1
+kind: NodeConfig
+spec:
+  kubelet:
+    flags:
+      - --node-labels={{ $first := true }}{{ range $key, $value := .NodeLabels }}{{if not $first}},{{end}}{{ $key }}={{ $value }}{{ $first = false}}{{- end}}
+      - --register-with-taints={{ $first := true }}{{- range .NodeTaints}}{{if not $first}},{{end}}{{ .Key }}={{ .Value }}:{{ .Effect }}{{ $first = false}}{{- end}}
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+set +o xtrace
+{{range $post := .PostBootstrap}}{{$post}}{{end}}
+--BOUNDARY--`
 	}
 
 	data := EKSUserData{
@@ -206,7 +261,9 @@ set +o xtrace
 		Arguments:        args,
 		PreBootstrap:     payload.PreBootstrap,
 		PostBootstrap:    payload.PostBootstrap,
+		NodeConfigYaml:   payload.NodeConfigYaml,
 		MountOptions:     mounts,
+		ClusterIP:        clusterIP,
 	}
 	out := &bytes.Buffer{}
 	tmpl := template.New("userData").Funcs(template.FuncMap{
@@ -246,6 +303,12 @@ func (ctx *EksInstanceGroupContext) GetUserDataStages() UserDataPayload {
 				ctx.Log.Error(err, "failed to decode base64 stage data", "stage", stage.Stage, "data", stage.Data)
 			}
 			payload.PostBootstrap = append(payload.PostBootstrap, data)
+		case strings.EqualFold(stage.Stage, v1alpha1.NodeConfigYamlStage):
+			data, err := common.GetDecodedString(stage.Data)
+			if err != nil {
+				ctx.Log.Error(err, "failed to decode base64 stage data", "stage", stage.Stage, "data", stage.Data)
+			}
+			payload.NodeConfigYaml = data
 		default:
 			ctx.Log.Info("invalid userdata stage will not be rendered", "stage", stage.Stage, "data", stage.Data)
 		}
@@ -562,7 +625,7 @@ func (ctx *EksInstanceGroupContext) GetBootstrapArgs() string {
 			sb.WriteString(fmt.Sprintf("-ContainerRuntime %v ", bootstrapOptions.ContainerRuntime))
 		}
 		sb.WriteString(fmt.Sprintf("-KubeletExtraArgs '%v'", ctx.GetKubeletExtraArgs()))
-	case OsFamilyAmazonLinux2:
+	case OsFamilyAmazonLinux2, OsFamilyAmazonLinux2023:
 		if bootstrapOptions != nil && bootstrapOptions.MaxPods > 0 {
 			sb.WriteString("--use-max-pods false ")
 		}
@@ -1225,10 +1288,13 @@ func (ctx *EksInstanceGroupContext) GetEksLatestAmi() (string, error) {
 	)
 	clusterVersion := state.GetClusterVersion()
 	annotations := instanceGroup.GetAnnotations()
+	overrideAmazonLinuxFamily := strings.Trim(ctx.AmazonLinuxOsFamily, "\" ")
 
 	var OSFamily string
 	if kubeprovider.HasAnnotation(annotations, OsFamilyAnnotation) {
 		OSFamily = annotations[OsFamilyAnnotation]
+	} else if overrideAmazonLinuxFamily != "" {
+		OSFamily = overrideAmazonLinuxFamily
 	} else {
 		OSFamily = OsFamilyAmazonLinux2
 	}