replace os family container arg with cm flag

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -78,6 +78,8 @@ spec:
                 properties:
                   configuration:
                     properties:
+                      nodeConfig:
+                        type: string
                       bootstrapArguments:
                         type: string
                       bootstrapOptions:
@@ -253,8 +255,6 @@ spec:
                           - stage
                           type: object
                         type: array
-                      amazonLinuxOsFamily:
-                        type: string
                       volumes:
                         items:
                           properties:

controllers/instancegroup_controller.go

@@ -56,7 +56,6 @@ type InstanceGroupReconciler struct {
 	Metrics                     *common.MetricsCollector
 	DisableWinClusterInjection  bool
 	DefaultScalingConfiguration *v1alpha1.ScalingConfigurationType
-	AmazonLinuxOsFamily         string
 }
 
 type InstanceGroupAuthenticator struct {
@@ -144,7 +143,6 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque
 		ConfigRetention:            r.ConfigRetention,
 		Metrics:                    r.Metrics,
 		DisableWinClusterInjection: r.DisableWinClusterInjection,
-		AmazonLinuxOsFamily:        r.AmazonLinuxOsFamily,
 	}
 
 	var (

controllers/provisioners/eks/eks.go

@@ -79,7 +79,6 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext {
 		ConfigRetention:            p.ConfigRetention,
 		Metrics:                    p.Metrics,
 		DisableWinClusterInjection: p.DisableWinClusterInjection,
-		AmazonLinuxOsFamily:        p.AmazonLinuxOsFamily,
 	}
 
 	ctx.SetState(v1alpha1.ReconcileInit)
@@ -101,7 +100,6 @@ type EksInstanceGroupContext struct {
 	ResourcePrefix             string
 	Metrics                    *common.MetricsCollector
 	DisableWinClusterInjection bool
-	AmazonLinuxOsFamily        string
 }
 
 type UserDataPayload struct {
@@ -145,19 +143,38 @@ func (ctx *EksInstanceGroupContext) GetOsFamily() string {
 		instanceGroup = ctx.GetInstanceGroup()
 		annotations   = instanceGroup.GetAnnotations()
 	)
-	overrideAmazonLinuxFamily := strings.Trim(ctx.AmazonLinuxOsFamily, "\" ")
 
-	if v, exists := annotations[OsFamilyAnnotation]; exists {
+	if ctx.IsAmazonLinux2023() {
+		ctx.Log.Info("using amazonlinux2023 for os family")
+		return OsFamilyAmazonLinux2023
+	} else if v, exists := annotations[OsFamilyAnnotation]; exists {
 		if common.ContainsEqualFold(AllowedOsFamilies, v) {
+			ctx.Log.Info("using amazon linux os family annotation", "value", v)
 			return annotations[OsFamilyAnnotation]
 		}
 		ctx.Log.Info("used unsupported annotation value '%v=%v', will default to 'amazonlinux2', allowed values: %+v", OsFamilyAnnotation, v, AllowedOsFamilies)
-	} else if common.ContainsEqualFold(AllowedOsFamilies, overrideAmazonLinuxFamily) {
-		return overrideAmazonLinuxFamily
 	}
 	return OsFamilyAmazonLinux2
 }
 
+func (ctx *EksInstanceGroupContext) IsAmazonLinux2023() bool {
+
+	isAmazonLinux2023 := false
+	var (
+		instanceGroup = ctx.GetInstanceGroup()
+		configuration = instanceGroup.GetEKSConfiguration()
+		userData      = configuration.GetUserData()
+	)
+
+	for _, stage := range userData {
+		if strings.EqualFold(stage.Stage, v1alpha1.NodeConfigYamlStage) {
+			return true
+		}
+
+	}
+	return isAmazonLinux2023
+}
+
 func (ctx *EksInstanceGroupContext) GetUpgradeStrategy() *v1alpha1.AwsUpgradeStrategy {
 	// Check if the upgrade strategy has been set (non-zero value)
 	if ctx.InstanceGroup.Spec.AwsUpgradeStrategy != (v1alpha1.AwsUpgradeStrategy{}) {

controllers/provisioners/eks/helpers.go

@@ -1288,13 +1288,13 @@ func (ctx *EksInstanceGroupContext) GetEksLatestAmi() (string, error) {
 	)
 	clusterVersion := state.GetClusterVersion()
 	annotations := instanceGroup.GetAnnotations()
-	overrideAmazonLinuxFamily := strings.Trim(ctx.AmazonLinuxOsFamily, "\" ")
 
 	var OSFamily string
-	if kubeprovider.HasAnnotation(annotations, OsFamilyAnnotation) {
+	if ctx.IsAmazonLinux2023() {
+		ctx.Log.Info("using amazonlinux2023 to get latest ami")
+		OSFamily = OsFamilyAmazonLinux2023
+	} else if kubeprovider.HasAnnotation(annotations, OsFamilyAnnotation) {
 		OSFamily = annotations[OsFamilyAnnotation]
-	} else if overrideAmazonLinuxFamily != "" {
-		OSFamily = overrideAmazonLinuxFamily
 	} else {
 		OSFamily = OsFamilyAmazonLinux2
 	}

controllers/provisioners/eks/helpers_test.go

@@ -34,6 +34,18 @@ import (
 	"github.com/pkg/errors"
 )
 
+func mockUserDataStages() []v1alpha1.UserDataStage {
+	preBootstrapData := base64.StdEncoding.EncodeToString([]byte("echo Pre-bootstrap actions"))
+	postBootstrapData := base64.StdEncoding.EncodeToString([]byte("echo Post-bootstrap actions"))
+	nodeConfigYamlData := base64.StdEncoding.EncodeToString([]byte("image: my-custom-image"))
+
+	return []v1alpha1.UserDataStage{
+		{Stage: v1alpha1.PreBootstrapStage, Data: preBootstrapData},
+		{Stage: v1alpha1.PostBootstrapStage, Data: postBootstrapData},
+		{Stage: v1alpha1.NodeConfigYamlStage, Data: nodeConfigYamlData},
+	}
+}
+
 func TestAutoscalerTags(t *testing.T) {
 	var (
 		k       = MockKubernetesClientSet()
@@ -1418,3 +1430,54 @@ func TestGetEksLatestAmi(t *testing.T) {
 
 	}
 }
+
+func TestGetEksLatestAmiForAL2023(t *testing.T) {
+	var (
+		k            = MockKubernetesClientSet()
+		ig           = MockInstanceGroup()
+		config       = ig.GetEKSConfiguration()
+		asgMock      = NewAutoScalingMocker()
+		iamMock      = NewIamMocker()
+		eksMock      = NewEksMocker()
+		ec2Mock      = NewEc2Mocker()
+		ssmMock      = NewSsmMocker()
+		instanceType = "m5.large"
+	)
+	w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock)
+	ig.GetEKSConfiguration().UserData = mockUserDataStages()
+
+	tests := []struct {
+		name          string
+		OSFamily      string
+		arch          string
+		expectedError error
+	}{
+		{
+			name:          "amazonlinux2023",
+			OSFamily:      "",
+			arch:          "x86_64",
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range tests {
+		config.InstanceType = instanceType
+		ctx := MockContext(ig, k, w)
+		ctx.GetDiscoveredState().SetInstanceTypeInfo([]*ec2.InstanceTypeInfo{
+			{
+				InstanceType: aws.String(instanceType),
+				ProcessorInfo: &ec2.ProcessorInfo{
+					SupportedArchitectures: []*string{aws.String(tc.arch)},
+				},
+			},
+		})
+		_, err := ctx.GetEksLatestAmi()
+		if err == nil && tc.expectedError == nil {
+			continue
+		}
+		if err != nil && tc.expectedError != nil && err.Error() != tc.expectedError.Error() {
+			t.Fatalf("expected %v got %v, test %s", tc.expectedError, err, tc.name)
+		}
+
+	}
+}

controllers/provisioners/provisioners.go

@@ -30,7 +30,6 @@ type ProvisionerInput struct {
 	ConfigRetention            int
 	Metrics                    *common.MetricsCollector
 	DisableWinClusterInjection bool
-	AmazonLinuxOsFamily        string
 }
 
 var (

controllers/reconcilers.go

@@ -190,7 +190,10 @@ func (r *InstanceGroupReconciler) namespaceReconciler(obj client.Object) []ctrl.
 		return nil
 	}
 
+	oldNsAnnotations := map[string]string{}
+
 	if val, ok := r.Namespaces[name]; ok {
+		oldNsAnnotations = val.GetAnnotations()
 		if reflect.DeepEqual(val.GetAnnotations(), ns.GetAnnotations()) {
 			// annotations not modified
 			return nil
@@ -209,6 +212,7 @@ func (r *InstanceGroupReconciler) namespaceReconciler(obj client.Object) []ctrl.
 
 	requests := make([]ctrl.Request, 0)
 	for _, ig := range instanceGroups.Items {
+		ctrl.Log.Info("found namespace diff for instancegroup", "instancegroup", namespacedName, "old", oldNsAnnotations, "new", ns.GetAnnotations())
 		requests = append(requests, ctrl.Request{
 			NamespacedName: types.NamespacedName{
 				Namespace: ig.GetNamespace(),
@@ -256,6 +260,8 @@ func (r *InstanceGroupReconciler) nodeReconciler(obj client.Object) []ctrl.Reque
 		},
 	}
 
+	ctrl.Log.Info("patching node label", "nodeName", nodeName, "label", nodeLabels)
+
 	patchJSON, err := json.Marshal(labelPatch)
 	if err != nil {
 		r.Log.Error(err, "failed to marshal node labels", "node", nodeName, "patch", string(patchJSON))
@@ -312,6 +318,7 @@ func (r *InstanceGroupReconciler) spotEventReconciler(obj client.Object) []ctrl.
 		return nil
 	}
 
+	ctrl.Log.Info("found spot recommendation for instancegroup", "instancegroup", instanceGroup)
 	return []ctrl.Request{
 		{
 			NamespacedName: instanceGroup,

main.go

@@ -80,7 +80,6 @@ func main() {
 		configRetention             int
 		err                         error
 		defaultScalingConfiguration string
-		amazonLinuxOsFamily         string
 	)
 
 	flag.IntVar(&maxParallel, "max-workers", 5, "The number of maximum parallel reconciles")
@@ -94,7 +93,6 @@ func main() {
 	flag.BoolVar(&nodeRelabel, "node-relabel", true, "relabel nodes as they join with kubernetes.io/role label via controller")
 	flag.BoolVar(&disableWinClusterInjection, "disable-windows-cluster-ca-injection", false, "Setting this to true will cause the ClusterCA and Endpoint to not be injected for Windows nodes")
 	flag.StringVar(&defaultScalingConfiguration, "default-scaling-configuration", "LaunchTemplate", "By default ASGs will have LaunchTemplate. Set this string to either 'LaunchConfiguration' or 'LaunchTemplate' to enforce defaults.")
-	flag.StringVar(&amazonLinuxOsFamily, "amazon-linux-os-family", "", "Setting this determines the amazon linux os family version for instance groups. Set this string to 'amazonlinux2023' or 'amazonlinux2'.")
 	flag.Parse()
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 
@@ -175,7 +173,6 @@ func main() {
 			Aws:        awsWorker,
 			Kubernetes: kube,
 		},
-		AmazonLinuxOsFamily: amazonLinuxOsFamily,
 	}).SetupWithManager(mgr)
 	if err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "instancegroup")