diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index 22fbc212..d294fd90 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -78,6 +78,8 @@ spec: properties: configuration: properties: + nodeConfig: + type: string bootstrapArguments: type: string bootstrapOptions: @@ -253,8 +255,6 @@ spec: - stage type: object type: array - amazonLinuxOsFamily: - type: string volumes: items: properties: diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index d3c5f094..6a4d7346 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -56,7 +56,6 @@ type InstanceGroupReconciler struct { Metrics *common.MetricsCollector DisableWinClusterInjection bool DefaultScalingConfiguration *v1alpha1.ScalingConfigurationType - AmazonLinuxOsFamily string } type InstanceGroupAuthenticator struct { @@ -144,7 +143,6 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque ConfigRetention: r.ConfigRetention, Metrics: r.Metrics, DisableWinClusterInjection: r.DisableWinClusterInjection, - AmazonLinuxOsFamily: r.AmazonLinuxOsFamily, } var ( diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index df165a85..0b33ad94 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -79,7 +79,6 @@ func New(p provisioners.ProvisionerInput) *EksInstanceGroupContext { ConfigRetention: p.ConfigRetention, Metrics: p.Metrics, DisableWinClusterInjection: p.DisableWinClusterInjection, - AmazonLinuxOsFamily: p.AmazonLinuxOsFamily, } ctx.SetState(v1alpha1.ReconcileInit) @@ -101,7 +100,6 @@ type EksInstanceGroupContext struct { ResourcePrefix string Metrics *common.MetricsCollector DisableWinClusterInjection bool - AmazonLinuxOsFamily string } type UserDataPayload struct { @@ -145,19 +143,38 @@ func (ctx *EksInstanceGroupContext) GetOsFamily() string { instanceGroup = ctx.GetInstanceGroup() annotations = instanceGroup.GetAnnotations() ) - overrideAmazonLinuxFamily := strings.Trim(ctx.AmazonLinuxOsFamily, "\" ") - if v, exists := annotations[OsFamilyAnnotation]; exists { + if ctx.IsAmazonLinux2023() { + ctx.Log.Info("using amazonlinux2023 for os family") + return OsFamilyAmazonLinux2023 + } else if v, exists := annotations[OsFamilyAnnotation]; exists { if common.ContainsEqualFold(AllowedOsFamilies, v) { + ctx.Log.Info("using amazon linux os family annotation", "value", v) return annotations[OsFamilyAnnotation] } ctx.Log.Info("used unsupported annotation value '%v=%v', will default to 'amazonlinux2', allowed values: %+v", OsFamilyAnnotation, v, AllowedOsFamilies) - } else if common.ContainsEqualFold(AllowedOsFamilies, overrideAmazonLinuxFamily) { - return overrideAmazonLinuxFamily } return OsFamilyAmazonLinux2 } +func (ctx *EksInstanceGroupContext) IsAmazonLinux2023() bool { + + isAmazonLinux2023 := false + var ( + instanceGroup = ctx.GetInstanceGroup() + configuration = instanceGroup.GetEKSConfiguration() + userData = configuration.GetUserData() + ) + + for _, stage := range userData { + if strings.EqualFold(stage.Stage, v1alpha1.NodeConfigYamlStage) { + return true + } + + } + return isAmazonLinux2023 +} + func (ctx *EksInstanceGroupContext) GetUpgradeStrategy() *v1alpha1.AwsUpgradeStrategy { // Check if the upgrade strategy has been set (non-zero value) if ctx.InstanceGroup.Spec.AwsUpgradeStrategy != (v1alpha1.AwsUpgradeStrategy{}) { diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go index 83849c04..a276e381 100644 --- a/controllers/provisioners/eks/helpers.go +++ b/controllers/provisioners/eks/helpers.go @@ -1288,13 +1288,13 @@ func (ctx *EksInstanceGroupContext) GetEksLatestAmi() (string, error) { ) clusterVersion := state.GetClusterVersion() annotations := instanceGroup.GetAnnotations() - overrideAmazonLinuxFamily := strings.Trim(ctx.AmazonLinuxOsFamily, "\" ") var OSFamily string - if kubeprovider.HasAnnotation(annotations, OsFamilyAnnotation) { + if ctx.IsAmazonLinux2023() { + ctx.Log.Info("using amazonlinux2023 to get latest ami") + OSFamily = OsFamilyAmazonLinux2023 + } else if kubeprovider.HasAnnotation(annotations, OsFamilyAnnotation) { OSFamily = annotations[OsFamilyAnnotation] - } else if overrideAmazonLinuxFamily != "" { - OSFamily = overrideAmazonLinuxFamily } else { OSFamily = OsFamilyAmazonLinux2 } diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go index ac15f681..59de29e4 100644 --- a/controllers/provisioners/eks/helpers_test.go +++ b/controllers/provisioners/eks/helpers_test.go @@ -34,6 +34,18 @@ import ( "github.com/pkg/errors" ) +func mockUserDataStages() []v1alpha1.UserDataStage { + preBootstrapData := base64.StdEncoding.EncodeToString([]byte("echo Pre-bootstrap actions")) + postBootstrapData := base64.StdEncoding.EncodeToString([]byte("echo Post-bootstrap actions")) + nodeConfigYamlData := base64.StdEncoding.EncodeToString([]byte("image: my-custom-image")) + + return []v1alpha1.UserDataStage{ + {Stage: v1alpha1.PreBootstrapStage, Data: preBootstrapData}, + {Stage: v1alpha1.PostBootstrapStage, Data: postBootstrapData}, + {Stage: v1alpha1.NodeConfigYamlStage, Data: nodeConfigYamlData}, + } +} + func TestAutoscalerTags(t *testing.T) { var ( k = MockKubernetesClientSet() @@ -1418,3 +1430,54 @@ func TestGetEksLatestAmi(t *testing.T) { } } + +func TestGetEksLatestAmiForAL2023(t *testing.T) { + var ( + k = MockKubernetesClientSet() + ig = MockInstanceGroup() + config = ig.GetEKSConfiguration() + asgMock = NewAutoScalingMocker() + iamMock = NewIamMocker() + eksMock = NewEksMocker() + ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() + instanceType = "m5.large" + ) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) + ig.GetEKSConfiguration().UserData = mockUserDataStages() + + tests := []struct { + name string + OSFamily string + arch string + expectedError error + }{ + { + name: "amazonlinux2023", + OSFamily: "", + arch: "x86_64", + expectedError: nil, + }, + } + + for _, tc := range tests { + config.InstanceType = instanceType + ctx := MockContext(ig, k, w) + ctx.GetDiscoveredState().SetInstanceTypeInfo([]*ec2.InstanceTypeInfo{ + { + InstanceType: aws.String(instanceType), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: []*string{aws.String(tc.arch)}, + }, + }, + }) + _, err := ctx.GetEksLatestAmi() + if err == nil && tc.expectedError == nil { + continue + } + if err != nil && tc.expectedError != nil && err.Error() != tc.expectedError.Error() { + t.Fatalf("expected %v got %v, test %s", tc.expectedError, err, tc.name) + } + + } +} diff --git a/controllers/provisioners/provisioners.go b/controllers/provisioners/provisioners.go index b242dece..c24e9e96 100644 --- a/controllers/provisioners/provisioners.go +++ b/controllers/provisioners/provisioners.go @@ -30,7 +30,6 @@ type ProvisionerInput struct { ConfigRetention int Metrics *common.MetricsCollector DisableWinClusterInjection bool - AmazonLinuxOsFamily string } var ( diff --git a/controllers/reconcilers.go b/controllers/reconcilers.go index aeed51f8..a8e4fc9c 100644 --- a/controllers/reconcilers.go +++ b/controllers/reconcilers.go @@ -190,7 +190,10 @@ func (r *InstanceGroupReconciler) namespaceReconciler(obj client.Object) []ctrl. return nil } + oldNsAnnotations := map[string]string{} + if val, ok := r.Namespaces[name]; ok { + oldNsAnnotations = val.GetAnnotations() if reflect.DeepEqual(val.GetAnnotations(), ns.GetAnnotations()) { // annotations not modified return nil @@ -209,6 +212,7 @@ func (r *InstanceGroupReconciler) namespaceReconciler(obj client.Object) []ctrl. requests := make([]ctrl.Request, 0) for _, ig := range instanceGroups.Items { + ctrl.Log.Info("found namespace diff for instancegroup", "instancegroup", namespacedName, "old", oldNsAnnotations, "new", ns.GetAnnotations()) requests = append(requests, ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: ig.GetNamespace(), @@ -256,6 +260,8 @@ func (r *InstanceGroupReconciler) nodeReconciler(obj client.Object) []ctrl.Reque }, } + ctrl.Log.Info("patching node label", "nodeName", nodeName, "label", nodeLabels) + patchJSON, err := json.Marshal(labelPatch) if err != nil { r.Log.Error(err, "failed to marshal node labels", "node", nodeName, "patch", string(patchJSON)) @@ -312,6 +318,7 @@ func (r *InstanceGroupReconciler) spotEventReconciler(obj client.Object) []ctrl. return nil } + ctrl.Log.Info("found spot recommendation for instancegroup", "instancegroup", instanceGroup) return []ctrl.Request{ { NamespacedName: instanceGroup, diff --git a/main.go b/main.go index 612033c5..927ea7bd 100644 --- a/main.go +++ b/main.go @@ -80,7 +80,6 @@ func main() { configRetention int err error defaultScalingConfiguration string - amazonLinuxOsFamily string ) flag.IntVar(&maxParallel, "max-workers", 5, "The number of maximum parallel reconciles") @@ -94,7 +93,6 @@ func main() { flag.BoolVar(&nodeRelabel, "node-relabel", true, "relabel nodes as they join with kubernetes.io/role label via controller") flag.BoolVar(&disableWinClusterInjection, "disable-windows-cluster-ca-injection", false, "Setting this to true will cause the ClusterCA and Endpoint to not be injected for Windows nodes") flag.StringVar(&defaultScalingConfiguration, "default-scaling-configuration", "LaunchTemplate", "By default ASGs will have LaunchTemplate. Set this string to either 'LaunchConfiguration' or 'LaunchTemplate' to enforce defaults.") - flag.StringVar(&amazonLinuxOsFamily, "amazon-linux-os-family", "", "Setting this determines the amazon linux os family version for instance groups. Set this string to 'amazonlinux2023' or 'amazonlinux2'.") flag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -175,7 +173,6 @@ func main() { Aws: awsWorker, Kubernetes: kube, }, - AmazonLinuxOsFamily: amazonLinuxOsFamily, }).SetupWithManager(mgr) if err != nil { setupLog.Error(err, "unable to create controller", "controller", "instancegroup")