diff --git a/controllers/provisioners/eks/cloud.go b/controllers/provisioners/eks/cloud.go index 4eb10b1b..5181a5d3 100644 --- a/controllers/provisioners/eks/cloud.go +++ b/controllers/provisioners/eks/cloud.go @@ -408,6 +408,19 @@ func (d *DiscoveredState) SetClusterNodes(nodes *corev1.NodeList) { func (d *DiscoveredState) GetClusterNodes() *corev1.NodeList { return d.ClusterNodes } +func (d *DiscoveredState) GetRunningInstanceTypes() []string { + types := make([]string, 0) + if d.ScalingGroup == nil { + return types + } + for _, t := range d.ScalingGroup.Instances { + instanceType := aws.StringValue(t.InstanceType) + if !common.ContainsEqualFold(types, instanceType) { + types = append(types, instanceType) + } + } + return types +} func (d *DiscoveredState) SetSubFamilyFlexiblePool(pool map[string][]InstanceSpec) { d.InstancePool.SubFamilyFlexiblePool = InstancePool{ diff --git a/controllers/provisioners/eks/eks_test.go b/controllers/provisioners/eks/eks_test.go index a50c1c60..2fd2f1d3 100644 --- a/controllers/provisioners/eks/eks_test.go +++ b/controllers/provisioners/eks/eks_test.go @@ -309,6 +309,17 @@ func MockTagDescription(key, value string) *autoscaling.TagDescription { } } +func MockTemplateOverrides(weight string, types ...string) []*autoscaling.LaunchTemplateOverrides { + overrides := make([]*autoscaling.LaunchTemplateOverrides, 0) + for _, t := range types { + overrides = append(overrides, &autoscaling.LaunchTemplateOverrides{ + InstanceType: aws.String(t), + WeightedCapacity: aws.String(weight), + }) + } + return overrides +} + func MockScalingGroup(name string, withTemplate bool, t ...*autoscaling.TagDescription) *autoscaling.Group { asg := &autoscaling.Group{ AutoScalingGroupName: aws.String(name), @@ -316,6 +327,11 @@ func MockScalingGroup(name string, withTemplate bool, t ...*autoscaling.TagDescr MinSize: aws.Int64(3), MaxSize: aws.Int64(6), VPCZoneIdentifier: aws.String("subnet-1,subnet-2,subnet-3"), + Instances: []*autoscaling.Instance{ + { + InstanceType: aws.String("m5.xlarge"), + }, + }, } if withTemplate { diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go index 37ebb645..32bfbee8 100644 --- a/controllers/provisioners/eks/helpers.go +++ b/controllers/provisioners/eks/helpers.go @@ -1100,10 +1100,15 @@ func (ctx *EksInstanceGroupContext) GetOverrides() []*autoscaling.LaunchTemplate primaryType = configuration.InstanceType mixedPolicy = configuration.GetMixedInstancesPolicy() state = ctx.GetDiscoveredState() + runningTypes = state.GetRunningInstanceTypes() ) overrides := []*autoscaling.LaunchTemplateOverrides{} + if mixedPolicy == nil { + return overrides + } - if mixedPolicy != nil && mixedPolicy.InstanceTypes != nil { + // Create overrides from specific instanceTypes or derive from instancePool + if mixedPolicy.InstanceTypes != nil { overrides = append(overrides, &autoscaling.LaunchTemplateOverrides{ InstanceType: aws.String(primaryType), WeightedCapacity: aws.String("1"), @@ -1115,18 +1120,33 @@ func (ctx *EksInstanceGroupContext) GetOverrides() []*autoscaling.LaunchTemplate WeightedCapacity: aws.String(weightStr), }) } - return overrides + } else if mixedPolicy.InstancePool != nil { + if strings.EqualFold(*mixedPolicy.InstancePool, string(SubFamilyFlexible)) { + if pool, ok := state.InstancePool.SubFamilyFlexiblePool.GetPool(primaryType); ok { + for _, p := range pool { + overrides = append(overrides, &autoscaling.LaunchTemplateOverrides{ + InstanceType: aws.String(p.Type), + WeightedCapacity: aws.String(p.Weight), + }) + } + } + } } - switch { - case strings.EqualFold(*mixedPolicy.InstancePool, string(SubFamilyFlexible)): - if pool, ok := state.InstancePool.SubFamilyFlexiblePool.GetPool(primaryType); ok { - for _, p := range pool { - overrides = append(overrides, &autoscaling.LaunchTemplateOverrides{ - InstanceType: aws.String(p.Type), - WeightedCapacity: aws.String(p.Weight), - }) - } + // if some type is already running in the group (when switching from LaunchConfiguration to LaunchTemplate), it must be included in overrides + // Once the type is replaced with the new primary type it will no longer be added as an override + var overrideTypes = make([]string, 0) + for _, o := range overrides { + override := aws.StringValue(o.InstanceType) + overrideTypes = append(overrideTypes, override) + } + + for _, t := range runningTypes { + if !common.ContainsEqualFold(overrideTypes, t) { + overrides = append(overrides, &autoscaling.LaunchTemplateOverrides{ + InstanceType: aws.String(t), + WeightedCapacity: aws.String("1"), + }) } } diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go index 2fdcc6e6..76dd236e 100644 --- a/controllers/provisioners/eks/helpers_test.go +++ b/controllers/provisioners/eks/helpers_test.go @@ -806,6 +806,85 @@ func TestGetMountOpts(t *testing.T) { } } +func TestGetOverrides(t *testing.T) { + var ( + g = gomega.NewGomegaWithT(t) + k = MockKubernetesClientSet() + ig = MockInstanceGroup() + configuration = ig.GetEKSConfiguration() + asgMock = NewAutoScalingMocker() + iamMock = NewIamMocker() + eksMock = NewEksMocker() + ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() + ) + + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) + ctx := MockContext(ig, k, w) + state := ctx.GetDiscoveredState() + + instancePool := "SubFamilyFlexible" + + tests := []struct { + primaryType string + scalingGroup *autoscaling.Group + mixedInstancesSpec *v1alpha1.MixedInstancesPolicySpec + expectedOverrides []*autoscaling.LaunchTemplateOverrides + }{ + { + primaryType: "m5.xlarge", + scalingGroup: MockScalingGroup("asg-1", true), + mixedInstancesSpec: &v1alpha1.MixedInstancesPolicySpec{ + InstanceTypes: []*v1alpha1.InstanceTypeSpec{ + { + Type: "m5a.xlarge", + Weight: 1, + }, + { + Type: "m5g.xlarge", + Weight: 1, + }, + }, + }, + expectedOverrides: MockTemplateOverrides("1", "m5a.xlarge", "m5g.xlarge", "m5.xlarge"), + }, + { + primaryType: "m5.xlarge", + scalingGroup: MockScalingGroup("asg-1", true), + mixedInstancesSpec: &v1alpha1.MixedInstancesPolicySpec{ + InstancePool: &instancePool, + }, + expectedOverrides: MockTemplateOverrides("1", "m5.xlarge"), + }, + { + primaryType: "t2.xlarge", + scalingGroup: MockScalingGroup("asg-1", true), + mixedInstancesSpec: &v1alpha1.MixedInstancesPolicySpec{ + InstanceTypes: []*v1alpha1.InstanceTypeSpec{ + { + Type: "t2a.xlarge", + Weight: 1, + }, + { + Type: "t2g.xlarge", + Weight: 1, + }, + }, + }, + expectedOverrides: MockTemplateOverrides("1", "t2a.xlarge", "t2g.xlarge", "t2.xlarge", "m5.xlarge"), + }, + } + + for i, tc := range tests { + t.Logf("Test #%v - %+v", i, tc.expectedOverrides) + configuration.MixedInstancesPolicy = tc.mixedInstancesSpec + state.ScalingGroup = tc.scalingGroup + ig.Spec.EKSSpec.EKSConfiguration.InstanceType = tc.primaryType + overrides := ctx.GetOverrides() + g.Expect(overrides).To(gomega.ConsistOf(tc.expectedOverrides)) + } +} + func TestGetUserDataStages(t *testing.T) { var ( g = gomega.NewGomegaWithT(t)