diff --git a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json index 0de36b3a56..4e29ad7455 100755 --- a/pkg/apis/eksctl.io/v1alpha5/assets/schema.json +++ b/pkg/apis/eksctl.io/v1alpha5/assets/schema.json @@ -1194,6 +1194,21 @@ "x-intellij-html-description": "holds any arbitrary JSON/YAML documents, such as extra config parameters or IAM policies", "default": "{}" }, + "InstanceMarketOptions": { + "properties": { + "marketType": { + "type": "string", + "description": "specifies the market type for the instances", + "x-intellij-html-description": "specifies the market type for the instances" + } + }, + "preferredOrder": [ + "marketType" + ], + "additionalProperties": false, + "description": "describes the market (purchasing) option for the instances", + "x-intellij-html-description": "describes the market (purchasing) option for the instances" + }, "InstanceSelector": { "properties": { "cpuArchitecture": { @@ -1419,6 +1434,11 @@ "iam": { "$ref": "#/definitions/NodeGroupIAM" }, + "instanceMarketOptions": { + "$ref": "#/definitions/InstanceMarketOptions", + "description": "describes the market (purchasing) option for the instances", + "x-intellij-html-description": "describes the market (purchasing) option for the instances" + }, "instanceName": { "type": "string" }, @@ -1629,6 +1649,7 @@ "bottlerocket", "enableDetailedMonitoring", "capacityReservation", + "instanceMarketOptions", "outpostARN", "instanceTypes", "spot", @@ -1792,6 +1813,11 @@ "iam": { "$ref": "#/definitions/NodeGroupIAM" }, + "instanceMarketOptions": { + "$ref": "#/definitions/InstanceMarketOptions", + "description": "describes the market (purchasing) option for the instances", + "x-intellij-html-description": "describes the market (purchasing) option for the instances" + }, "instanceName": { "type": "string" }, @@ -1999,6 +2025,7 @@ "bottlerocket", "enableDetailedMonitoring", "capacityReservation", + "instanceMarketOptions", "outpostARN", "instancesDistribution", "asgMetricsCollection", diff --git a/pkg/apis/eksctl.io/v1alpha5/types.go b/pkg/apis/eksctl.io/v1alpha5/types.go index 9803ec48ab..71820ef56e 100644 --- a/pkg/apis/eksctl.io/v1alpha5/types.go +++ b/pkg/apis/eksctl.io/v1alpha5/types.go @@ -1753,6 +1753,10 @@ type NodeGroupBase struct { // CapacityReservation defines reservation policy for a nodegroup CapacityReservation *CapacityReservation `json:"capacityReservation,omitempty"` + // InstanceMarketOptions describes the market (purchasing) option for the instances + // +optional + InstanceMarketOptions *InstanceMarketOptions `json:"instanceMarketOptions,omitempty"` + // OutpostARN specifies the Outpost ARN in which the nodegroup should be created. // +optional OutpostARN string `json:"outpostARN,omitempty"` @@ -1773,6 +1777,12 @@ type CapacityReservationTarget struct { CapacityReservationResourceGroupARN *string `json:"capacityReservationResourceGroupARN,omitempty"` } +// InstanceMarketOptions describes the market (purchasing) option for the instances +type InstanceMarketOptions struct { + // MarketType specifies the market type for the instances + MarketType *string `json:"marketType,omitempty"` +} + // Placement specifies placement group information type Placement struct { GroupName string `json:"groupName,omitempty"` diff --git a/pkg/apis/eksctl.io/v1alpha5/validation.go b/pkg/apis/eksctl.io/v1alpha5/validation.go index 4b1bd2a7b1..0dcf40f0cc 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation.go @@ -768,6 +768,18 @@ func validateNodeGroupBase(np NodePool, path string, controlPlaneOnOutposts bool return errors.New("only one of CapacityReservationID or CapacityReservationResourceGroupARN may be specified at a time") } } + + if ng.InstanceMarketOptions != nil { + if ng.InstanceMarketOptions.MarketType != nil { + if *ng.InstanceMarketOptions.MarketType != "capacity-block" { + return fmt.Errorf(`only accepted value is "capacity-block"; got "%s"`, *ng.InstanceMarketOptions.MarketType) + } + } + } + } else { + if ng.InstanceMarketOptions != nil { + return errors.New("instanceMarketOptions cannot be set without capacityReservation") + } } return nil diff --git a/pkg/apis/eksctl.io/v1alpha5/validation_test.go b/pkg/apis/eksctl.io/v1alpha5/validation_test.go index a001ad0bdf..ab54e7d635 100644 --- a/pkg/apis/eksctl.io/v1alpha5/validation_test.go +++ b/pkg/apis/eksctl.io/v1alpha5/validation_test.go @@ -2602,6 +2602,52 @@ var _ = Describe("ClusterConfig validation", func() { }) }) + Describe("Instance Market Options validation", func() { + var ( + cfg *api.ClusterConfig + ng *api.NodeGroup + ) + + BeforeEach(func() { + cfg = api.NewClusterConfig() + ng = cfg.NewNodeGroup() + ng.Name = "ng" + }) + + When("InstanceMarketOptions is set", func() { + When("it is set to 'capacity-block'", func() { + It("does not fail", func() { + ng.CapacityReservation = &api.CapacityReservation{ + CapacityReservationTarget: &api.CapacityReservationTarget{ + CapacityReservationID: aws.String("id"), + }, + } + ng.InstanceMarketOptions = &api.InstanceMarketOptions{MarketType: aws.String("capacity-block")} + Expect(api.ValidateNodeGroup(0, ng, cfg)).To(Succeed()) + }) + }) + + When("it is set to 'capacity-block' but Capacity Reservation not set", func() { + It("does fail", func() { + ng.InstanceMarketOptions = &api.InstanceMarketOptions{MarketType: aws.String("capacity-block")} + Expect(api.ValidateNodeGroup(0, ng, cfg)).To(MatchError(ContainSubstring(`instanceMarketOptions cannot be set without capacityReservation`))) + }) + }) + + When("it is set to 'foobar'", func() { + It("does fail", func() { + ng.CapacityReservation = &api.CapacityReservation{ + CapacityReservationTarget: &api.CapacityReservationTarget{ + CapacityReservationID: aws.String("id"), + }, + } + ng.InstanceMarketOptions = &api.InstanceMarketOptions{MarketType: aws.String("foobar")} + Expect(api.ValidateNodeGroup(0, ng, cfg)).To(MatchError(ContainSubstring(`only accepted value is "capacity-block"; got "foobar"`))) + }) + }) + }) + }) + DescribeTable("ToPodIdentityAssociationID", func(piaARN, expectedID, expectedErr string) { piaID, err := api.ToPodIdentityAssociationID(piaARN) if expectedErr != "" { diff --git a/pkg/cfn/builder/managed_launch_template.go b/pkg/cfn/builder/managed_launch_template.go index 697641cad8..f65b2f38e7 100644 --- a/pkg/cfn/builder/managed_launch_template.go +++ b/pkg/cfn/builder/managed_launch_template.go @@ -33,6 +33,11 @@ func (m *ManagedNodeGroupResourceSet) makeLaunchTemplateData(ctx context.Context CapacityReservationResourceGroupArn: valueOrNil(mng.CapacityReservation.CapacityReservationTarget.CapacityReservationResourceGroupARN), } } + if mng.InstanceMarketOptions != nil { + launchTemplateData.InstanceMarketOptions = &gfnec2.LaunchTemplate_InstanceMarketOptions{ + MarketType: valueOrNil(mng.InstanceMarketOptions.MarketType), + } + } } userData, err := m.bootstrapper.UserData() diff --git a/pkg/cfn/builder/managed_launch_template_test.go b/pkg/cfn/builder/managed_launch_template_test.go index e6746a5c1d..bb7a0c794c 100644 --- a/pkg/cfn/builder/managed_launch_template_test.go +++ b/pkg/cfn/builder/managed_launch_template_test.go @@ -266,6 +266,24 @@ API_SERVER_URL=https://test.com resourcesFilename: "spot.json", }), + Entry("With Capacity Block instances", &mngCase{ + ng: &api.ManagedNodeGroup{ + NodeGroupBase: &api.NodeGroupBase{ + Name: "cb-test", + CapacityReservation: &api.CapacityReservation{ + CapacityReservationTarget: &api.CapacityReservationTarget{ + CapacityReservationID: aws.String("res-id"), + }, + }, + InstanceMarketOptions: &api.InstanceMarketOptions{ + MarketType: aws.String("capacity-block"), + }, + }, + InstanceTypes: []string{"p5en.48xlarge"}, + }, + resourcesFilename: "capacity_block.json", + }), + Entry("With node repair enabled", &mngCase{ ng: &api.ManagedNodeGroup{ NodeGroupBase: &api.NodeGroupBase{ diff --git a/pkg/cfn/builder/managed_nodegroup.go b/pkg/cfn/builder/managed_nodegroup.go index 59ef839487..5fc29dfd9b 100644 --- a/pkg/cfn/builder/managed_nodegroup.go +++ b/pkg/cfn/builder/managed_nodegroup.go @@ -130,6 +130,14 @@ func (m *ManagedNodeGroupResourceSet) AddAllResources(ctx context.Context) error managedResource.CapacityType = gfnt.NewString("SPOT") } + isCapacityBlockEnabled := false + if m.nodeGroup.InstanceMarketOptions != nil && + m.nodeGroup.InstanceMarketOptions.MarketType != nil && + *m.nodeGroup.InstanceMarketOptions.MarketType == "capacity-block" { + isCapacityBlockEnabled = true + managedResource.CapacityType = gfnt.NewString("CAPACITY_BLOCK") + } + if m.nodeGroup.ReleaseVersion != "" { managedResource.ReleaseVersion = gfnt.NewString(m.nodeGroup.ReleaseVersion) } @@ -167,7 +175,14 @@ func (m *ManagedNodeGroupResourceSet) AddAllResources(ctx context.Context) error } if launchTemplateData.InstanceType == "" { - managedResource.InstanceTypes = gfnt.NewStringSlice(instanceTypes...) + if isCapacityBlockEnabled { + if len(instanceTypes) > 1 { + return errors.New("when using capacity type CAPACITY_BLOCK please specify only one instance type") + } + launchTemplateData.InstanceType = ec2types.InstanceType(instanceTypes[0]) + } else { + managedResource.InstanceTypes = gfnt.NewStringSlice(instanceTypes...) + } } } else { launchTemplateData, err := m.makeLaunchTemplateData(ctx) @@ -177,7 +192,15 @@ func (m *ManagedNodeGroupResourceSet) AddAllResources(ctx context.Context) error if launchTemplateData.ImageId == nil { managedResource.AmiType = makeAMIType() } - managedResource.InstanceTypes = gfnt.NewStringSlice(instanceTypes...) + + if isCapacityBlockEnabled { + if len(instanceTypes) > 1 { + return errors.New("cannot specify multiple instance types when using capacity block") + } + launchTemplateData.InstanceType = gfnt.NewString(instanceTypes[0]) + } else { + managedResource.InstanceTypes = gfnt.NewStringSlice(instanceTypes...) + } ltRef := m.newResource("LaunchTemplate", &gfnec2.LaunchTemplate{ LaunchTemplateName: gfnt.MakeFnSubString(fmt.Sprintf("${%s}", gfnt.StackName)), diff --git a/pkg/cfn/builder/nodegroup.go b/pkg/cfn/builder/nodegroup.go index 25852ca217..617d5be813 100644 --- a/pkg/cfn/builder/nodegroup.go +++ b/pkg/cfn/builder/nodegroup.go @@ -470,6 +470,11 @@ func newLaunchTemplateData(ctx context.Context, n *NodeGroupResourceSet) (*gfnec CapacityReservationResourceGroupArn: valueOrNil(ng.CapacityReservation.CapacityReservationTarget.CapacityReservationResourceGroupARN), } } + if ng.InstanceMarketOptions != nil { + launchTemplateData.InstanceMarketOptions = &gfnec2.LaunchTemplate_InstanceMarketOptions{ + MarketType: valueOrNil(ng.InstanceMarketOptions.MarketType), + } + } } if err := buildNetworkInterfaces(ctx, launchTemplateData, ng.InstanceTypeList(), api.IsEnabled(ng.EFAEnabled), n.securityGroups, n.ec2API); err != nil { diff --git a/pkg/cfn/builder/testdata/launch_template/capacity_block.json b/pkg/cfn/builder/testdata/launch_template/capacity_block.json new file mode 100644 index 0000000000..088a2c521e --- /dev/null +++ b/pkg/cfn/builder/testdata/launch_template/capacity_block.json @@ -0,0 +1,179 @@ +{ + "LaunchTemplate": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "Iops": 3000, + "Throughput": 125, + "VolumeSize": 80, + "VolumeType": "gp3" + } + } + ], + "CapacityReservationSpecification": { + "CapacityReservationTarget": { + "CapacityReservationId": "res-id" + } + }, + "InstanceMarketOptions": { + "MarketType": "capacity-block" + }, + "InstanceType": "p5en.48xlarge", + "MetadataOptions": { + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required" + }, + "SecurityGroupIds": [ + { + "Fn::ImportValue": "eksctl-lt::ClusterSecurityGroupId" + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "lt-cb-test-Node" + }, + { + "Key": "alpha.eksctl.io/nodegroup-name", + "Value": "cb-test" + }, + { + "Key": "alpha.eksctl.io/nodegroup-type", + "Value": "managed" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "lt-cb-test-Node" + }, + { + "Key": "alpha.eksctl.io/nodegroup-name", + "Value": "cb-test" + }, + { + "Key": "alpha.eksctl.io/nodegroup-type", + "Value": "managed" + } + ] + }, + { + "ResourceType": "network-interface", + "Tags": [ + { + "Key": "Name", + "Value": "lt-cb-test-Node" + }, + { + "Key": "alpha.eksctl.io/nodegroup-name", + "Value": "cb-test" + }, + { + "Key": "alpha.eksctl.io/nodegroup-type", + "Value": "managed" + } + ] + } + ] + }, + "LaunchTemplateName": { + "Fn::Sub": "${AWS::StackName}" + } + } + }, + "ManagedNodeGroup": { + "Type": "AWS::EKS::Nodegroup", + "Properties": { + "AmiType": "AL2023_x86_64_NVIDIA", + "CapacityType": "CAPACITY_BLOCK", + "ClusterName": "lt", + "Labels": { + "alpha.eksctl.io/cluster-name": "lt", + "alpha.eksctl.io/nodegroup-name": "cb-test" + }, + "LaunchTemplate": { + "Id": { + "Ref": "LaunchTemplate" + } + }, + "NodeRole": { + "Fn::GetAtt": [ + "NodeInstanceRole", + "Arn" + ] + }, + "NodegroupName": "cb-test", + "ScalingConfig": { + "DesiredSize": 2, + "MaxSize": 2, + "MinSize": 2 + }, + "Tags": { + "alpha.eksctl.io/nodegroup-name": "cb-test", + "alpha.eksctl.io/nodegroup-type": "managed" + } + } + }, + "NodeInstanceRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + { + "Fn::FindInMap": [ + "ServicePrincipalPartitionMap", + { + "Ref": "AWS::Partition" + }, + "EC2" + ] + } + ] + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + }, + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + ], + "Path": "/", + "Tags": [ + { + "Key": "Name", + "Value": { + "Fn::Sub": "${AWS::StackName}/NodeInstanceRole" + } + } + ] + } + } +}