Skip to content

Commit 53c5d7a

Browse files
backjobackjo
authored
feat: use IMDS for autoscaling information (#357)
* feat: use IMDS for autoscaling information Signed-off-by: Jonah Back <[email protected]> * remove reference to AS cmdlet Signed-off-by: Jonah Back <[email protected]>
1 parent 8084fc4 commit 53c5d7a

File tree

5 files changed

+8
-16
lines changed

5 files changed

+8
-16
lines changed

‎controllers/provisioners/eks/eks.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ var (
5757
AllowedOsFamilies = []string{OsFamilyWindows, OsFamilyBottleRocket, OsFamilyAmazonLinux2}
5858
DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
5959
CNIManagedPolicy = "AmazonEKS_CNI_Policy"
60-
AutoscalingReadOnlyPolicy = "AutoScalingReadOnlyAccess"
6160
SupportedArchitectures = []string{"x86_64", "arm64"}
6261
)
6362

‎controllers/provisioners/eks/helpers.go‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ func (ctx *EksInstanceGroupContext) GetBasicUserData(clusterName, args string, k
137137
[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
138138
[string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
139139
[string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
140-
[string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
140+
[string]$Lifecycle=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
141141
if ($Lifecycle -like "*Warmed*") {
142142
Echo "Not starting Kubelet due to warmed state."
143143
& C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
@@ -182,7 +182,7 @@ if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
182182
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
183183
INSTANCE_ID=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
184184
REGION=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
185-
LIFECYCLE=$(aws autoscaling describe-auto-scaling-instances --region $REGION --instance-id $INSTANCE_ID | jq ".AutoScalingInstances[].LifecycleState" || true)
185+
LIFECYCLE=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
186186
if [[ $LIFECYCLE == *"Warmed"* ]]; then
187187
rm /var/lib/cloud/instances/$INSTANCE_ID/sem/config_scripts_user
188188
exit 0
@@ -1018,7 +1018,6 @@ func (ctx *EksInstanceGroupContext) UpdateLifecycleHooks(asgName string) error {
10181018
func (ctx *EksInstanceGroupContext) GetManagedPoliciesList(additionalPolicies []string) []string {
10191019
var (
10201020
instanceGroup = ctx.GetInstanceGroup()
1021-
spec = instanceGroup.GetEKSSpec()
10221021
annotations = instanceGroup.GetAnnotations()
10231022
)
10241023

@@ -1049,10 +1048,6 @@ func (ctx *EksInstanceGroupContext) GetManagedPoliciesList(additionalPolicies []
10491048
managedPolicies = append(managedPolicies, fmt.Sprintf("%s/%s", awsprovider.IAMPolicyPrefix, CNIManagedPolicy))
10501049
}
10511050

1052-
if spec.HasWarmPool() {
1053-
managedPolicies = append(managedPolicies, fmt.Sprintf("%s/%s", awsprovider.IAMPolicyPrefix, AutoscalingReadOnlyPolicy))
1054-
}
1055-
10561051
return managedPolicies
10571052
}
10581053

‎controllers/provisioners/eks/helpers_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
155155
TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
156156
INSTANCE_ID=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)
157157
REGION=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region)
158-
LIFECYCLE=$(aws autoscaling describe-auto-scaling-instances --region $REGION --instance-id $INSTANCE_ID | jq ".AutoScalingInstances[].LifecycleState" || true)
158+
LIFECYCLE=$(curl url -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
159159
if [[ $LIFECYCLE == *"Warmed"* ]]; then
160160
rm /var/lib/cloud/instances/$INSTANCE_ID/sem/config_scripts_user
161161
exit 0
@@ -224,7 +224,7 @@ func TestGetBasicUserDataWindows(t *testing.T) {
224224
[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
225225
[string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
226226
[string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
227-
[string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
227+
[string]$Lifecycle=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
228228
if ($Lifecycle -like "*Warmed*") {
229229
Echo "Not starting Kubelet due to warmed state."
230230
& C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
@@ -300,7 +300,7 @@ func TestGetBasicUserDataWindowsWithInjectionDisabled(t *testing.T) {
300300
[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
301301
[string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
302302
[string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
303-
[string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
303+
[string]$Lifecycle=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
304304
if ($Lifecycle -like "*Warmed*") {
305305
Echo "Not starting Kubelet due to warmed state."
306306
& C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule

‎controllers/provisioners/eks/update_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -632,7 +632,7 @@ func TestUpdateManagedPolicies(t *testing.T) {
632632

633633
defaultPolicies := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly"}
634634
defaultPoliciesIrsa := []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
635-
defaultPoliciesWarmPool := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly", "AutoScalingReadOnlyAccess"}
635+
defaultPoliciesWarmPool := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly"}
636636

637637
w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock)
638638
ctx := MockContext(ig, k, w)
@@ -652,9 +652,9 @@ func TestUpdateManagedPolicies(t *testing.T) {
652652
// when IRSA is disabled, cni policy needs to be attached
653653
{attachedPolicies: MockAttachedPolicies(defaultPoliciesIrsa...), additionalPolicies: []string{}, irsaEnabled: false, expectedAttached: 1, expectedDetached: 0},
654654
// when warm pool is enabled, managed role is added
655-
{attachedPolicies: MockAttachedPolicies(defaultPolicies...), additionalPolicies: []string{}, hasWarmPool: true, expectedAttached: 1, expectedDetached: 0},
655+
{attachedPolicies: MockAttachedPolicies(defaultPolicies...), additionalPolicies: []string{}, hasWarmPool: true, expectedAttached: 0, expectedDetached: 0},
656656
// when warm pool is disabled, managed role is removed
657-
{attachedPolicies: MockAttachedPolicies(defaultPoliciesWarmPool...), additionalPolicies: []string{}, hasWarmPool: false, expectedAttached: 0, expectedDetached: 1},
657+
{attachedPolicies: MockAttachedPolicies(defaultPoliciesWarmPool...), additionalPolicies: []string{}, hasWarmPool: false, expectedAttached: 0, expectedDetached: 0},
658658
// default policies not attached
659659
{attachedPolicies: MockAttachedPolicies(), additionalPolicies: []string{}, expectedAttached: 3, expectedDetached: 0},
660660
// additional policies need to be attached

‎docs/EKS.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -418,8 +418,6 @@ You can configure your scaling group to use [AWS Warm Pools for Auto Scaling](ht
418418

419419
Warm Pools is not officially supported with EKS, hence the following requirements exist in order to use it with EKS:
420420
- For Amazon Linux 2 instances, your AMI must have `awscli` & `jq`, you can also install it in pre-bootstrap userdata.
421-
- For Windows instances, your AMI must have the `Get-ASAutoScalingInstance` cmdlet installed.
422-
- If you use your own IAM role for the instance group, you must make sure it has access to `DescribeAutoScalingInstances`, this is required in order to figure out the current lifecycle state within userdata. If you are provisioning your IAM role through the controller, simply be aware that the controller will add the managed policy `AutoScalingReadOnlyAccess` to the role it creates.
423421
- This is currently only supported for AmazonLinux2 and Windows based AMIs.
424422

425423
We hope to get rid of these requirements in the future once AWS offers native support for Warm Pools in EKS, these are currently required in order to avoid warming instances to join the cluster briefly while they are provisioned, we are able to avoid this problem by checking if the instances is in the Warmed* lifecycle, and skipping bootstrapping in that case. Skipping on these requirements and using warm pools might mean having nodes join the cluster when the are being warmed, and cause unneeded scheduling of pods on nodes that are about to shutdown.

0 commit comments

Comments
 (0)