Skip to content

Commit 6e702bd

Browse files
eytan-avisroreytan-avisror
authored
Merge branch 'master' into tags-update-fix
2 parents c61fb8e + ea5c8ae commit 6e702bd

File tree

8 files changed

+149
-30
lines changed

8 files changed

+149
-30
lines changed

‎controllers/providers/aws/aws.go‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323

2424
"github.com/aws/aws-sdk-go/aws"
2525
"github.com/aws/aws-sdk-go/aws/ec2metadata"
26-
"github.com/aws/aws-sdk-go/aws/session"
2726
"github.com/aws/aws-sdk-go/service/autoscaling"
2827
"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
2928
"github.com/aws/aws-sdk-go/service/ec2"
@@ -112,11 +111,12 @@ var (
112111
)
113112

114113
type AwsWorker struct {
115-
AsgClient autoscalingiface.AutoScalingAPI
116-
EksClient eksiface.EKSAPI
117-
IamClient iamiface.IAMAPI
118-
Ec2Client ec2iface.EC2API
119-
Parameters map[string]interface{}
114+
AsgClient autoscalingiface.AutoScalingAPI
115+
EksClient eksiface.EKSAPI
116+
IamClient iamiface.IAMAPI
117+
Ec2Client ec2iface.EC2API
118+
Ec2Metadata *ec2metadata.EC2Metadata
119+
Parameters map[string]interface{}
120120
}
121121

122122
func (w *AwsWorker) WithRetries(f func() bool) error {
@@ -164,18 +164,12 @@ func GetTagValueByKey(tags []*autoscaling.TagDescription, key string) string {
164164
return ""
165165
}
166166

167-
func GetRegion() (string, error) {
167+
func GetRegion(metadata *ec2metadata.EC2Metadata) (string, error) {
168168
if os.Getenv("AWS_REGION") != "" {
169169
return os.Getenv("AWS_REGION"), nil
170170
}
171-
// Try Derive
172-
var config aws.Config
173-
sess := session.Must(session.NewSessionWithOptions(session.Options{
174-
SharedConfigState: session.SharedConfigEnable,
175-
Config: config,
176-
}))
177-
c := ec2metadata.New(sess)
178-
region, err := c.Region()
171+
172+
region, err := metadata.Region()
179173
if err != nil {
180174
return "", err
181175
}

‎controllers/providers/aws/ec2.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"strings"
2020

2121
"github.com/aws/aws-sdk-go/aws"
22+
"github.com/aws/aws-sdk-go/aws/ec2metadata"
2223
"github.com/aws/aws-sdk-go/aws/request"
2324
"github.com/aws/aws-sdk-go/aws/session"
2425
"github.com/aws/aws-sdk-go/service/ec2"
@@ -56,6 +57,15 @@ func GetAwsEc2Client(region string, cacheCfg *cache.Config, maxRetries int, coll
5657
return ec2.New(sess)
5758
}
5859

60+
func GetAwsEc2MetadataClient() *ec2metadata.EC2Metadata {
61+
var config aws.Config
62+
sess := session.Must(session.NewSessionWithOptions(session.Options{
63+
SharedConfigState: session.SharedConfigEnable,
64+
Config: config,
65+
}))
66+
return ec2metadata.New(sess)
67+
}
68+
5969
func (w *AwsWorker) DescribeInstanceOfferings() ([]*ec2.InstanceTypeOffering, error) {
6070
offerings := []*ec2.InstanceTypeOffering{}
6171
err := w.Ec2Client.DescribeInstanceTypeOfferingsPages(&ec2.DescribeInstanceTypeOfferingsInput{}, func(page *ec2.DescribeInstanceTypeOfferingsOutput, lastPage bool) bool {

‎controllers/providers/aws/eks.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ limitations under the License.
1616
package aws
1717

1818
import (
19+
"strings"
20+
1921
"github.com/aws/aws-sdk-go/aws"
2022
"github.com/aws/aws-sdk-go/aws/awserr"
2123
"github.com/aws/aws-sdk-go/aws/request"
@@ -210,3 +212,12 @@ func (w *AwsWorker) DescribeFargateProfile() (*eks.FargateProfile, error) {
210212
}
211213
return output.FargateProfile, nil
212214
}
215+
216+
func (w *AwsWorker) GetDNSClusterIP(cluster *eks.Cluster) string {
217+
if cluster == nil {
218+
return ""
219+
}
220+
serviceCidr := aws.StringValue(cluster.KubernetesNetworkConfig.ServiceIpv4Cidr)
221+
// addresses assigned from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks
222+
return strings.ReplaceAll(serviceCidr, "0/16", "10")
223+
}

‎controllers/provisioners/eks/cloud.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,10 @@ func (d *DiscoveredState) SetCluster(cluster *eks.Cluster) {
285285
d.Cluster = cluster
286286
}
287287

288+
func (d *DiscoveredState) GetCluster() *eks.Cluster {
289+
return d.Cluster
290+
}
291+
288292
func (d *DiscoveredState) SetVPCId(id string) {
289293
d.VPCId = id
290294
}

‎controllers/provisioners/eks/eks_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,14 @@ func MockAwsWorker(asgClient *MockAutoScalingClient, iamClient *MockIamClient, e
8686
func MockEksCluster(version string) *eks.Cluster {
8787
return &eks.Cluster{
8888
CertificateAuthority: &eks.Certificate{
89-
Data: aws.String(""),
89+
Data: aws.String("dGVzdA=="),
9090
},
9191
Endpoint: aws.String("foo.amazonaws.com"),
9292
ResourcesVpcConfig: &eks.VpcConfigResponse{},
93-
Version: &version,
93+
KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{
94+
ServiceIpv4Cidr: aws.String("172.20.0.0/16"),
95+
},
96+
Version: &version,
9497
}
9598
}
9699

‎controllers/provisioners/eks/helpers.go‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ func (ctx *EksInstanceGroupContext) GetBasicUserData(clusterName, args string, k
133133
Echo "Not starting Kubelet due to warmed state."
134134
& C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
135135
} else {
136-
& $EKSBootstrapScriptFile -EKSClusterName {{ .ClusterName }} -KubeletExtraArgs '{{ .KubeletExtraArgs }}' 3>&1 4>&1 5>&1 6>&1
136+
& $EKSBootstrapScriptFile -EKSClusterName {{ .ClusterName }} {{ .Arguments }} 3>&1 4>&1 5>&1 6>&1
137137
{{range $post := .PostBootstrap}}{{$post}}{{end}}
138138
}
139139
</powershell>`
@@ -513,13 +513,34 @@ func (ctx *EksInstanceGroupContext) GetComputedBootstrapOptions() *v1alpha1.Boot
513513
func (ctx *EksInstanceGroupContext) GetBootstrapArgs() string {
514514
var (
515515
bootstrapOptions = ctx.GetComputedBootstrapOptions()
516+
state = ctx.GetDiscoveredState()
517+
osFamily = ctx.GetOsFamily()
518+
cluster = state.GetCluster()
519+
clusterIP = ctx.AwsWorker.GetDNSClusterIP(cluster)
516520
)
517521
var sb strings.Builder
522+
switch strings.ToLower(osFamily) {
523+
case OsFamilyWindows:
524+
if state.Cluster != nil {
525+
sb.WriteString(fmt.Sprintf("-Base64ClusterCA %v ", aws.StringValue(state.Cluster.CertificateAuthority.Data)))
526+
sb.WriteString(fmt.Sprintf("-APIServerEndpoint %v ", aws.StringValue(state.Cluster.Endpoint)))
527+
}
528+
sb.WriteString(fmt.Sprintf("-KubeletExtraArgs '%v'", ctx.GetKubeletExtraArgs()))
529+
case OsFamilyAmazonLinux2:
530+
if bootstrapOptions != nil && bootstrapOptions.MaxPods > 0 {
531+
sb.WriteString("--use-max-pods false ")
532+
}
533+
if state.Cluster != nil {
534+
sb.WriteString(fmt.Sprintf("--b64-cluster-ca %v ", aws.StringValue(state.Cluster.CertificateAuthority.Data)))
535+
sb.WriteString(fmt.Sprintf("--apiserver-endpoint %v ", aws.StringValue(state.Cluster.Endpoint)))
536+
if !common.StringEmpty(clusterIP) {
537+
sb.WriteString(fmt.Sprintf("--dns-cluster-ip %v ", clusterIP))
538+
}
539+
}
518540

519-
if bootstrapOptions != nil && bootstrapOptions.MaxPods > 0 {
520-
sb.WriteString("--use-max-pods false ")
541+
sb.WriteString(fmt.Sprintf("--kubelet-extra-args '%v'", ctx.GetKubeletExtraArgs()))
521542
}
522-
sb.WriteString(fmt.Sprintf("--kubelet-extra-args '%v'", ctx.GetKubeletExtraArgs()))
543+
523544
return sb.String()
524545
}
525546

‎controllers/provisioners/eks/helpers_test.go‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ func TestGetBasicUserDataAmazonLinux2(t *testing.T) {
142142
mounts = ctx.GetMountOpts()
143143
)
144144

145-
expectedData := `#!/bin/bash
145+
expectedDataLinux := `#!/bin/bash
146146
foo
147147
mkfs.xfs /dev/xvda
148148
mkdir /mnt/foo
@@ -159,16 +159,90 @@ if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
159159
fi
160160
fi
161161
set -o xtrace
162-
/etc/eks/bootstrap.sh foo --use-max-pods false --kubelet-extra-args '--node-labels=foo=bar,instancemgr.keikoproj.io/image=ami-123456789012,node.kubernetes.io/role=instance-group-1 --register-with-taints=foo=bar:NoSchedule --eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2 --max-pods=4'
162+
/etc/eks/bootstrap.sh foo --use-max-pods false --b64-cluster-ca dGVzdA== --apiserver-endpoint foo.amazonaws.com --dns-cluster-ip 172.20.0.10 --kubelet-extra-args '--node-labels=foo=bar,instancemgr.keikoproj.io/image=ami-123456789012,node.kubernetes.io/role=instance-group-1 --register-with-taints=foo=bar:NoSchedule --eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2 --max-pods=4'
163163
set +o xtrace
164164
bar`
165165
userData := ctx.GetBasicUserData("foo", args, kubeletArgs, userDataPayload, mounts)
166166
basicUserDataDecoded, _ := base64.StdEncoding.DecodeString(userData)
167167
basicUserDataString := string(basicUserDataDecoded)
168-
if basicUserDataString != expectedData {
169-
t.Fatalf("\nExpected: START>%v<END\n Got: START>%v<END", expectedData, basicUserDataString)
168+
if basicUserDataString != expectedDataLinux {
169+
t.Fatalf("\nExpected: START>%v<END\n Got: START>%v<END", expectedDataLinux, basicUserDataString)
170170
}
171+
}
172+
173+
func TestGetBasicUserDataWindows(t *testing.T) {
174+
var (
175+
k = MockKubernetesClientSet()
176+
ig = MockInstanceGroup()
177+
asgMock = NewAutoScalingMocker()
178+
iamMock = NewIamMocker()
179+
eksMock = NewEksMocker()
180+
ec2Mock = NewEc2Mocker()
181+
configuration = ig.GetEKSConfiguration()
182+
)
183+
184+
w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock)
185+
ctx := MockContext(ig, k, w)
186+
187+
configuration.BootstrapOptions = &v1alpha1.BootstrapOptions{
188+
MaxPods: 4,
189+
}
190+
configuration.Labels = map[string]string{
191+
"foo": "bar",
192+
}
193+
configuration.Taints = []corev1.Taint{
194+
{
195+
Key: "foo",
196+
Value: "bar",
197+
Effect: "NoSchedule",
198+
},
199+
}
200+
201+
configuration.BootstrapArguments = "--eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2"
202+
configuration.UserData = []v1alpha1.UserDataStage{
203+
{
204+
Stage: "PreBootstrap",
205+
Data: "foo",
206+
},
207+
{
208+
Stage: "PostBootstrap",
209+
Data: "bar",
210+
},
211+
}
212+
213+
ig.Annotations[OsFamilyAnnotation] = OsFamilyWindows
214+
215+
expectedDataWindows := `
216+
<powershell>
217+
foo
218+
[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS"
219+
[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1'
220+
[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName"
221+
[string]$IMDSToken=(curl -UseBasicParsing -Method PUT "http://169.254.169.254/latest/api/token" -H @{ "X-aws-ec2-metadata-token-ttl-seconds" = "21600"} | % { Echo $_.Content})
222+
[string]$InstanceID=(curl -UseBasicParsing -Method GET "http://169.254.169.254/latest/meta-data/instance-id" -H @{ "X-aws-ec2-metadata-token" = "$IMDSToken"} | % { Echo $_.Content})
223+
[string]$Lifecycle = Get-ASAutoScalingInstance $InstanceID | % { Echo $_.LifecycleState}
224+
if ($Lifecycle -like "*Warmed*") {
225+
Echo "Not starting Kubelet due to warmed state."
226+
& C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 –Schedule
227+
} else {
228+
& $EKSBootstrapScriptFile -EKSClusterName foo -Base64ClusterCA dGVzdA== -APIServerEndpoint foo.amazonaws.com -KubeletExtraArgs '--node-labels=foo=bar,instancemgr.keikoproj.io/image=ami-123456789012,node.kubernetes.io/role=instance-group-1 --register-with-taints=foo=bar:NoSchedule --eviction-hard=memory.available<300Mi,nodefs.available<5% --system-reserved=memory=2.5Gi --v=2 --max-pods=4' 3>&1 4>&1 5>&1 6>&1
229+
bar
230+
}
231+
</powershell>`
171232

233+
var (
234+
args = ctx.GetBootstrapArgs()
235+
kubeletArgs = ctx.GetKubeletExtraArgs()
236+
userDataPayload = ctx.GetUserDataStages()
237+
mounts = ctx.GetMountOpts()
238+
)
239+
240+
userData := ctx.GetBasicUserData("foo", args, kubeletArgs, userDataPayload, mounts)
241+
basicUserDataDecoded, _ := base64.StdEncoding.DecodeString(userData)
242+
basicUserDataString := string(basicUserDataDecoded)
243+
if basicUserDataString != expectedDataWindows {
244+
t.Fatalf("\nExpected: START>%v<END\n Got: START>%v<END", expectedDataWindows, basicUserDataString)
245+
}
172246
}
173247

174248
func TestCustomNetworkingMaxPods(t *testing.T) {

‎main.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ func main() {
9898
os.Exit(1)
9999
}
100100

101-
awsRegion, err := aws.GetRegion()
101+
metadata := aws.GetAwsEc2MetadataClient()
102+
awsRegion, err := aws.GetRegion(metadata)
102103
if err != nil {
103104
setupLog.Error(err, "unable to get AWS region")
104105
os.Exit(1)
@@ -120,10 +121,11 @@ func main() {
120121
cacheCollector := cacheCfg.NewCacheCollector("instance_manager")
121122
controllerCollector := common.NewMetricsCollector()
122123
awsWorker := aws.AwsWorker{
123-
Ec2Client: aws.GetAwsEc2Client(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
124-
IamClient: aws.GetAwsIamClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
125-
AsgClient: aws.GetAwsAsgClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
126-
EksClient: aws.GetAwsEksClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
124+
Ec2Client: aws.GetAwsEc2Client(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
125+
IamClient: aws.GetAwsIamClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
126+
AsgClient: aws.GetAwsAsgClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
127+
EksClient: aws.GetAwsEksClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector),
128+
Ec2Metadata: metadata,
127129
}
128130

129131
metrics.Registry.MustRegister(cacheCollector, controllerCollector)

0 commit comments

Comments
 (0)