Skip to content

disable leaked eni cleanup routine when vpc-resource-controller is deployed #2854

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
orsenthil merged 3 commits into from
Jun 13, 2024
Merged

disable leaked eni cleanup routine when vpc-resource-controller is deployed #2854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef9f1cf
disable leaked ENI cleanup routine when vpc-resource-controller is de…
sushrk Jun 4, 2024
4f1db93
update helm version
sushrk Jun 5, 2024
84884c2
Merge branch 'master' into disable-cleanup
Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@ require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
github.com/vishvananda/netlink v1.2.1-beta.2
go.uber.org/zap v1.27.0
go.uber.org/zap v1.26.0
golang.org/x/net v0.24.0
golang.org/x/sys v0.19.0
google.golang.org/grpc v1.62.0
google.golang.org/protobuf v1.33.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v2 v2.4.0
helm.sh/helm/v3 v3.14.3
k8s.io/api v0.30.1
k8s.io/apimachinery v0.30.1
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
k8s.io/cli-runtime v0.29.0
k8s.io/client-go v0.29.3
sigs.k8s.io/controller-runtime v0.17.0
Expand Down
12 changes: 6 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,8 +460,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -608,12 +608,12 @@ helm.sh/helm/v3 v3.14.3 h1:HmvRJlwyyt9HjgmAuxHbHv3PhMz9ir/XNWHyXfmnOP4=
helm.sh/helm/v3 v3.14.3/go.mod h1:v6myVbyseSBJTzhmeE39UcPLNv6cQK6qss3dvgAySaE=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY=
k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM=
k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0=
k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc=
k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U=
k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o=
k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM=
k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4=
Expand Down
45 changes: 30 additions & 15 deletions pkg/awsutils/awsutils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore"

"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/awssession"
"github.com/aws/amazon-vpc-cni-k8s/pkg/config"
"github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper"
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder"
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
Expand All @@ -54,11 +55,11 @@ const (

// AllocENI need to choose a first free device number between 0 and maxENI
// 100 is a hard limit because we use vlanID + 100 for pod networking table names
maxENIs = 100
clusterNameEnvVar = "CLUSTER_NAME"
eniNodeTagKey = "node.k8s.amazonaws.com/instance_id"
eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt"
eniClusterTagKey = "cluster.k8s.amazonaws.com/name"
maxENIs = 100

// ENI tags
eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt"

additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS"
reservedTagKeyPrefix = "k8s.amazonaws.com"
subnetDiscoveryTagKey = "kubernetes.io/role/cni"
Expand Down Expand Up @@ -213,6 +214,8 @@ type EC2InstanceMetadataCache struct {
enablePrefixDelegation bool

clusterName string
clusterNameEnvVal string
nodeName string
additionalENITags map[string]string

imds TypedIMDS
Expand Down Expand Up @@ -353,15 +356,17 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string)
}

// New creates an EC2InstanceMetadataCache
func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool) (*EC2InstanceMetadataCache, error) {
func New(useSubnetDiscovery, useCustomNetworking, disableLeakedENICleanup, v4Enabled, v6Enabled bool, clusterName, nodeName string) (*EC2InstanceMetadataCache, error) {
// ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run
ctx := context.Background()

sess := awssession.New()
ec2Metadata := ec2metadata.New(sess)
cache := &EC2InstanceMetadataCache{}
cache.imds = TypedIMDS{instrumentedIMDS{ec2Metadata}}
cache.clusterName = os.Getenv(clusterNameEnvVar)
cache.clusterName = clusterName
cache.clusterNameEnvVal = os.Getenv(config.ClusterNameEnv)
cache.nodeName = nodeName
cache.additionalENITags = loadAdditionalENITags()

region, err := ec2Metadata.Region()
Expand Down Expand Up @@ -982,14 +987,24 @@ func (cache *EC2InstanceMetadataCache) tryCreateNetworkInterface(input *ec2.Crea
// buildENITags computes the desired AWS Tags for eni
func (cache *EC2InstanceMetadataCache) buildENITags() map[string]string {
tags := map[string]string{
eniNodeTagKey: cache.instanceID,
// TODO: deprecate instance ID tag to replace with nodename to align with tag used in vpc-resource-controller
config.ENIInstanceIDTag: cache.instanceID,
}

// If clusterName is provided,
// tag the ENI with "cluster.k8s.amazonaws.com/name=<cluster_name>"
// clusterName is set from CNINode created by vpc-resource-controller, add the new tags only when it is set so controller can deleted leaked ENIs
// If it is not set then likely the controller is not running, so skip
if cache.clusterName != "" {
tags[eniClusterTagKey] = cache.clusterName
tags[fmt.Sprintf(config.ClusterNameTagKeyFormat, cache.clusterName)] = config.ClusterNameTagValue
tags[config.ENINodeNameTagKey] = cache.nodeName
tags[config.ENIOwnerTagKey] = config.ENIOwnerTagValue
}

if cache.clusterNameEnvVal != "" {
// TODO: deprecate this tag to replace with "kubernetes.io/cluster/<cluster-name>:owned" to align with tag used in vpc-resource-controller
// for backward compatibily, add tag if CLUSTER_NAME ENV is set
tags[config.ClusterNameTagKey] = cache.clusterNameEnvVal
}

for key, value := range cache.additionalENITags {
tags[key] = value
}
Expand Down Expand Up @@ -1877,7 +1892,7 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]*ec2.NetworkInterface,
{
Name: aws.String("tag-key"),
Values: []*string{
aws.String(eniNodeTagKey),
aws.String(config.ENIInstanceIDTag),
},
},
{
Expand All @@ -1893,11 +1908,11 @@ func (cache *EC2InstanceMetadataCache) getLeakedENIs() ([]*ec2.NetworkInterface,
},
},
}
if cache.clusterName != "" {
if cache.clusterNameEnvVal != "" {
leakedENIFilters = append(leakedENIFilters, &ec2.Filter{
Name: aws.String(fmt.Sprintf("tag:%s", eniClusterTagKey)),
Name: aws.String(fmt.Sprintf("tag:%s", config.ClusterNameTagKey)),
Values: []*string{
aws.String(cache.clusterName),
aws.String(cache.clusterNameEnvVal),
},
})
}
Expand Down
Loading