Skip to content

Commit cd02f0d

Browse files
fseldowCopilot
fseldow
and
Copilot
authored
feat: download azure acr credential provider via oras in network isolated windows cluster (#8152)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 280c8f4 commit cd02f0d

2 files changed

Lines changed: 126 additions & 17 deletions

File tree

‎staging/cse/windows/configfunc.ps1‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -435,9 +435,8 @@ function Install-CredentialProvider {
435435
[Parameter(Mandatory = $false)][string]
436436
$CustomCloudContainerRegistryDNSSuffix
437437
)
438-
439438
try {
440-
# Out of tree credential provider is turned on as a must after 1.30, and is optinal in 1.29, for cluster < 1.29, it's not enabled.
439+
# Out of tree credential provider is turned on as a must after 1.30, and is optional in 1.29, for cluster < 1.29, it's not enabled.
441440
# And only when it's enabled, the credential provider flags are set.
442441
$global:credentialProviderConfigPath = ""
443442
$global:credentialProviderBinDir = ""
@@ -454,12 +453,45 @@ function Install-CredentialProvider {
454453

455454
Write-Log "Download credential provider binary from $global:CredentialProviderURL to $global:credentialProviderBinDir"
456455
$tempDir = New-TemporaryDirectory
457-
$credentialproviderbinaryPackage = "$tempDir\credentialprovider.tar.gz"
458-
DownloadFileOverHttp -Url $global:CredentialProviderURL -DestinationPath $credentialproviderbinaryPackage -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER
459-
tar -xzf $credentialproviderbinaryPackage -C $tempDir
460-
if ($LASTEXITCODE -ne 0) {
461-
throw "Failed to extract the '$credentialproviderbinaryPackage' archive."
456+
if ([string]::IsNullOrEmpty($global:BootstrapProfileContainerRegistryServer)) {
457+
$credentialproviderbinaryPackage = "$tempDir\credentialprovider.tar.gz"
458+
DownloadFileOverHttp -Url $global:CredentialProviderURL -DestinationPath $credentialproviderbinaryPackage -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER
459+
tar -xzf $credentialproviderbinaryPackage -C $tempDir
460+
if ($LASTEXITCODE -ne 0) {
461+
throw "Failed to extract the '$credentialproviderbinaryPackage' archive."
462+
}
463+
} else {
464+
# network isolated cluster
465+
# download azure acr credential provider binaries via oras if BootstrapProfileContainerRegistryServer is set
466+
if (-not (Get-Command 'DownloadFileWithOras' -ErrorAction SilentlyContinue)) {
467+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER -ErrorMessage "DownloadFileWithOras function is not available. networkisolatedclusterfunc.ps1 may not be sourced."
468+
}
469+
$credentialproviderbinaryPackage = "$tempDir\credentialprovider.zip"
470+
# Handle both URL styles:
471+
# 1) .../azure-acr-credential-provider/1.34.0/windows/... for dalec url, e.g. https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.34.0/windows/amd64/azure-acr-credential-provider_1.34.0-1_amd64.zip
472+
# 2) .../cloud-provider-azure/v1.34.0/binaries/... for legacy binaries url, e.g. https://packages.aks.azure.com/cloud-provider-azure/v1.34.0/binaries/azure-acr-credential-provider-linux-amd64-v1.34.0.tar.gz
473+
# If version is missing in URL, fall back to KubeBinariesVersion.
474+
$packageVersion = ([regex]::Match($global:CredentialProviderURL, '/v?(\d+(?:\.\d+)+)(?=/)')).Groups[1].Value
475+
if ([string]::IsNullOrEmpty($packageVersion)) {
476+
$packageVersion = ([regex]::Match($global:CredentialProviderURL, '(?:^|[._-])v?(\d+(?:\.\d+)+)(?:$|[._-])')).Groups[1].Value
477+
}
478+
if ([string]::IsNullOrEmpty($packageVersion)) {
479+
Write-Warning "Unexpected CredentialProviderURL format, version is not found in URL. CredentialProviderURL: $global:CredentialProviderURL. Fall back to KubeBinariesVersion: $global:KubeBinariesVersion"
480+
$packageVersion = $global:KubeBinariesVersion
481+
$packageVersion = $packageVersion.TrimStart('v')
482+
}
483+
Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadCredentialProviderBinariesWithOras" -TaskMessage "Start to download azure acr credential provider binaries with oras. KubeBinariesVersion: $global:KubeBinariesVersion, BootstrapProfileContainerRegistryServer: $global:BootstrapProfileContainerRegistryServer"
484+
$orasReference = "$($global:BootstrapProfileContainerRegistryServer)/aks/packages/kubernetes/azure-acr-credential-provider:v$($packageVersion)"
485+
try {
486+
Retry-Command -Command "DownloadFileWithOras" -Args @{Reference = $orasReference; DestinationPath = $credentialproviderbinaryPackage } -Retries 5 -RetryDelaySeconds 10
487+
}
488+
catch {
489+
del $tempDir -Recurse -Force -ErrorAction SilentlyContinue
490+
Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER -ErrorMessage "Exhausted retries for oras pull $orasReference. Error: $_"
491+
}
492+
AKS-Expand-Archive -Path $credentialproviderbinaryPackage -DestinationPath $tempDir
462493
}
494+
463495
Create-Directory -FullPath $global:credentialProviderBinDir
464496
cp "$tempDir\azure-acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider.exe"
465497
# acr-credential-provider.exe cannot be found by kubelet through provider name before the fix https://github.com/kubernetes/kubernetes/pull/120291

‎staging/cse/windows/configfunc.tests.ps1‎

Lines changed: 87 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
BeforeAll {
22
. $PSScriptRoot\..\..\..\parts\windows\windowscsehelper.ps1
3+
. $PSScriptRoot\networkisolatedclusterfunc.ps1
34
. $PSCommandPath.Replace('.tests.ps1','.ps1')
45

56
$capturedContent = $null
6-
Mock Set-Content -MockWith {
7+
Mock Set-Content -MockWith {
78
param($Path, $Value)
8-
$script:capturedContent = $Value
9+
$script:capturedContent = $Value
910
} -Verifiable
1011

1112
Mock Remove-Item
@@ -67,7 +68,7 @@ Describe 'Resize-OSDrive' {
6768
Write-Host "Invoke-Executable $Executable $ArgList $ExitCode"
6869
} -Verifiable
6970
}
70-
71+
7172
Context 'success' {
7273
It "Should call Invoke-Executable to Diskpart once" {
7374
Mock Get-Disk -MockWith {
@@ -114,7 +115,7 @@ Describe 'Resize-OSDrive' {
114115
}
115116
}
116117

117-
Describe 'Config-CredentialProvider' {
118+
Describe 'Config-CredentialProvider' {
118119
BeforeEach {
119120
$global:credentialProviderConfigDir = "staging\cse\windows\credentialProvider.tests.suites"
120121
$CredentialProviderConfPATH=[Io.path]::Combine("$global:credentialProviderConfigDir", "credential-provider-config.yaml")
@@ -126,26 +127,26 @@ Describe 'Config-CredentialProvider' {
126127

127128
AfterEach {
128129
Remove-Item -Path $CredentialProviderConfPATH
129-
}
130+
}
130131

131132
Context 'CustomCloudContainerRegistryDNSSuffix is empty' {
132133
It "should match the expected config file content" {
133134
$expectedCredentialProviderConfig = Read-Format-Yaml ([Io.path]::Combine($credentialProviderConfigDir, "CustomCloudContainerRegistryDNSSuffixEmpty.config.yaml"))
134135
Config-CredentialProvider -KubeDir $credentialProviderConfigDir -CredentialProviderConfPath $CredentialProviderConfPATH -CustomCloudContainerRegistryDNSSuffix ""
135-
136+
136137
$acutalCredentialProviderConfig = Read-Format-Yaml $CredentialProviderConfPATH
137138
# Compare the content by normalizing whitespace and line endings
138139
$normalizedExpected = $expectedCredentialProviderConfig.Trim().Replace("`r`n", "`n")
139140
$normalizedActual = $acutalCredentialProviderConfig.Trim().Replace("`r`n", "`n")
140141
$normalizedActual | Should -Be $normalizedExpected
141142
}
142-
}
143+
}
143144
Context 'CustomCloudContainerRegistryDNSSuffix is not empty' {
144145
It "should match the expected config file content" {
145146
$expectedCredentialProviderConfig = Read-Format-Yaml ([Io.path]::Combine($credentialProviderConfigDir, "CustomCloudContainerRegistryDNSSuffixNotEmpty.config.yaml"))
146147
Config-CredentialProvider -KubeDir $credentialProviderConfigDir -CredentialProviderConfPath $CredentialProviderConfPATH -CustomCloudContainerRegistryDNSSuffix ".azurecr.microsoft.fakecloud"
147148
$acutalCredentialProviderConfig = Read-Format-Yaml $CredentialProviderConfPATH
148-
149+
149150
# Compare the content by normalizing whitespace and line endings
150151
$normalizedExpected = $expectedCredentialProviderConfig.Trim().Replace("`r`n", "`n")
151152
$normalizedActual = $acutalCredentialProviderConfig.Trim().Replace("`r`n", "`n")
@@ -170,7 +171,7 @@ Describe 'Validate-CredentialProviderConfigFlags' {
170171
Write-Host "Set-ExitCode $ExitCode $ErrorMessage"
171172
} -Verifiable
172173
}
173-
174+
174175
Context 'success' {
175176
It "Should return expected config path and bin path" {
176177
$expectedCredentialProviderConfigPath="c:\k\credential-provider-config.yaml"
@@ -211,4 +212,80 @@ Describe 'Validate-CredentialProviderConfigFlags' {
211212
Assert-MockCalled -CommandName "Set-ExitCode" -Exactly -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_CREDENTIAL_PROVIDER_CONFIG }
212213
}
213214
}
214-
}
215+
}
216+
217+
Describe 'Install-CredentialProvider' {
218+
BeforeEach {
219+
$global:credentialProviderConfigPath = ""
220+
$global:credentialProviderBinDir = ""
221+
$global:KubeletConfigArgs = @(
222+
"--image-credential-provider-config=c:\k\credential-provider-config.yaml",
223+
"--image-credential-provider-bin-dir=c:\var\lib\kubelet\credential-provider"
224+
)
225+
$global:CredentialProviderURL = "https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.34.0/windows/amd64/azure-acr-credential-provider_1.34.0-1_amd64.zip"
226+
$global:BootstrapProfileContainerRegistryServer = "myregistry.azurecr.io"
227+
$global:KubeBinariesVersion = "1.31.9"
228+
$script:lastDownloadReference = ""
229+
230+
Mock Config-CredentialProvider
231+
Mock New-TemporaryDirectory -MockWith { "C:\temp\credprovider" }
232+
Mock DownloadFileOverHttp
233+
Mock DownloadFileWithOras -MockWith {
234+
param(
235+
[string]$Reference,
236+
[string]$DestinationPath,
237+
[string]$Platform
238+
)
239+
$script:lastDownloadReference = $Reference
240+
}
241+
Mock AKS-Expand-Archive
242+
Mock Create-Directory
243+
Mock cp
244+
Mock del
245+
Mock tar -MockWith { $global:LASTEXITCODE = 0 }
246+
Mock Get-Command -MockWith {
247+
[pscustomobject]@{ Name = "DownloadFileWithOras" }
248+
} -ParameterFilter { $Name -eq 'DownloadFileWithOras' }
249+
Mock Set-ExitCode -MockWith {
250+
Param($ExitCode, $ErrorMessage)
251+
throw "Set-ExitCode:${ExitCode}:${ErrorMessage}"
252+
return
253+
}
254+
}
255+
AfterEach {
256+
$global:BootstrapProfileContainerRegistryServer = $null
257+
}
258+
259+
It 'returns early when out-of-tree credential provider flags are not configured' {
260+
$global:KubeletConfigArgs = @("--address=0.0.0.0")
261+
262+
{ Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' } | Should -Not -Throw
263+
Assert-MockCalled -CommandName 'Config-CredentialProvider' -Times 0
264+
$script:lastDownloadReference | Should -Be ""
265+
Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 0
266+
}
267+
268+
It 'uses legacy binaries URL for non-ni cluster' {
269+
$global:BootstrapProfileContainerRegistryServer = ""
270+
$global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.34.0/binaries/azure-acr-credential-provider-linux-amd64-v1.34.0.tar.gz'
271+
Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix ''
272+
Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1
273+
}
274+
275+
It 'uses version parsed from dalec URL for ORAS reference' {
276+
Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix ''
277+
$script:lastDownloadReference | Should -Be 'myregistry.azurecr.io/aks/packages/kubernetes/azure-acr-credential-provider:v1.34.0'
278+
}
279+
280+
It 'uses version parsed from legacy binaries URL for ORAS reference' {
281+
$global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.34.0/binaries/azure-acr-credential-provider-linux-amd64-v1.34.0.tar.gz'
282+
Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix ''
283+
$script:lastDownloadReference | Should -Be 'myregistry.azurecr.io/aks/packages/kubernetes/azure-acr-credential-provider:v1.34.0'
284+
}
285+
286+
It 'falls back to KubeBinariesVersion when URL contains no parseable version' {
287+
$global:CredentialProviderURL = 'https://packages.aks.azure.com/invalid/credential-provider.zip'
288+
Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix ''
289+
$script:lastDownloadReference | Should -Be 'myregistry.azurecr.io/aks/packages/kubernetes/azure-acr-credential-provider:v1.31.9'
290+
}
291+
}

0 commit comments

Comments
 (0)