diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 93aecf540c..b1aed568e3 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -3,9 +3,9 @@
-
+
https://github.com/dotnet/arcade
- 35bddd4fbfab8da3518fb920250d7c9e0c3138ff
+ 938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe
diff --git a/eng/Versions.props b/eng/Versions.props
index 35e2938735..b6860a7ff2 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -9,7 +9,8 @@
0.1.0
true
-
+
+
false
diff --git a/eng/common/build.sh b/eng/common/build.sh
index 252b63604e..55b298f16c 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -81,7 +81,7 @@ runtime_source_feed_key=''
properties=''
while [[ $# > 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)
usage
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index e7f12edb56..42516bbeeb 100644
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
for i in "$@"
do
- lowerI="$(echo $i | awk '{print tolower($0)}')"
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 6d59e181c8..ae0cd1aac7 100644
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -82,7 +82,7 @@ while :; do
break
fi
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index d981d7bbf3..39abdbecdc 100755
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
verbosity='minimal'
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--darcversion)
darcVersion=$2
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
index ead6a1d9a2..d6efeb4434 100644
--- a/eng/common/dotnet-install.sh
+++ b/eng/common/dotnet-install.sh
@@ -19,7 +19,7 @@ runtime='dotnet'
runtimeSourceFeed=''
runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-version|-v)
shift
@@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do
shift
done
-# Use uname to determine what the CPU is.
-cpuname=$(uname -p)
-# Some Linux platforms report unknown for platform, but the arch for machine.
-if [[ "$cpuname" == "unknown" ]]; then
- cpuname=$(uname -m)
-fi
-
+# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
+cpuname=$(uname -m)
case $cpuname in
aarch64)
buildarch=arm64
diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
index 29fc5db8ae..5bd205b5da 100644
--- a/eng/common/init-tools-native.sh
+++ b/eng/common/init-tools-native.sh
@@ -16,7 +16,7 @@ declare -A native_assets
. $scriptroot/native/common-library.sh
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -76,24 +76,89 @@ while (($# > 0)); do
done
function ReadGlobalJsonNativeTools {
- # Get the native-tools section from the global.json.
- local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
- # Only extract the contents of the object.
- local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
- native_tools_list=${native_tools_list//[\" ]/}
- native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
-
- local old_IFS=$IFS
- while read -r line; do
- # Lines are of the form: 'tool:version'
- IFS=:
- while read -r key value; do
- native_assets[$key]=$value
- done <<< "$line"
- done <<< "$native_tools_list"
- IFS=$old_IFS
-
- return 0;
+ # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
+ if command -v jq &> /dev/null; then
+
+ # jq: read each key/value pair under "native-tools" entry and emit:
+ # KEY="" VALUE=""
+ # followed by a null byte.
+ #
+ # bash: read line with null byte delimeter and push to array (for later `eval`uation).
+
+ while IFS= read -rd '' line; do
+ native_assets+=("$line")
+ done < <(jq -r '. |
+ select(has("native-tools")) |
+ ."native-tools" |
+ keys[] as $k |
+ @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
+
+ return
+ fi
+
+ # Warning: falling back to manually parsing JSON, which is not recommended.
+
+ # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
+ # It has been tested with several weird strings with escaped characters in entries (key and value)
+ # and results were compared with the output of jq(1) in binary representation using xxd(1);
+ # just before the assignment to 'native_assets' array (above and below).
+
+ # try to capture the section under "native-tools".
+ if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
+ return
+ fi
+
+ section="${BASH_REMATCH[1]}"
+
+ parseStarted=0
+ possibleEnd=0
+ escaping=0
+ escaped=0
+ isKey=1
+
+ for (( i=0; i<${#section}; i++ )); do
+ char="${section:$i:1}"
+ if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
+
+ if ! ((escaping)) && [[ "$char" == "\\" ]]; then
+ escaping=1
+ elif ((escaping)) && ! ((escaped)); then
+ escaped=1
+ fi
+
+ if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
+ parseStarted=1
+ possibleEnd=0
+ elif [[ "$char" == "'" ]]; then
+ token="$token'\\\''"
+ possibleEnd=0
+ elif ((escaping)) || [[ "$char" != "\"" ]]; then
+ token="$token$char"
+ possibleEnd=1
+ fi
+
+ if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
+ # Use printf to unescape token to match jq(1)'s @sh formatting rules.
+ # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
+ printf -v token "'$token'"
+
+ if ((isKey)); then
+ KEY="$token"
+ isKey=0
+ else
+ line="KEY=$KEY VALUE=$token"
+ native_assets+=("$line")
+ isKey=1
+ fi
+
+ # reset for next token
+ parseStarted=0
+ token=
+ elif ((escaping)) && ((escaped)); then
+ escaping=0
+ escaped=0
+ fi
+ done
}
native_base_dir=$install_directory
@@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
exit 0;
else
native_installer_dir="$scriptroot/native"
- for tool in "${!native_assets[@]}"
- do
- tool_version=${native_assets[$tool]}
- installer_path="$native_installer_dir/install-$tool.sh"
+ for index in "${!native_assets[@]}"; do
+ eval "${native_assets["$index"]}"
+
+ installer_path="$native_installer_dir/install-$KEY.sh"
installer_command="$installer_path"
installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin"
- installer_command+=" --version $tool_version"
+ installer_command+=" --version $VALUE"
echo $installer_command
if [[ $force = true ]]; then
diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1
index b8f6529fdc..418c09930c 100644
--- a/eng/common/internal-feed-operations.ps1
+++ b/eng/common/internal-feed-operations.ps1
@@ -63,8 +63,6 @@ function SetupCredProvider {
}
if (($endpoints | Measure-Object).Count -gt 0) {
- # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
- # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
$endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
# Create the environment variables the AzDo way
diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh
index 9ed225e7e5..e2233e7812 100644
--- a/eng/common/internal-feed-operations.sh
+++ b/eng/common/internal-feed-operations.sh
@@ -62,8 +62,6 @@ function SetupCredProvider {
endpoints+=']'
if [ ${#endpoints} -gt 2 ]; then
- # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
- # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
@@ -103,7 +101,7 @@ authToken=''
repoName=''
while [[ $# > 0 ]]; do
- opt="$(echo "$1" | awk '{print tolower($0)}')"
+ opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--operation)
operation=$2
diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh
index 8160cd5a59..20d3dad543 100755
--- a/eng/common/msbuild.sh
+++ b/eng/common/msbuild.sh
@@ -19,7 +19,7 @@ prepare_machine=false
extra_args=''
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--verbosity)
verbosity=$2
diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1
index d7d1a65109..adf707c8fe 100644
--- a/eng/common/native/CommonLibrary.psm1
+++ b/eng/common/native/CommonLibrary.psm1
@@ -48,7 +48,7 @@ function DownloadAndExtract {
-Verbose:$Verbose
if ($DownloadStatus -Eq $False) {
- Write-Error "Download failed"
+ Write-Error "Download failed from $Uri"
return $False
}
diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh
index 12339a4076..8a5e7cf0db 100644
--- a/eng/common/native/install-cmake-test.sh
+++ b/eng/common/native/install-cmake-test.sh
@@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -63,7 +63,7 @@ done
tool_name="cmake-test"
tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
-exit 0
\ No newline at end of file
+exit 0
diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh
index 18041be876..de496beebc 100644
--- a/eng/common/native/install-cmake.sh
+++ b/eng/common/native/install-cmake.sh
@@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@@ -63,7 +63,7 @@ done
tool_name="cmake"
tool_os=$(GetCurrentOS)
-tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
-exit 0
\ No newline at end of file
+exit 0
diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj
index 94b6efbc92..318ca5f1b8 100644
--- a/eng/common/performance/microbenchmarks.proj
+++ b/eng/common/performance/microbenchmarks.proj
@@ -141,4 +141,4 @@
4:00
-
\ No newline at end of file
+
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
index c8e211bcb1..7321feb5c7 100644
--- a/eng/common/performance/performance-setup.sh
+++ b/eng/common/performance/performance-setup.sh
@@ -29,7 +29,7 @@ using_wasm=false
use_latest_dotnet=false
while (($# > 0)); do
- lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--sourcedirectory)
source_directory=$2
diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1
index 8484451f3a..8e422c561e 100644
--- a/eng/common/pipeline-logging-functions.ps1
+++ b/eng/common/pipeline-logging-functions.ps1
@@ -29,14 +29,14 @@ function Write-PipelineTelemetryError {
[switch]$AsOutput,
[switch]$Force)
- $PSBoundParameters.Remove('Category') | Out-Null
+ $PSBoundParameters.Remove('Category') | Out-Null
- if($Force -Or ((Test-Path variable:ci) -And $ci)) {
- $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
- }
- $PSBoundParameters.Remove('Message') | Out-Null
- $PSBoundParameters.Add('Message', $Message)
- Write-PipelineTaskError @PSBoundParameters
+ if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
+ $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
+ }
+ $PSBoundParameters.Remove('Message') | Out-Null
+ $PSBoundParameters.Add('Message', $Message)
+ Write-PipelineTaskError @PSBoundParameters
}
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
@@ -55,8 +55,8 @@ function Write-PipelineTaskError {
[switch]$Force
)
- if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
- if($Type -eq 'error') {
+ if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
+ if ($Type -eq 'error') {
Write-Host $Message -ForegroundColor Red
return
}
@@ -66,47 +66,61 @@ function Write-PipelineTaskError {
}
}
- if(($Type -ne 'error') -and ($Type -ne 'warning')) {
+ if (($Type -ne 'error') -and ($Type -ne 'warning')) {
Write-Host $Message
return
}
$PSBoundParameters.Remove('Force') | Out-Null
- if(-not $PSBoundParameters.ContainsKey('Type')) {
+ if (-not $PSBoundParameters.ContainsKey('Type')) {
$PSBoundParameters.Add('Type', 'error')
}
Write-LogIssue @PSBoundParameters
- }
+}
- function Write-PipelineSetVariable {
+function Write-PipelineSetVariable {
[CmdletBinding()]
param(
- [Parameter(Mandatory = $true)]
- [string]$Name,
- [string]$Value,
- [switch]$Secret,
- [switch]$AsOutput,
- [bool]$IsMultiJobVariable=$true)
-
- if((Test-Path variable:ci) -And $ci) {
+ [Parameter(Mandatory = $true)]
+ [string]$Name,
+ [string]$Value,
+ [switch]$Secret,
+ [switch]$AsOutput,
+ [bool]$IsMultiJobVariable = $true)
+
+ if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
- 'variable' = $Name
- 'isSecret' = $Secret
- 'isOutput' = $IsMultiJobVariable
+ 'variable' = $Name
+ 'isSecret' = $Secret
+ 'isOutput' = $IsMultiJobVariable
} -AsOutput:$AsOutput
- }
- }
+ }
+}
- function Write-PipelinePrependPath {
+function Write-PipelinePrependPath {
[CmdletBinding()]
param(
- [Parameter(Mandatory=$true)]
- [string]$Path,
- [switch]$AsOutput)
+ [Parameter(Mandatory = $true)]
+ [string]$Path,
+ [switch]$AsOutput)
- if((Test-Path variable:ci) -And $ci) {
+ if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
- }
- }
+ }
+}
+
+function Write-PipelineSetResult {
+ [CmdletBinding()]
+ param(
+ [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
+ [Parameter(Mandatory = $true)]
+ [string]$Result,
+ [string]$Message)
+ if ((Test-Path variable:ci) -And $ci) {
+ Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
+ 'result' = $Result
+ }
+ }
+}
<########################################
# Private functions.
@@ -123,7 +137,8 @@ function Format-LoggingCommandData {
foreach ($mapping in $script:loggingCommandEscapeMappings) {
$Value = $Value.Replace($mapping.Token, $mapping.Replacement)
}
- } else {
+ }
+ else {
for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
$mapping = $script:loggingCommandEscapeMappings[$i]
$Value = $Value.Replace($mapping.Replacement, $mapping.Token)
@@ -156,7 +171,8 @@ function Format-LoggingCommand {
if ($first) {
$null = $sb.Append(' ')
$first = $false
- } else {
+ }
+ else {
$null = $sb.Append(';')
}
@@ -193,7 +209,8 @@ function Write-LoggingCommand {
$command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
if ($AsOutput) {
$command
- } else {
+ }
+ else {
Write-Host $command
}
}
@@ -212,12 +229,12 @@ function Write-LogIssue {
[switch]$AsOutput)
$command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
- 'type' = $Type
- 'code' = $ErrCode
- 'sourcepath' = $SourcePath
- 'linenumber' = $LineNumber
- 'columnnumber' = $ColumnNumber
- }
+ 'type' = $Type
+ 'code' = $ErrCode
+ 'sourcepath' = $SourcePath
+ 'linenumber' = $LineNumber
+ 'columnnumber' = $ColumnNumber
+ }
if ($AsOutput) {
return $command
}
@@ -229,7 +246,8 @@ function Write-LogIssue {
$foregroundColor = [System.ConsoleColor]::Red
$backgroundColor = [System.ConsoleColor]::Black
}
- } else {
+ }
+ else {
$foregroundColor = $host.PrivateData.WarningForegroundColor
$backgroundColor = $host.PrivateData.WarningBackgroundColor
if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {
diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh
index 6cd0a3400e..6a0b2255e9 100644
--- a/eng/common/pipeline-logging-functions.sh
+++ b/eng/common/pipeline-logging-functions.sh
@@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
local function_args=()
local message=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-category|-c)
telemetry_category=$2
@@ -48,7 +48,7 @@ function Write-PipelineTaskError {
local force=false
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-type|-t)
message_type=$2
@@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
local is_multi_job_variable=true
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-name|-n)
name=$2
@@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
local prepend_path=''
while [[ $# -gt 0 ]]; do
- opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-path|-p)
prepend_path=$2
@@ -179,4 +179,28 @@ function Write-PipelinePrependPath {
if [[ "$ci" == true ]]; then
echo "##vso[task.prependpath]$prepend_path"
fi
-}
\ No newline at end of file
+}
+
+function Write-PipelineSetResult {
+ local result=''
+ local message=''
+
+ while [[ $# -gt 0 ]]; do
+ opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+ case "$opt" in
+ -result|-r)
+ result=$2
+ shift
+ ;;
+ -message|-m)
+ message=$2
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ if [[ "$ci" == true ]]; then
+ echo "##vso[task.complete result=$result;]$message"
+ fi
+}
diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1
index 7d49744795..534f6988d5 100644
--- a/eng/common/post-build/post-build-utils.ps1
+++ b/eng/common/post-build/post-build-utils.ps1
@@ -69,9 +69,9 @@ function Trigger-Subscription([string]$SubscriptionId) {
function Validate-MaestroVars {
try {
- Get-Variable MaestroApiEndPoint -Scope Global | Out-Null
- Get-Variable MaestroApiVersion -Scope Global | Out-Null
- Get-Variable MaestroApiAccessToken -Scope Global | Out-Null
+ Get-Variable MaestroApiEndPoint | Out-Null
+ Get-Variable MaestroApiVersion | Out-Null
+ Get-Variable MaestroApiAccessToken | Out-Null
if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
index 31cf276741..599a117241 100644
--- a/eng/common/post-build/publish-using-darc.ps1
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -16,18 +16,18 @@ param(
try {
. $PSScriptRoot\post-build-utils.ps1
- # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
- $darc = Get-Darc "1.1.0-beta.20418.1"
+
+ $darc = Get-Darc
$optionalParams = [System.Collections.ArrayList]::new()
if ("" -ne $ArtifactsPublishingAdditionalParameters) {
- $optionalParams.Add("artifact-publishing-parameters") | Out-Null
+ $optionalParams.Add("--artifact-publishing-parameters") | Out-Null
$optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null
}
if ("" -ne $SymbolPublishingAdditionalParameters) {
- $optionalParams.Add("symbol-publishing-parameters") | Out-Null
+ $optionalParams.Add("--symbol-publishing-parameters") | Out-Null
$optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null
}
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
index a68bf0b88e..bb6a429711 100644
--- a/eng/common/sdl/init-sdl.ps1
+++ b/eng/common/sdl/init-sdl.ps1
@@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
-try {
- # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
- Write-Host 'Downloading gdn folder from internal config repostiory...'
- Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
- if (Test-Path $gdnFolder) {
- # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
- Remove-Item -Force -Recurse $gdnFolder
- }
- [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
- Write-Host $gdnFolder
- ExitWithExitCode 0
-} catch [System.Net.WebException] { } # Catch and ignore webexception
+
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host 'Initializing Guardian...'
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index 968b39bef5..3bd8b29ebd 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index c64c4f5686..53c100222b 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -83,7 +83,7 @@ jobs:
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1
- -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
+ -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index d0c3cc2b3b..66ef736417 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -37,6 +37,7 @@ jobs:
- name: _BuildConfig
value: ${{ parameters.configuration }}
- group: Publish-Build-Assets
+ - group: AzureDevOps-Artifact-Feeds-Pats
# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
@@ -57,6 +58,12 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0
+ - task: PowerShell@2
+ displayName: Enable cross-org NuGet feed authentication
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
+
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
index 9332f5ecc3..aad4146492 100644
--- a/eng/common/templates/job/source-build.yml
+++ b/eng/common/templates/job/source-build.yml
@@ -28,6 +28,11 @@ parameters:
# container and pool.
platform: {}
+ # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+ # source-build builds run in Docker, including the default managed platform.
+ defaultContainerHostPool:
+ vmImage: ubuntu-20.04
+
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
@@ -37,6 +42,9 @@ jobs:
${{ if ne(parameters.platform.container, '') }}:
container: ${{ parameters.platform.container }}
+
+ ${{ if eq(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.defaultContainerHostPool }}
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
new file mode 100644
index 0000000000..6126330745
--- /dev/null
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -0,0 +1,58 @@
+parameters:
+ runAsPublic: false
+ sourceIndexPackageVersion: 1.0.0-beta5
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+ sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+ preSteps: []
+ binlogPath: artifacts/log/Debug/Build.binlog
+ pool:
+ vmImage: vs2017-win2016
+
+jobs:
+- job: SourceIndexStage1
+ variables:
+ - name: SourceIndexPackageVersion
+ value: ${{ parameters.sourceIndexPackageVersion }}
+ - name: SourceIndexPackageSource
+ value: ${{ parameters.sourceIndexPackageSource }}
+ - name: BinlogPath
+ value: ${{ parameters.binlogPath }}
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - group: source-dot-net stage1 variables
+
+ pool: ${{ parameters.pool }}
+ steps:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk 3.1
+ inputs:
+ packageType: sdk
+ version: 3.1.x
+
+ - task: UseDotNet@2
+ displayName: Use .NET Core sdk
+ inputs:
+ useGlobalJson: true
+
+ - script: |
+ dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
+ echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
+ displayName: Download Tools
+
+ - script: ${{ parameters.sourceIndexBuildCommand }}
+ displayName: Build Repository
+
+ - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: Process Binlog into indexable sln
+ env:
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+ displayName: Upload stage1 artifacts to source index
+ env:
+ BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
+ DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index 08845950f4..a1f8fce96c 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -7,7 +7,14 @@ parameters:
# Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false
-
+
+ # Optional: Enable running the source-build jobs to build repo from source
+ enableSourceBuild: false
+
+ # Optional: Parameters for source-build template.
+ # See /eng/common/templates/jobs/source-build.yml for options
+ sourceBuildParameters: []
+
graphFileGeneration:
# Optional: Enable generating the graph files at the end of the build
enabled: false
@@ -24,12 +31,8 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
- # Optional: Enable running the source-build jobs to build repo from source
- runSourceBuild: false
-
- # Optional: Parameters for source-build template.
- # See /eng/common/templates/jobs/source-build.yml for options
- sourceBuildParameters: []
+ enableSourceIndex: false
+ sourceIndexParams: {}
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
@@ -50,14 +53,22 @@ jobs:
name: ${{ job.job }}
-- ${{ if eq(parameters.runSourceBuild, true) }}:
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
- template: /eng/common/templates/jobs/source-build.yml
parameters:
allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }}
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+ - template: ../job/source-index-stage1.yml
+ parameters:
+ runAsPublic: ${{ parameters.runAsPublic }}
+ ${{ each parameter in parameters.sourceIndexParams }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
parameters:
@@ -69,7 +80,7 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
- - ${{ if eq(parameters.runSourceBuild, true) }}:
+ - ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete
pool:
vmImage: vs2017-win2016
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
index f463011e79..00aa98eb3b 100644
--- a/eng/common/templates/jobs/source-build.yml
+++ b/eng/common/templates/jobs/source-build.yml
@@ -11,16 +11,14 @@ parameters:
# See /eng/common/templates/job/source-build.yml
jobNamePrefix: 'Source_Build'
- # If changed to true, causes this template to include the default platform for a managed-only
- # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
- # but since the repo is supposed to be managed-only, the risk should be very low.
- includeDefaultManagedPlatform: false
+ # This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
- # object in this array is sent to the job template as 'platform'.
+ # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+ # one job runs on 'defaultManagedPlatform'.
platforms: []
jobs:
@@ -32,7 +30,7 @@ jobs:
dependsOn:
- ${{ each platform in parameters.platforms }}:
- ${{ parameters.jobNamePrefix }}_${{ platform.name }}
- - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+ - ${{ if eq(length(parameters.platforms), 0) }}:
- ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
- ${{ each platform in parameters.platforms }}:
@@ -41,7 +39,7 @@ jobs:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
-- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 41f2d96a60..822ff5975b 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -61,7 +61,9 @@ parameters:
VS167ChannelId: 1011
VS168ChannelId: 1154
VSMasterChannelId: 1012
-
+ VS169ChannelId: 1473
+ VS1610ChannelId: 1692
+
stages:
- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- stage: Validate
@@ -90,7 +92,7 @@ stages:
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
- -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
+ -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
- job:
displayName: NuGet Validation
@@ -147,6 +149,9 @@ stages:
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@@ -224,7 +229,7 @@ stages:
- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
- stage: publish_using_darc
${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- dependsOn: Validate
+ dependsOn: ${{ parameters.publishDependsOn }}
${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc
@@ -549,3 +554,33 @@ stages:
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_9_Publishing'
+ channelName: 'VS 16.9'
+ channelId: ${{ parameters.VS169ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+
+ - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'VS_16_10_Publishing'
+ channelName: 'VS 16.10'
+ channelId: ${{ parameters.VS1610ChannelId }}
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index fc09059ffe..572da3b9f1 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -439,11 +439,26 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
Write-Host 'Downloading vswhere'
- try {
- Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
- }
- catch {
- Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ $maxRetries = 5
+ $retries = 1
+
+ while($true) {
+ try {
+ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
+ break
+ }
+ catch{
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+ }
+
+ if (++$retries -le $maxRetries) {
+ $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
+ Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
+ Start-Sleep -Seconds $delayInSeconds
+ }
+ else {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
+ }
}
}
@@ -493,7 +508,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' }
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -629,9 +644,26 @@ function MSBuild() {
}
$toolsetBuildProject = InitializeToolset
- $path = Split-Path -parent $toolsetBuildProject
- $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')
- $args += "/logger:$path"
+ $basePath = Split-Path -parent $toolsetBuildProject
+ $possiblePaths = @(
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
+ )
+ $selectedPath = $null
+ foreach ($path in $possiblePaths) {
+ if (Test-Path $path -PathType Leaf) {
+ $selectedPath = $path
+ break
+ }
+ }
+ if (-not $selectedPath) {
+ Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
+ ExitWithExitCode 1
+ }
+ $args += "/logger:$selectedPath"
}
MSBuild-Core @args
@@ -677,14 +709,23 @@ function MSBuild-Core() {
$exitCode = Exec-Process $buildTool.Path $cmdArgs
if ($exitCode -ne 0) {
- Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.'
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
$buildLog = GetMSBuildBinaryLogCommandLineArgument $args
- if ($buildLog -ne $null) {
+ if ($null -ne $buildLog) {
Write-Host "See log: $buildLog" -ForegroundColor DarkGray
}
- ExitWithExitCode $exitCode
+ if ($ci) {
+ Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ } else {
+ ExitWithExitCode $exitCode
+ }
}
}
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 98186e7849..9019b7f5cc 100644
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -89,16 +89,16 @@ function ResolvePath {
function ReadGlobalVersion {
local key=$1
- local line=$(awk "/$key/ {print; exit}" "$global_json_file")
- local pattern="\"$key\" *: *\"(.*)\""
+ if command -v jq &> /dev/null; then
+ _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
+ elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
+ _ReadGlobalVersion=${BASH_REMATCH[1]}
+ fi
- if [[ ! $line =~ $pattern ]]; then
+ if [[ -z "$_ReadGlobalVersion" ]]; then
Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1
fi
-
- # return value
- _ReadGlobalVersion=${BASH_REMATCH[1]}
}
function InitializeDotNetCli {
@@ -249,7 +249,7 @@ function with_retries {
return 0
fi
- timeout=$((2**$retries-1))
+ timeout=$((3**$retries-1))
echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
sleep $timeout
done
@@ -271,10 +271,19 @@ function GetDotNetInstallScript {
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
- with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
- local exit_code=$?
- Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
- ExitWithExitCode $exit_code
+ # first, try directly, if this fails we will retry with verbose logging
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ if command -v openssl &> /dev/null
+ then
+ echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
+ echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
+ fi
+ echo "Will now retry the same URL with verbose logging."
+ with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
}
else
with_retries wget -v -O "$install_script" "$install_script_url" || {
@@ -298,7 +307,7 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
- _InitializeBuildToolFramework="netcoreapp2.1"
+ _InitializeBuildToolFramework="netcoreapp3.1"
}
# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
@@ -405,8 +414,24 @@ function MSBuild {
fi
local toolset_dir="${_InitializeToolset%/*}"
- local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
- args=( "${args[@]}" "-logger:$logger_path" )
+ # new scripts need to work with old packages, so we need to look for the old names/versions
+ local selectedPath=
+ local possiblePaths=()
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
+ for path in "${possiblePaths[@]}"; do
+ if [[ -f $path ]]; then
+ selectedPath=$path
+ break
+ fi
+ done
+ if [[ -z "$selectedPath" ]]; then
+ Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
+ ExitWithExitCode 1
+ fi
+ args+=( "-logger:$selectedPath" )
fi
MSBuild-Core ${args[@]}
@@ -437,8 +462,17 @@ function MSBuild-Core {
"$_InitializeBuildTool" "$@" || {
local exit_code=$?
- Write-PipelineTaskError "Build failed (exit code '$exit_code')."
- ExitWithExitCode $exit_code
+ # We should not Write-PipelineTaskError here because that message shows up in the build summary
+ # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
+ echo "Build failed with exit code $exit_code. Check errors above."
+ if [[ "$ci" == "true" ]]; then
+ Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
+ # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
+ # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
+ ExitWithExitCode 0
+ else
+ ExitWithExitCode $exit_code
+ fi
}
}
@@ -461,8 +495,11 @@ temp_dir="$artifacts_dir/tmp/$configuration"
global_json_file="$repo_root/global.json"
# determine if global.json contains a "runtimes" entry
global_json_has_runtimes=false
-dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true
-if [[ -n "$dotnetlocal_key" ]]; then
+if command -v jq &> /dev/null; then
+ if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
+ global_json_has_runtimes=true
+ fi
+elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
global_json_has_runtimes=true
fi
diff --git a/global.json b/global.json
index 7705f434cc..8277123501 100644
--- a/global.json
+++ b/global.json
@@ -7,6 +7,6 @@
"xcopy-msbuild": "16.8.0-preview2.1"
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20573.2"
+ "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.12"
}
}