diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml index bfc4bb6cfacb..013c5429d5f9 100644 --- a/.azure/pipelines/ci-public.yml +++ b/.azure/pipelines/ci-public.yml @@ -129,6 +129,7 @@ stages: -ci -prepareMachine -nativeToolsOnMachine + -CentralizedRestore -arch x64 -pack -all @@ -570,7 +571,7 @@ stages: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -CentralizedRestore -nobl -all -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) /p:VsTestUseMSBuildOutput=false displayName: Build shared fx @@ -599,7 +600,7 @@ stages: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -CentralizedRestore -nobl -all -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) /p:VsTestUseMSBuildOutput=false displayName: Build shared fx diff --git a/.azure/pipelines/ci-unofficial.yml b/.azure/pipelines/ci-unofficial.yml index eb6e293c07e3..1177befe2d3c 100644 --- a/.azure/pipelines/ci-unofficial.yml +++ b/.azure/pipelines/ci-unofficial.yml @@ -148,6 +148,7 @@ extends: -ci -prepareMachine -nativeToolsOnMachine + -CentralizedRestore -arch x64 -pack -all @@ -608,7 +609,7 @@ extends: displayName: Update submodules steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -CentralizedRestore -nobl -all -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) env: MSBUILDUSESERVER: "1" @@ -640,7 +641,7 @@ extends: displayName: Update submodules steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -CentralizedRestore -nobl -all -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) env: MSBUILDUSESERVER: "1" diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index 8d65fcf1c396..2a23c8a80634 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -169,6 +169,7 @@ extends: -ci -prepareMachine -nativeToolsOnMachine + -CentralizedRestore -arch x64 -pack -all @@ -600,7 +601,7 @@ extends: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -CentralizedRestore -nobl -all -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) env: MSBUILDUSESERVER: "1" @@ -629,7 +630,7 @@ extends: timeoutInMinutes: 240 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -nobl -all -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -CentralizedRestore -nobl -all -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) env: MSBUILDUSESERVER: "1" diff --git a/.azure/pipelines/quarantined-pr.yml b/.azure/pipelines/quarantined-pr.yml index 3ee1c1a7eb6f..9d822255990b 100644 --- a/.azure/pipelines/quarantined-pr.yml +++ b/.azure/pipelines/quarantined-pr.yml @@ -57,7 +57,7 @@ jobs: timeoutInMinutes: 150 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -noBuildJava -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -CentralizedRestore -nobl -all -noBuildJava -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Build shared fx # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. @@ -84,7 +84,7 @@ jobs: isAzDOTestingJob: true enablePublishTestResults: false steps: - - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -noBuildJava -pack + - script: ./eng/build.cmd -ci -prepareMachine -CentralizedRestore -nobl -all -noBuildJava -pack displayName: Build - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -noBuildRepoTasks -noBuildNative -NoBuild -noBuildJava -test /p:RunQuarantinedTests=true /p:SkipHelixReadyTests=true diff --git a/.azure/pipelines/quarantined-tests.yml b/.azure/pipelines/quarantined-tests.yml index 9bcc91c3c439..435290dd1b8f 100644 --- a/.azure/pipelines/quarantined-tests.yml +++ b/.azure/pipelines/quarantined-tests.yml @@ -27,7 +27,7 @@ jobs: timeoutInMinutes: 480 steps: # Build the shared framework - - script: ./eng/build.cmd -ci -prepareMachine -nobl -all -noBuildJava -pack -arch x64 + - script: ./eng/build.cmd -ci -prepareMachine -CentralizedRestore -nobl -all -noBuildJava -pack -arch x64 /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log displayName: Build shared fx # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. diff --git a/eng/CentralizedRestore.proj b/eng/CentralizedRestore.proj new file mode 100644 index 000000000000..43a11265a966 --- /dev/null +++ b/eng/CentralizedRestore.proj @@ -0,0 +1,73 @@ + + + + + + + $([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', '..')) + + + + + + + + + + + <_DelayedProjectsToRestore Include="@(RequiresDelayedBuild)" + Condition="'%(Extension)' != '.nodeproj' AND '%(Extension)' != '.proj'" /> + + + + + + + + + + + + + + + + + + + <_DelayedProjectsToRestore Include="@(RequiresDelayedBuild)" + Condition="'%(Extension)' != '.nodeproj' AND '%(Extension)' != '.proj'" /> + + + + + + + + + diff --git a/eng/build.ps1 b/eng/build.ps1 index 854fe0679036..625b8550963c 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -206,6 +206,10 @@ param( [Alias('pb')] [switch]$ProductBuild, + # Use centralized restore to avoid NuGet parallel restore race conditions (NuGet/Home#7648) + # When set, performs a single upfront restore then builds without restore + [switch]$CentralizedRestore, + # Intentionally lowercase as tools.ps1 depends on it [switch]$fromVMR, @@ -271,7 +275,9 @@ $RunBuild = if ($NoBuild) { $false } else { $true } # Run restore by default unless -NoRestore is set. # -NoBuild implies -NoRestore, unless -Restore is explicitly set (as in restore.cmd) +# -CentralizedRestore performs a single upfront restore then disables per-project restore $RunRestore = if ($NoRestore) { $false } + elseif ($CentralizedRestore) { $false } # Will do centralized restore separately elseif ($Restore) { $true } elseif ($NoBuild) { $false } else { $true } @@ -484,6 +490,50 @@ try { $global:VerbosePreference = 'Continue' } + # Perform centralized restore if requested - this restores everything upfront in a single + # coordinated operation to avoid NuGet race conditions (https://github.com/NuGet/Home/issues/7648) + if ($CentralizedRestore) { + Write-Host + Write-Host "===== Centralized Restore =====" -ForegroundColor Cyan + Write-Host "Restoring all projects upfront to avoid NuGet parallel restore race conditions" -ForegroundColor Cyan + + # Create artifacts package directories so projects that reference them as RestoreAdditionalProjectSources + # don't fail restore (e.g., InteropWebsite.csproj references ArtifactsShippingPackagesDir) + $packagesDir = "$RepoRoot\artifacts\packages\$Configuration" + New-Item -ItemType Directory -Force -Path "$packagesDir\Shipping" | Out-Null + New-Item -ItemType Directory -Force -Path "$packagesDir\NonShipping" | Out-Null + + # Restore via the solution file with --disable-parallel to avoid NuGet internal race conditions + # NuGet's RestoreTask is not thread-safe when same project is restored in parallel + Write-Host "Restoring AspNetCore.slnx (sequential)..." -ForegroundColor Yellow + & dotnet restore "$RepoRoot\AspNetCore.slnx" --disable-parallel --verbosity $Verbosity + if ($LASTEXITCODE -ne 0) { + Write-Error "Solution restore failed" + exit $LASTEXITCODE + } + + # Generate files that other projects depend on (normally done AfterTargets="Restore" in Tools.props) + # Must be done BEFORE any other project evaluation since they import these generated files + Write-Host "Generating build files..." -ForegroundColor Yellow + & dotnet restore "$RepoRoot\eng\tools\GenerateFiles\GenerateFiles.csproj" --verbosity $Verbosity + if ($LASTEXITCODE -ne 0) { + Write-Error "GenerateFiles restore failed" + exit $LASTEXITCODE + } + & dotnet msbuild "$RepoRoot\eng\tools\GenerateFiles\GenerateFiles.csproj" /t:GenerateDirectoryBuildFiles /p:Configuration=$Configuration /verbosity:$Verbosity + if ($LASTEXITCODE -ne 0) { + Write-Error "GenerateFiles build failed" + exit $LASTEXITCODE + } + + # Note: Delayed build projects (RequiresDelayedBuild) are NOT restored here because they + # require the targeting pack to be built first. + # They will be restored later by BuildAfterTargetingPack.csproj with sequential restore. + + Write-Host "===== Centralized Restore Complete =====" -ForegroundColor Cyan + Write-Host + } + if (-not $NoBuildRepoTasks) { Write-Host diff --git a/eng/targets/FunctionalTestWithAssets.targets b/eng/targets/FunctionalTestWithAssets.targets index 99fbcbe78d0f..08de401b9731 100644 --- a/eng/targets/FunctionalTestWithAssets.targets +++ b/eng/targets/FunctionalTestWithAssets.targets @@ -37,8 +37,13 @@ false true $(IsUnitTestProject) + + <_RestoreInParallel Condition="'$(CentralizedRestore)' != 'true'">true + <_RestoreInParallel Condition="'$(CentralizedRestore)' == 'true'">false @@ -45,7 +52,7 @@ Condition=" '$(DotNetBuildSourceOnly)' != 'true' AND ('$(TargetOsName)' == 'win' AND '$(TargetArchitecture)' == 'x64') " Returns="@(TargetPathWithTargetPlatformMoniker)">