Multi agent parallel testing in CI

azure-pipelines-PR.yml

@@ -306,21 +306,32 @@ stages:
             name: $(DncEngPublicBuildPool)
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 120
+          strategy:
+            parallel: 4
           steps:
           - checkout: self
             clean: true
 
-          - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release
+          - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release -job $(System.JobPositionInPhase)
             env:
               DOTNET_DbgEnableMiniDump: 1
               DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing.
               DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp
               NativeToolsOnMachine: true
             displayName: Build
 
+          - task: PublishTestResults@2
+            displayName: Publish Test Results
+            inputs:
+              testResultsFormat: 'XUnit'
+              testRunTitle: WindowsNoRealsig_testDesktop $(System.JobPositionInPhase)
+              mergeTestResults: true
+              testResultsFiles: '*.xml'
+              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
+            continueOnError: true
           - task: PublishBuildArtifacts@1
             displayName: Publish Build BinLog
-            condition: always()
+            condition: eq(variables['System.JobPositionInPhase'], 1)
             continueOnError: true
             inputs:
                 PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog'
@@ -333,7 +344,7 @@ stages:
             continueOnError: true
             inputs:
                 PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release'
-                ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps'
+                ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps $(System.JobPositionInPhase)'
                 ArtifactType: Container
                 parallel: true
 
@@ -429,11 +440,8 @@ stages:
             demands: ImageOverride -equals $(WindowsMachineQueueName)
           timeoutInMinutes: 120
           strategy:
-            maxParallel: 5
+            maxParallel: 4
             matrix:
-              desktop_release:
-                _configuration: Release
-                _testKind: testDesktop
               coreclr_release:
                 _configuration: Release
                 _testKind: testCoreclr
@@ -537,6 +545,84 @@ stages:
             continueOnError: true
             condition: failed()
 
+          # Windows With Compressed Metadata Desktop
+        - job: WindowsCompressedMetadata_Desktop
+          variables:
+          - name: XUNIT_LOGS
+            value: $(Build.SourcesDirectory)\artifacts\TestResults\Release
+          - name: __VSNeverShowWhatsNew
+            value: 1
+          pool:
+            # The PR build definition sets this variable:
+            #   WindowsMachineQueueName=Windows.vs2022.amd64.open
+            # and there is an alternate build definition that sets this to a queue that is always scouting the
+            # next preview of Visual Studio.
+            name: $(DncEngPublicBuildPool)
+            demands: ImageOverride -equals $(WindowsMachineQueueName)
+          timeoutInMinutes: 120
+          strategy:
+            parallel: 4
+
+          steps:
+          - checkout: self
+            clean: true
+
+          - script: eng\CIBuild.cmd -compressallmetadata -configuration Release -testDesktop -job $(System.JobPositionInPhase)
+            env:
+              DOTNET_DbgEnableMiniDump: 1
+              DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing.
+              DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp
+              NativeToolsOnMachine: true
+            displayName: Build / Test
+
+          - task: PublishTestResults@2
+            displayName: Publish Test Results
+            inputs:
+              testResultsFormat: 'XUnit'
+              testRunTitle: WindowsCompressedMetadata testDesktop $(System.JobPositionInPhase)
+              mergeTestResults: true
+              testResultsFiles: '*.xml'
+              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
+            continueOnError: true
+
+          - task: PublishBuildArtifacts@1
+            displayName: Publish Tests BinLog
+            condition: eq(variables['System.JobPositionInPhase'], 1)
+            continueOnError: true
+            inputs:
+                PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog'
+                ArtifactName: 'Windows testDesktop test binlogs'
+                ArtifactType: Container
+                parallel: true
+          - task: PublishBuildArtifacts@1
+            displayName: Publish Dumps
+            condition: failed()
+            continueOnError: true
+            inputs:
+                PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release'
+                ArtifactName: 'Windows testDesktop process dumps $(System.JobPositionInPhase)'
+                ArtifactType: Container
+                parallel: true
+          - task: PublishBuildArtifacts@1
+            displayName: Publish Test Logs
+            inputs:
+              PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release'
+              ArtifactName: 'Windows testDesktop test logs $(System.JobPositionInPhase)'
+              publishLocation: Container
+            continueOnError: true
+            condition: always()
+          - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj
+            displayName: Dump NuGet cache contents
+            condition: failed()
+          - task: PublishBuildArtifacts@1
+            displayName: Publish NuGet cache contents
+            inputs:
+              PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents'
+              ArtifactName: 'NuGetPackageContents Windows testDesktop $(System.JobPositionInPhase)'
+              publishLocation: Container
+            continueOnError: true
+            condition: failed()
+
         # Mock official build
         - job: MockOfficial
           pool:
@@ -556,16 +642,18 @@ stages:
           variables:
           - name: _SignType
             value: Test
+          strategy:
+            parallel: 4
           steps:
           - checkout: self
             clean: true
-          - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr
-            displayName: Build / Test
+          - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr -job $(System.JobPositionInPhase)
+            displayName: Build / Test -job $(System.JobPositionInPhase)
           - task: PublishTestResults@2
             displayName: Publish Test Results
             inputs:
               testResultsFormat: 'XUnit'
-              testRunTitle: Linux
+              testRunTitle: Linux -job $(System.JobPositionInPhase)
               testResultsFiles: '*.xml'
               mergeTestResults: true
               searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
@@ -575,7 +663,7 @@ stages:
             displayName: Publish Test Logs
             inputs:
               PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-              ArtifactName: 'Linux $(_BuildConfig) test logs'
+              ArtifactName: 'Linux $(_BuildConfig) test logs -job $(System.JobPositionInPhase)'
               publishLocation: Container
             continueOnError: true
             condition: failed()
@@ -586,7 +674,7 @@ stages:
             displayName: Publish NuGet cache contents
             inputs:
               PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents'
-              ArtifactName: 'NuGetPackageContents Linux'
+              ArtifactName: 'NuGetPackageContents Linux -job $(System.JobPositionInPhase)'
               publishLocation: Container
             continueOnError: true
             condition: failed()

eng/Build.ps1

@@ -73,6 +73,7 @@ param (
     [switch]$compressAllMetadata,
     [switch]$buildnorealsig = $true,
     [switch]$verifypackageshipstatus = $false,
+    [string]$job = "",
     [parameter(ValueFromRemainingArguments = $true)][string[]]$properties)
 
 Set-StrictMode -version 2.0
@@ -357,6 +358,10 @@ function VerifyAssemblyVersionsAndSymbols() {
 }
 
 function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false, [string] $settings = "") {
+    if ($job) {
+        Write-Host("Testing batch: $job")
+        $filter = ""
+    }
     $dotnetPath = InitializeDotNetCli
     $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
     $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
@@ -376,6 +381,9 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     }
 
     $args += " $settings"
+    if ($job) {
+        $args += " --filter batch=$job"
+    }
 
     if ($asBackgroundJob) {
         Write-Host
@@ -604,24 +612,10 @@ try {
     }
 
     if ($testCoreClr) {
-        $cpuLimit = if ($ci) { "-m:1 -- xUnit.MaxParallelThreads=1" } else { "" }
-        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit
-    }
-
-    if ($testDesktop -and $ci) {
-        $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
-
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Test.Utilities\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework  -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
-
-        Receive -job $bgJob
+        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
     }
 
-    if ($testDesktop -and -not $ci ) {
+    if ($testDesktop) {
         TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
     }
 

eng/build.sh

@@ -76,6 +76,7 @@ prepare_machine=false
 source_build=false
 product_build=false
 buildnorealsig=true
+job=""
 properties=""
 
 docker=false
@@ -104,6 +105,11 @@ while [[ $# > 0 ]]; do
       args="$args $1"
       shift
       ;;
+    --job|-job)
+      job=$2
+      args="$args $1"
+      shift
+      ;;
     --verbosity|-v)
       verbosity=$2
       args="$args $1"
@@ -226,7 +232,10 @@ function Test() {
   projectname="${projectname%.*}"
   testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
   args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
-  args+=" -- xUnit.MaxParallelThreads=1"
+  if [[ "$job" != "" ]]; then
+    args="$args --filter batch=$job"
+  fi
+
   "$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
 }
 

tests/FSharp.Test.Utilities/XunitHelpers.fs

@@ -90,6 +90,14 @@ module TestCaseCustomizations =
         else
             testCase.TestMethod
 
+    let sha = Security.Cryptography.SHA256.Create()
+
+    let addBatchTrait (testCase: ITestCase) =
+        let data = Text.Encoding.UTF8.GetBytes testCase.DisplayName
+        let hashCode = BitConverter.ToUInt32(sha.ComputeHash(data), 0)
+        let batch = hashCode % 4u + 1u
+        testCase.Traits.Add("batch", ResizeArray [ string batch ])
+
 type CustomTestCase =
     inherit XunitTestCase
     // xUinit demands this constructor for deserialization.
@@ -109,6 +117,7 @@ type CustomTestCase =
     override testCase.Initialize () =
         base.Initialize()
         testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase
+        TestCaseCustomizations.addBatchTrait testCase
 
 type CustomTheoryTestCase =
     inherit XunitTheoryTestCase
@@ -127,6 +136,7 @@ type CustomTheoryTestCase =
     override testCase.Initialize () =
         base.Initialize()
         testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase
+        TestCaseCustomizations.addBatchTrait testCase
 
 #endif