Parallelize tests (#17872)

* rebase

* orchestrate instead of Async.Sleep

* xunit customizations behind conditional compilation

* make console capture explicit

* diff

* try fix test

* lazy pattern

* reenable extras

* test temptath

* cleaup test dir

* remove unneeded file

* fix merge

* another shot at fixing gc test

* another attempt at improving Async.Parallel test

* smaller diff

* restore test

* restore tests

* diff

* unblock service tests

* rename attributes

* fix

* revert - should now work without

* add comment

* diff

* diff

Co-authored-by: Petr <[email protected]>

* just exclude gc test

* diff

* exclude test case

* try to trigger clr error in ci

* try unflake test

* unflake asyncmemo

* ilverify

* fix merge

* some cleanup

* Update tests/fsharp/Compiler/CodeGen/EmittedIL/DeterministicTests.fs

Co-authored-by: Petr <[email protected]>

* remove timeout

* Make FsiTests sequential

* restore ci throttling

* min diff

* add comment

* bring down the parallelizm a notch on mac and linux

---------

Co-authored-by: Petr <[email protected]>
Co-authored-by: Vlad Zarytovskii <[email protected]>
Co-authored-by: Kevin Ransom (msft) <[email protected]>

Author: 4 people

eng/Build.ps1

@@ -362,14 +362,16 @@ function VerifyAssemblyVersionsAndSymbols() {
     }
 }
 
-function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false) {
+function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false, [string] $settings = "") {
     $dotnetPath = InitializeDotNetCli
     $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
     $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
-    $testLogPath = "$ArtifactsDir\TestResults\$configuration\${projectName}_$targetFramework.xml"
+    # {assembly} and {framework} will expand respectively. See https://github.com/spekt/testlogger/wiki/Logger-Configuration#logfilepath
+    # This is useful to deconflict log filenames when there are many test assemblies, e.g. when testing a whole solution.
+    $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}.xml"
     $testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog"
     $args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
-    $args += " --blame --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=false"
+    $args += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=true"
 
     if (-not $noVisualStudio -or $norestore) {
         $args += " --no-restore"
@@ -379,17 +381,20 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
         $args += " --no-build"
     }
 
+    $args += " $settings"
+
     if ($asBackgroundJob) {
+        Write-Host
         Write-Host("Starting on the background: $args")
         Write-Host("------------------------------------")
-        $bgJob = Start-Job -ScriptBlock {
-            & $using:dotnetExe test $using:testProject -c $using:configuration -f $using:targetFramework -v n --test-adapter-path $using:testadapterpath --logger "xunit;LogFilePath=$using:testLogPath" /bl:$using:testBinLogPath  --blame --results-directory $using:ArtifactsDir\TestResults\$using:configuration
+        Start-Job -ScriptBlock {
+            $argArray = $using:args -Split " "
+            & $using:dotnetExe $argArray
             if ($LASTEXITCODE -ne 0) {
                 throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args"
             }
         }
-        return $bgJob
-    } else{
+    } else {
         Write-Host("$args")
         Exec-Console $dotnetExe $args
     }
@@ -595,21 +600,20 @@ try {
     $script:BuildCategory = "Test"
     $script:BuildMessage = "Failure running tests"
 
-    if ($testCoreClr) {
-        $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
-
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
+    function Receive($job) {
+        while($job.HasMoreData) {
+            Receive-Job $job | Write-Host
+            Start-Sleep -Seconds 1
+        }
+        Receive-Job $job -Wait -ErrorAction Stop
+    }
 
-        # Collect output from  background jobs
-        Wait-job $bgJob | out-null
-        Receive-Job $bgJob -ErrorAction Stop
+    if ($testCoreClr) {
+        $cpuLimit = if ($ci) { "-m:2 -- xUnit.MaxParallelThreads=0.25x" } else { "" }
+        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit
     }
 
-    if ($testDesktop) {
+    if ($testDesktop -and $ci) {
         $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
 
         TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
@@ -618,9 +622,11 @@ try {
         TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
         TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
 
-        # Collect output from  background jobs
-        Wait-job $bgJob | out-null
-        Receive-Job $bgJob -ErrorAction Stop
+        Receive -job $bgJob
+    }
+
+    if ($testDesktop -and -not $ci ) {
+        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
     }
 
     if ($testFSharpQA) {

eng/build.sh

@@ -219,7 +219,8 @@ function Test() {
   projectname=$(basename -- "$testproject")
   projectname="${projectname%.*}"
   testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
-  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args+=" -- xUnit.MaxParallelThreads=3"
   "$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
 }
 

tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj

@@ -4,7 +4,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
-    <TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
     <OutputType>Library</OutputType>
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
     <UnitTestType>xunit</UnitTestType>
@@ -18,6 +18,12 @@
     <Compile Include="MapSourceRootsTests.fs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content> 
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />

tests/FSharp.Build.UnitTests/xunit.runner.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeAssembly": true
+}

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs

@@ -7,6 +7,7 @@ open FSharp.Test
 open FSharp.Test.Compiler
 open System
 
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module utf8output =
 
     [<Fact>]

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs

@@ -8,6 +8,8 @@ open FSharp.Test.Compiler
 open System
 open System.IO
 
+// reportTime uses global state.
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module times =
 
     // This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times)

tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs

@@ -28,7 +28,7 @@ let internal observe (cache: AsyncMemoize<_,_,_>) =
 
     cache.Event.Add arrivals.Post
 
-    let next () = collected.Receive(10_000)
+    let next () = collected.Receive()
 
     next
 

tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs

@@ -14,5 +14,5 @@ let success =
     asm.Version.Major = 1 &&
     asm.Version.Minor = 2 &&
     asm.Version.Build = 3 &&
-    (abs (asm.Version.Revision - (int defaultRevision))) < 10  // default value is seconds in the current day / 2.  Check if within 10 sec of that.
+    (abs (asm.Version.Revision - (int defaultRevision))) < 60  // default value is seconds in the current day / 2.  Check if within 60 sec of that.
 if success then () else failwith "Failed: 1"

tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs

@@ -6,6 +6,7 @@ open Xunit
 open FSharp.Test
 open FSharp.Test.Compiler
 
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module Events =
 
     let verifyCompile compilation =

tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs

@@ -1,4 +1,5 @@
-module FSharpChecker.TransparentCompiler
+[<Xunit.Collection(nameof FSharp.Test.NotThreadSafeResourceCollection)>]
+module FSharpChecker.TransparentCompiler
 
 open System.Collections.Concurrent
 open System.Diagnostics
@@ -991,6 +992,8 @@ type private LoadClosureTestShim(currentFileSystem: IFileSystem) =
                 ?shouldShadowCopy = shouldShadowCopy
             )
 
+// Because it is mutating FileSystem!
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module TestsMutatingFileSystem =
 
     [<Theory>]