Parallelise mutation tests (#2395)

- Run each mutation test project in parallel in its own GitHub Actions job.
- Install the .NET 8 SDK when required now we only explicitly install .NET 6 and 9.
- Minimise redundant work when running mutation tests.

Author: martincostello

.github/workflows/build.yml

@@ -62,6 +62,7 @@ jobs:
       with:
         dotnet-version: |
           6.0.x
+          8.0.x
 
     - name: Setup .NET SDK
       uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2.0

.github/workflows/mutation-tests.yml

@@ -31,8 +31,14 @@ jobs:
         include:
           - name: core
             target: Core
+          - name: extensions
+            target: Extensions
           - name: legacy
             target: Legacy
+          - name: ratelimiting
+            target: RateLimiting
+          - name: testing
+            target: Testing
 
     steps:
 
@@ -46,26 +52,37 @@ jobs:
       with:
         dotnet-version: |
           6.0.x
+          8.0.x
 
     - name: Setup .NET SDK
       uses: actions/setup-dotnet@87b7050bc53ea08284295505d98d2aa94301e852 # v4.2.0
 
-    - name: Setup NuGet cache
-      uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
-      with:
-        path: ~/.nuget/packages
-        key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props') }}
-        restore-keys: ${{ runner.os }}-nuget-
-
-    - name: Run mutation tests
+    - name: Run mutation tests for ${{ matrix.target }}
       shell: pwsh
       env:
-        MUTATION_TARGET: 'Mutation${{ matrix.target }}'
+        MUTATION_TARGET: 'MutationTests${{ matrix.target }}'
+        SKIP_POLLY_ANALYZERS: 'true'
       run: ./build.ps1 -Target ${env:MUTATION_TARGET}
 
     - name: Upload Mutation Report
-      if: always()
+      if: ${{ !cancelled() }}
       uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
       with:
         name: mutation-report-${{ matrix.name }}
         path: ./artifacts/mutation-report
+
+  mutation-tests:
+    needs: [ mutations ]
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    env:
+      MUTATIONS_SUCCESS: ${{ !contains(needs.*.result, 'failure') }}
+    steps:
+      - run: |
+          if [ "$MUTATIONS_SUCCESS" == "true" ]
+          then
+            echo 'Mutation tests successful ✅'
+          else
+            echo 'One or more mutation test runs failed ❌'
+            exit 1
+          fi

build.cake

@@ -159,21 +159,6 @@ Task("__RunTests")
     }
 });
 
-Task("__RunCoreMutationTests")
-    .Does((context) =>
-{
-    MutationTestProject(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj"), "Polly.Core.csproj");
-    MutationTestProject(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"), "Polly.RateLimiting.csproj");
-    MutationTestProject(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"), "Polly.Extensions.csproj");
-    MutationTestProject(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj"), "Polly.Testing.csproj");
-});
-
-Task("__RunLegacyMutationTests")
-    .Does((context) =>
-{
-    MutationTestProject(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj"), "Polly.csproj");
-});
-
 Task("__CreateNuGetPackages")
     .Does(() =>
 {
@@ -215,9 +200,12 @@ Task("__ValidateDocs")
     }
 });
 
-Task("__CommonBuild")
+Task("__Setup")
     .IsDependentOn("__Clean")
-    .IsDependentOn("__RestoreNuGetPackages")
+    .IsDependentOn("__RestoreNuGetPackages");
+
+Task("__CommonBuild")
+    .IsDependentOn("__Setup")
     .IsDependentOn("__ValidateDocs")
     .IsDependentOn("__BuildSolutions");
 
@@ -238,13 +226,51 @@ Task("Build")
 Task("Default")
     .IsDependentOn("Build");
 
-Task("MutationCore")
-    .IsDependentOn("__CommonBuild")
-    .IsDependentOn("__RunCoreMutationTests");
+///////////////////////////////////////////////////////////////////////////////
+// MUTATION TESTING TARGETS
+///////////////////////////////////////////////////////////////////////////////
 
-Task("MutationLegacy")
-    .IsDependentOn("__CommonBuild")
-    .IsDependentOn("__RunLegacyMutationTests");
+Task("MutationTestsCore")
+    .IsDependentOn("__Setup")
+    .Does((context) =>
+{
+    RunMutationTests(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj"));
+});
+
+Task("MutationTestsRateLimiting")
+    .IsDependentOn("__Setup")
+    .Does((context) =>
+{
+    RunMutationTests(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj"));
+});
+
+Task("MutationTestsExtensions")
+    .IsDependentOn("__Setup")
+    .Does((context) =>
+{
+    RunMutationTests(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj"));
+});
+
+Task("MutationTestsTesting")
+    .IsDependentOn("__Setup")
+    .Does((context) =>
+{
+    RunMutationTests(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj"));
+});
+
+Task("MutationTestsLegacy")
+    .IsDependentOn("__Setup")
+    .Does((context) =>
+{
+    RunMutationTests(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj"));
+});
+
+Task("MutationTests")
+    .IsDependentOn("MutationTestsCore")
+    .IsDependentOn("MutationTestsRateLimiting")
+    .IsDependentOn("MutationTestsExtensions")
+    .IsDependentOn("MutationTestsTesting")
+    .IsDependentOn("MutationTestsLegacy");
 
 ///////////////////////////////////////////////////////////////////////////////
 // EXECUTION
@@ -261,28 +287,20 @@ string ToolsExePath(string exeFileName) {
     return exePath;
 }
 
-void MutationTestProject(FilePath proj, FilePath testProj, string project)
+void RunMutationTests(FilePath target, FilePath testProject)
 {
-    var dotNetBuildSettings = new DotNetBuildSettings
-    {
-        Configuration = "Debug",
-        Verbosity = DotNetVerbosity.Minimal,
-        NoRestore = true
-    };
-
-    DotNetBuild(proj.ToString(), dotNetBuildSettings);
-
     var strykerPath = Context.Tools.Resolve("Stryker.CLI.dll");
-    var mutationScore = XmlPeek(proj, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true });
+    var mutationScore = XmlPeek(target, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true });
     var score = int.Parse(mutationScore);
+    var targetFileName = target.GetFilename();
 
-    Information($"Running mutation tests for '{proj}'. Test Project: '{testProj}'");
+    Information($"Running mutation tests for '{targetFileName}'. Test Project: '{testProject}'");
 
-    var args = $"{strykerPath} --project {project} --test-project {testProj.FullPath} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{project}";
+    var args = $"{strykerPath} --project {targetFileName} --test-project {testProject.FullPath} --break-at {score} --config-file {strykerConfig} --output {strykerOutput}/{targetFileName}";
 
     var result = StartProcess("dotnet", args);
     if (result != 0)
     {
-        throw new InvalidOperationException($"The mutation testing of '{project}' project failed.");
+        throw new InvalidOperationException($"The mutation testing of '{targetFileName}' project failed.");
     }
 }