Skip to content

Commit 0af9e78

Browse files
Youssef1313Youssef1313
authored
Revert to MTP and disable server capability as a workaround (#8833)
* Revert "Disable testing platform protocol mode in Workspace (#8825)" This reverts commit 9a59c69. * Reapply "Migrate from VSTest to Microsoft.Testing.Platform (#8498)" (#8802) This reverts commit d2b3016. * Add xunit.runner.visualstudio * Temporary workaround
1 parent e2556ad commit 0af9e78

28 files changed

+112
-323
lines changed

‎.github/workflows/run-tests.yml‎

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ on:
1313
# relative the repo root
1414
required: false
1515
type: string
16-
testSessionTimeoutMs:
16+
testSessionTimeout:
1717
required: false
1818
type: string
19-
default: "900000"
19+
default: "15m"
2020
testHangTimeout:
2121
required: false
2222
type: string
@@ -220,18 +220,14 @@ jobs:
220220
TEST_LOG_PATH: ${{ github.workspace }}/artifacts/log/test-logs
221221
TestsRunningOutsideOfRepo: true
222222
run: >
223-
dotnet test -s .runsettings -v:n ${{ env.TEST_ASSEMBLY_NAME }}.dll
224-
-l "console;verbosity=normal"
225-
-l "trx;LogFilePrefix=${{ inputs.testShortName }}"
226-
-l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
227-
--blame
228-
--blame-hang-timeout ${{ inputs.testHangTimeout }}
229-
--blame-crash
223+
dotnet ${{ env.TEST_ASSEMBLY_NAME }}.dll
224+
--report-trx --report-trx-filename "${{ inputs.testShortName }}.trx"
225+
--hangdump --hangdump-timeout ${{ inputs.testHangTimeout }}
226+
--crashdump
230227
--results-directory ${{ github.workspace }}/testresults
228+
--filter-not-trait "category=failing"
229+
--timeout ${{ inputs.testSessionTimeout }}
231230
${{ inputs.extraTestArgs }}
232-
--
233-
RunConfiguration.CollectSourceInformation=true
234-
RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }}
235231
236232
- name: Run tests
237233
if: ${{ ! inputs.requiresNugets }}
@@ -240,24 +236,24 @@ jobs:
240236
CI: false
241237
DCP_DIAGNOSTICS_LOG_LEVEL: debug
242238
DCP_DIAGNOSTICS_LOG_FOLDER: ${{ github.workspace }}/testresults/dcp
239+
# During restore and build, we use -ci, which causes NUGET_PACKAGES to point to a local cache (Arcade behavior).
240+
# In this step, we are not using Arcade, but want to make sure that MSBuild is able to evaluate correctly.
241+
# So, we manually set NUGET_PACKAGES
242+
NUGET_PACKAGES: ${{ github.workspace }}/.packages
243243
run: >
244244
${{ env.DOTNET_SCRIPT }} test ${{ env.TEST_PROJECT_PATH }}
245245
/p:ContinuousIntegrationBuild=true
246-
-s eng/testing/.runsettings
247-
-l "console;verbosity=normal"
248-
-l "trx;LogFilePrefix=${{ inputs.testShortName }}"
249-
-l "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true"
250-
"--blame"
251-
--blame-hang-timeout ${{ inputs.testHangTimeout }}
252-
--blame-crash
253-
--results-directory testresults
246+
/p:TrxFileNamePrefix="${{ inputs.testShortName }}"
247+
-bl:${{ github.workspace }}/testresults/test.binlog
254248
--no-restore
255249
--no-build
256-
-bl:${{ github.workspace }}/testresults/test.binlog
257-
${{ inputs.extraTestArgs }}
258250
--
259-
RunConfiguration.CollectSourceInformation=true
260-
RunConfiguration.TestSessionTimeout=${{ inputs.testSessionTimeoutMs }}
251+
--report-trx
252+
--hangdump --hangdump-timeout ${{ inputs.testHangTimeout }}
253+
--crashdump
254+
--results-directory ${{ github.workspace }}/testresults
255+
--timeout ${{ inputs.testSessionTimeout }}
256+
${{ inputs.extraTestArgs }}
261257
262258
# Save the result of the previous steps - success or failure
263259
# in the form of a file result-success/result-failure -{name}.rst

‎.github/workflows/tests.yml‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ jobs:
7777
with:
7878
testShortName: ${{ matrix.shortname }}
7979
os: "ubuntu-latest"
80-
extraTestArgs: "--filter \"(quarantined!=true)\""
80+
extraTestArgs: "--filter-not-trait \"quarantined=true\""
8181

8282
integrations_test_win:
8383
uses: ./.github/workflows/run-tests.yml
@@ -90,7 +90,7 @@ jobs:
9090
with:
9191
testShortName: ${{ matrix.shortname }}
9292
os: "windows-latest"
93-
extraTestArgs: "--filter \"(quarantined!=true)\""
93+
extraTestArgs: "--filter-not-trait \"quarantined=true\""
9494

9595
templates_test_lin:
9696
name: Templates Linux
@@ -103,10 +103,9 @@ jobs:
103103
testShortName: ${{ matrix.shortname }}
104104
os: "ubuntu-latest"
105105
testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
106-
testSessionTimeoutMs: 1200000
106+
testSessionTimeout: 20m
107107
testHangTimeout: 12m
108-
# append '.' to the name so only the test class with exactly that name is run
109-
extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\""
108+
extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
110109
requiresNugets: true
111110
requiresTestSdk: true
112111

@@ -121,10 +120,9 @@ jobs:
121120
testShortName: ${{ matrix.shortname }}
122121
os: "windows-latest"
123122
testProjectPath: tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
124-
testSessionTimeoutMs: 1200000
123+
testSessionTimeout: 20m
125124
testHangTimeout: 12m
126-
# append '.' to the name so only the test class with exactly that name is run
127-
extraTestArgs: "--filter \"(quarantined!=true)&(FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}.)\""
125+
extraTestArgs: "--filter-not-trait quarantined=true --filter-class Aspire.Templates.Tests.${{ matrix.shortname }}"
128126
requiresNugets: true
129127
requiresTestSdk: true
130128

‎eng/.runsettings‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets‎

Lines changed: 15 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,13 @@
5858
<PropertyGroup>
5959
<_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment>
6060
<_TestAssembly>%(TestToRun.Identity)</_TestAssembly>
61+
<_TestAssembly Condition="'$(OS)'=='Windows_NT'">$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe'))</_TestAssembly>
62+
<_TestAssembly Condition="'$(OS)'!='Windows_NT'">$(_TestAssembly.TrimEnd('.dll'))</_TestAssembly>
6163
<_TestTimeout>%(TestToRun.TestTimeout)</_TestTimeout>
62-
<_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments>
6364

6465
<_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\</_TargetDir>
6566

66-
<_TestRunnerCommand>&quot;$(DotNetTool)&quot; test $(_TestAssembly) --list-tests --logger:"console%3Bverbosity=normal" "--Framework:%(TestToRun.TargetFrameworkIdentifier),Version=%(TestToRun.TargetFrameworkVersion)"</_TestRunnerCommand>
67-
<_TestRunnerCommand Condition="'$(VSTestRunSettingsFile)' != ''">$(_TestRunnerCommand) "--settings:$(VSTestRunSettingsFile)"</_TestRunnerCommand>
68-
<_TestRunnerCommand Condition="'$(_TestRunnerAdditionalArguments)' != ''">$(_TestRunnerCommand) $(_TestRunnerAdditionalArguments)</_TestRunnerCommand>
67+
<_TestRunnerCommand>$(_TestAssembly) --list-tests $(_QuarantinedTestRunAdditionalArgs)</_TestRunnerCommand>
6968

7069
<!--
7170
Redirect std output of the runner.
@@ -92,53 +91,27 @@
9291
WorkingDirectory="$(_TargetDir)"
9392
IgnoreExitCode="true"
9493
Timeout="$(_TestTimeout)"
94+
EnvironmentVariables="DOTNET_ROOT=$(DotNetRoot);DOTNET_ROOT_X86=$(DotNetRoot)x86"
9595
ContinueOnError="WarnAndContinue">
9696
<Output TaskParameter="ExitCode" PropertyName="_TestErrorCode" />
9797
</Exec>
9898

99-
<!--
100-
Report test status.
101-
-->
102-
<Message Text="Search complete: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />
103-
10499
<PropertyGroup>
105100
<_ResultsFileToDisplay>%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay>
106101
</PropertyGroup>
107102

108103
<!--
109-
Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project
110-
we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961.
111-
-->
112-
<Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' != '0'" File="QuarantinedTestRunsheetBuilder" />
113-
114-
<!--
115-
Read the test result and see if the string "No test matches the given testcase filter" is present.
116-
This indicates that there are no quarantined tests to run.
104+
Report test status.
117105
-->
118-
<ReadLinesFromFile File="$(_ResultsFileToDisplay)">
119-
<Output TaskParameter="Lines" ItemName="FileLines" />
120-
</ReadLinesFromFile>
121-
122-
<ItemGroup>
123-
<Lines Include="@(FileLines)" >
124-
<Sanitized>$([System.String]::Copy('%(FileLines.Identity)').Replace('`', ''))</Sanitized>
125-
</Lines>
126-
127-
<_UnhandledExceptionLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Unhandled exception`)) " />
128-
<_ExceptionDiscoveringTestsLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Exception discovering tests`)) " />
129-
<_NoTestMatchesLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`No test matches the given testcase filter`)) " />
130-
</ItemGroup>
106+
<Message Text="Search complete, no quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '8'" Importance="high" />
107+
<Message Text="💡 Search complete, quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" />
108+
<Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition=" '$(_TestErrorCode)' != '0' and '$(_TestErrorCode)' != '8' " File="QuarantinedTestRunsheetBuilder" />
131109

132110
<PropertyGroup>
133-
<_FailedToEnumerateTests Condition=" '@(_UnhandledExceptionLines)' != '' or '@(_ExceptionDiscoveringTestsLines)' != '' ">true</_FailedToEnumerateTests>
134-
135111
<_HasQuarantinedTests>true</_HasQuarantinedTests>
136-
<_HasQuarantinedTests Condition=" '@(_NoTestMatchesLines)' != '' ">false</_HasQuarantinedTests>
112+
<_HasQuarantinedTests Condition=" '$(_TestErrorCode)' == '8' ">false</_HasQuarantinedTests>
137113
</PropertyGroup>
138114

139-
<Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_FailedToEnumerateTests)' == 'true'" File="QuarantinedTestRunsheetBuilder" />
140-
<Message Text="💡 Quarantined tests found in $(_TestAssembly) [$(_TestEnvironment)]" Condition=" '$(_HasQuarantinedTests)' == 'true' " Importance="high" />
141-
142115
<!--
143116
Generate test runsheet, if there are quarantined tests.
144117
-->
@@ -160,6 +133,12 @@
160133
<_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux>
161134
<_TestCommand>-restore -build -test -projects &quot;$(_RelativeTestProjectPath)&quot; /bl:&quot;$(_RelativeTestBinLog)&quot; -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false</_TestCommand>
162135

136+
<!--
137+
Some quarantinted test may only be executable on Windows or Linux, however we can't possibly know that at this time.
138+
The MTP runner will return exit code 8 if no tests are found, and we need to ignore it instead of failing the test.
139+
-->
140+
<_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true</_TestCommand>
141+
163142
<!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
164143
<_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>
165144

‎eng/Testing.props‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,20 @@
99
<RunOnAzdoCILinux>true</RunOnAzdoCILinux>
1010
<RunOnAzdoHelixWindows>true</RunOnAzdoHelixWindows>
1111
<RunOnAzdoHelixLinux>true</RunOnAzdoHelixLinux>
12-
</PropertyGroup>
13-
14-
<PropertyGroup Condition="'$(UseVSTestRunner)' != 'true'">
15-
<_QuarantinedTestRunAdditionalArgs>-trait "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
16-
<_NonQuarantinedTestRunAdditionalArgs>-notrait "quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
17-
</PropertyGroup>
1812

19-
<PropertyGroup Condition="'$(UseVSTestRunner)' == 'true'">
20-
<_QuarantinedTestRunAdditionalArgs>--filter "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
21-
<_NonQuarantinedTestRunAdditionalArgs>--filter "quarantined!=true"</_NonQuarantinedTestRunAdditionalArgs>
13+
<_QuarantinedTestRunAdditionalArgs>--filter-trait "quarantined=true"</_QuarantinedTestRunAdditionalArgs>
14+
<_NonQuarantinedTestRunAdditionalArgs>--filter-not-trait "quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
2215
</PropertyGroup>
2316

2417
<PropertyGroup>
2518
<BlameHangTimeout Condition="'$(BlameHangTimeout)' == '' and '$(MSBuildProjectName)' == 'Aspire.Hosting.SqlServer.Tests'">20m</BlameHangTimeout>
2619
<BlameHangTimeout Condition="'$(BlameHangTimeout)' == ''">10m</BlameHangTimeout>
27-
<_BlameArgs>--blame --blame-hang-timeout $(BlameHangTimeout) --blame-crash</_BlameArgs>
20+
<_BlameArgs>--hangdump --hangdump-timeout $(BlameHangTimeout) --crashdump</_BlameArgs>
21+
22+
<TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --filter-not-trait &quot;category=failing&quot;</TestRunnerAdditionalArguments>
23+
<TestRunnerAdditionalArguments Condition=" '$(IgnoreZeroTestResult)' == 'true' ">$(TestRunnerAdditionalArguments) --ignore-exit-code 8</TestRunnerAdditionalArguments>
2824

29-
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == ''">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs)</TestRunnerAdditionalArguments>
30-
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments) $(_BlameArgs)</TestRunnerAdditionalArguments>
25+
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' != 'true'">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
26+
<TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
3127
</PropertyGroup>
3228
</Project>

‎eng/Versions.props‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@
1212
<DotNetRuntimePreviousVersionForTesting>8.0.13</DotNetRuntimePreviousVersionForTesting>
1313
<!-- dotnet 8.0 versions for running tests - used for templates tests -->
1414
<DotNetSdkPreviousVersionForTesting>8.0.406</DotNetSdkPreviousVersionForTesting>
15-
<UseVSTestRunner>true</UseVSTestRunner>
1615
<XunitV3Version>2.0.0</XunitV3Version>
1716
<XUnitAnalyzersVersion>1.20.0</XUnitAnalyzersVersion>
1817
<XunitRunnerVisualStudioVersion>3.0.2</XunitRunnerVisualStudioVersion>
18+
<MicrosoftTestingPlatformVersion>1.6.3</MicrosoftTestingPlatformVersion>
19+
<MicrosoftNETTestSdkVersion>17.13.0</MicrosoftNETTestSdkVersion>
1920
<!-- Enable to remove prerelease label. -->
2021
<StabilizePackageVersion Condition="'$(StabilizePackageVersion)' == ''">false</StabilizePackageVersion>
2122
<DotNetFinalVersionKind Condition="'$(StabilizePackageVersion)' == 'true'">release</DotNetFinalVersionKind>

‎eng/Xunit3/Xunit3.targets‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,25 @@
44
<PackageVersion Include="xunit.v3.core" Version="$(XunitV3Version)" />
55
<PackageVersion Include="xunit.analyzers" Version="$(XunitAnalyzersVersion)" />
66
<PackageVersion Include="xunit.v3.assert" Version="$(XunitV3Version)" />
7-
<PackageVersion Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioVersion)" />
87
<PackageVersion Include="xunit.v3.runner.console" Version="$(XunitV3Version)" />
8+
<PackageVersion Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioVersion)" />
9+
<PackageVersion Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformVersion)" />
10+
<PackageVersion Include="Microsoft.Testing.Platform.MSBuild" Version="$(MicrosoftTestingPlatformVersion)" />
11+
<PackageVersion Include="Microsoft.Testing.Extensions.HangDump" Version="$(MicrosoftTestingPlatformVersion)" />
12+
<PackageVersion Include="Microsoft.Testing.Extensions.CrashDump" Version="$(MicrosoftTestingPlatformVersion)" />
13+
<PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="$(MicrosoftTestingPlatformVersion)" />
914

1015
<PackageReference Include="xunit.v3" />
1116
<PackageReference Include="xunit.v3.core" />
1217
<PackageReference Include="xunit.analyzers" />
1318
<PackageReference Include="xunit.v3.assert" />
1419
<PackageReference Include="xunit.runner.visualstudio" />
20+
<PackageReference Include="Microsoft.Testing.Platform" />
21+
<PackageReference Include="Microsoft.Testing.Platform.MSBuild" />
22+
<PackageReference Include="Microsoft.Testing.Extensions.HangDump" />
23+
<PackageReference Include="Microsoft.Testing.Extensions.CrashDump" />
24+
<PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
1525
</ItemGroup>
1626

17-
<Import Project="..\tools\VSTest.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
18-
</Project>
27+
<Import Project="Microsoft.Testing.Platform.targets" />
28+
</Project>

‎eng/testing/.runsettings‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎tests/Aspire.Components.Common.Tests/Aspire.Components.Common.Tests.csproj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@
99
<IsTestUtilityProject>true</IsTestUtilityProject>
1010
<SkipTests>true</SkipTests>
1111
<DeployOutsideOfRepoSupportFiles>false</DeployOutsideOfRepoSupportFiles>
12+
13+
<!-- https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-exit-codes -->
14+
<!-- Exit code 8 is "zero tests ran" -->
15+
<!-- Currently, none of the tests in this project run in CI. All are ignored -->
16+
<TestingPlatformCommandLineArguments>$(TestingPlatformCommandLineArguments) --ignore-exit-code 8</TestingPlatformCommandLineArguments>
1217
</PropertyGroup>
1318

1419
<ItemGroup>

0 commit comments

Comments
 (0)