|
58 | 58 | <PropertyGroup> |
59 | 59 | <_TestEnvironment>%(TestToRun.EnvironmentDisplay)</_TestEnvironment> |
60 | 60 | <_TestAssembly>%(TestToRun.Identity)</_TestAssembly> |
61 | | - <_TestAssembly Condition="'$(OS)'=='Windows_NT'">$([System.IO.Path]::ChangeExtension($(_TestAssembly), '.exe'))</_TestAssembly> |
62 | | - <_TestAssembly Condition="'$(OS)'!='Windows_NT'">$(_TestAssembly.TrimEnd('.dll'))</_TestAssembly> |
63 | 61 | <_TestTimeout>%(TestToRun.TestTimeout)</_TestTimeout> |
| 62 | + <_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments)</_TestRunnerAdditionalArguments> |
64 | 63 |
|
65 | 64 | <_TargetDir>$([System.IO.Path]::GetDirectoryName('$(_TestAssembly)'))\</_TargetDir> |
66 | 65 |
|
67 | | - <_TestRunnerCommand>$(_TestAssembly) --list-tests $(_QuarantinedTestRunAdditionalArgs)</_TestRunnerCommand> |
| 66 | + <_TestRunnerCommand>"$(DotNetTool)" test $(_TestAssembly) --list-tests --logger:"console%3Bverbosity=normal" "--Framework:%(TestToRun.TargetFrameworkIdentifier),Version=%(TestToRun.TargetFrameworkVersion)"</_TestRunnerCommand> |
| 67 | + <_TestRunnerCommand Condition="'$(VSTestRunSettingsFile)' != ''">$(_TestRunnerCommand) "--settings:$(VSTestRunSettingsFile)"</_TestRunnerCommand> |
| 68 | + <_TestRunnerCommand Condition="'$(_TestRunnerAdditionalArguments)' != ''">$(_TestRunnerCommand) $(_TestRunnerAdditionalArguments)</_TestRunnerCommand> |
68 | 69 |
|
69 | 70 | <!-- |
70 | 71 | Redirect std output of the runner. |
|
91 | 92 | WorkingDirectory="$(_TargetDir)" |
92 | 93 | IgnoreExitCode="true" |
93 | 94 | Timeout="$(_TestTimeout)" |
94 | | - EnvironmentVariables="DOTNET_ROOT=$(DotNetRoot);DOTNET_ROOT_X86=$(DotNetRoot)x86" |
95 | 95 | ContinueOnError="WarnAndContinue"> |
96 | 96 | <Output TaskParameter="ExitCode" PropertyName="_TestErrorCode" /> |
97 | 97 | </Exec> |
98 | 98 |
|
| 99 | + <!-- |
| 100 | + Report test status. |
| 101 | + --> |
| 102 | + <Message Text="Search complete: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" /> |
| 103 | + |
99 | 104 | <PropertyGroup> |
100 | 105 | <_ResultsFileToDisplay>%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay> |
101 | 106 | </PropertyGroup> |
102 | 107 |
|
103 | 108 | <!-- |
104 | | - Report test status. |
| 109 | + Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project |
| 110 | + we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961. |
| 111 | + --> |
| 112 | + <Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' != '0'" File="QuarantinedTestRunsheetBuilder" /> |
| 113 | + |
| 114 | + <!-- |
| 115 | + Read the test result and see if the string "No test matches the given testcase filter" is present. |
| 116 | + This indicates that there are no quarantined tests to run. |
105 | 117 | --> |
106 | | - <Message Text="Search complete, no quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '8'" Importance="high" /> |
107 | | - <Message Text="💡 Search complete, quarantined tests found: $(_TestAssembly) [$(_TestEnvironment)]" Condition="'$(_TestErrorCode)' == '0'" Importance="high" /> |
108 | | - <Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition=" '$(_TestErrorCode)' != '0' and '$(_TestErrorCode)' != '8' " File="QuarantinedTestRunsheetBuilder" /> |
| 118 | + <ReadLinesFromFile File="$(_ResultsFileToDisplay)"> |
| 119 | + <Output TaskParameter="Lines" ItemName="FileLines" /> |
| 120 | + </ReadLinesFromFile> |
| 121 | + |
| 122 | + <ItemGroup> |
| 123 | + <Lines Include="@(FileLines)" > |
| 124 | + <Sanitized>$([System.String]::Copy('%(FileLines.Identity)').Replace('`', ''))</Sanitized> |
| 125 | + </Lines> |
| 126 | + |
| 127 | + <_UnhandledExceptionLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Unhandled exception`)) " /> |
| 128 | + <_ExceptionDiscoveringTestsLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`Exception discovering tests`)) " /> |
| 129 | + <_NoTestMatchesLines Include="@(Lines)" Condition=" $([MSBuild]::ValueOrDefault(`%(Lines.Sanitized)`, '').Contains(`No test matches the given testcase filter`)) " /> |
| 130 | + </ItemGroup> |
109 | 131 |
|
110 | 132 | <PropertyGroup> |
| 133 | + <_FailedToEnumerateTests Condition=" '@(_UnhandledExceptionLines)' != '' or '@(_ExceptionDiscoveringTestsLines)' != '' ">true</_FailedToEnumerateTests> |
| 134 | + |
111 | 135 | <_HasQuarantinedTests>true</_HasQuarantinedTests> |
112 | | - <_HasQuarantinedTests Condition=" '$(_TestErrorCode)' == '8' ">false</_HasQuarantinedTests> |
| 136 | + <_HasQuarantinedTests Condition=" '@(_NoTestMatchesLines)' != '' ">false</_HasQuarantinedTests> |
113 | 137 | </PropertyGroup> |
114 | 138 |
|
| 139 | + <Error Text="Search failed: $(_ResultsFileToDisplay) [$(_TestEnvironment)]" Condition="'$(_FailedToEnumerateTests)' == 'true'" File="QuarantinedTestRunsheetBuilder" /> |
| 140 | + <Message Text="💡 Quarantined tests found in $(_TestAssembly) [$(_TestEnvironment)]" Condition=" '$(_HasQuarantinedTests)' == 'true' " Importance="high" /> |
| 141 | + |
115 | 142 | <!-- |
116 | 143 | Generate test runsheet, if there are quarantined tests. |
117 | 144 | --> |
|
133 | 160 | <_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux> |
134 | 161 | <_TestCommand>-restore -build -test -projects "$(_RelativeTestProjectPath)" /bl:"$(_RelativeTestBinLog)" -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false</_TestCommand> |
135 | 162 |
|
136 | | - <!-- |
137 | | - Some quarantinted test may only be executable on Windows or Linux, however we can't possibly know that at this time. |
138 | | - The MTP runner will return exit code 8 if no tests are found, and we need to ignore it instead of failing the test. |
139 | | - --> |
140 | | - <_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true</_TestCommand> |
141 | | - |
142 | 163 | <!-- Replace \ with /, and then escape " with \", so we have a compliant JSON --> |
143 | 164 | <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"'))</_TestCommand> |
144 | 165 |
|
|
0 commit comments