diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/Condition.cs b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/Condition.cs index 9c072cdb0f..e178355126 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/Condition.cs +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/Condition.cs @@ -80,8 +80,13 @@ internal bool Evaluate(Func propertyValueProvider) switch (Operation) { case Operation.Equal: + // Special case: empty string filter value matches null/empty/whitespace property (uncategorized tests) + if (string.IsNullOrWhiteSpace(Value)) + { + result = multiValue is null or { Length: 0 }; + } // if any value in multi-valued property matches 'this.Value', for Equal to evaluate true. - if (multiValue != null) + else if (multiValue != null) { foreach (string propertyValue in multiValue) { @@ -96,18 +101,27 @@ internal bool Evaluate(Func propertyValueProvider) break; case Operation.NotEqual: - // all values in multi-valued property should not match 'this.Value' for NotEqual to evaluate true. - result = true; - - // if value is null. - if (multiValue != null) + // Special case: empty string filter value matches null/empty property (uncategorized tests) + // So NotEqual to empty string should match tests WITH categories + if (string.IsNullOrWhiteSpace(Value)) { - foreach (string propertyValue in multiValue) + result = multiValue is not null and { Length: > 0 }; + } + else + { + // all values in multi-valued property should not match 'this.Value' for NotEqual to evaluate true. + result = true; + + // if value is null. + if (multiValue != null) { - result = result && !string.Equals(propertyValue, Value, StringComparison.OrdinalIgnoreCase); - if (!result) + foreach (string propertyValue in multiValue) { - break; + result = result && !string.Equals(propertyValue, Value, StringComparison.OrdinalIgnoreCase); + if (!result) + { + break; + } } } } @@ -115,8 +129,14 @@ internal bool Evaluate(Func propertyValueProvider) break; case Operation.Contains: + // Special case: empty string filter value matches null/empty property (uncategorized tests) + if (string.IsNullOrWhiteSpace(Value)) + { + result = multiValue is null or { Length: 0 }; + } + // if any value in multi-valued property contains 'this.Value' for 'Contains' to be true. - if (multiValue != null) + else if (multiValue != null) { foreach (string propertyValue in multiValue) { @@ -132,18 +152,27 @@ internal bool Evaluate(Func propertyValueProvider) break; case Operation.NotContains: - // all values in multi-valued property should not contain 'this.Value' for NotContains to evaluate true. - result = true; - - if (multiValue != null) + // Special case: empty string filter value matches null/empty property (uncategorized tests) + // So NotContains empty string should match tests WITH categories + if (string.IsNullOrWhiteSpace(Value)) { - foreach (string propertyValue in multiValue) + result = multiValue is not null and { Length: > 0 }; + } + else + { + // all values in multi-valued property should not contain 'this.Value' for NotContains to evaluate true. + result = true; + + if (multiValue != null) { - RoslynDebug.Assert(propertyValue != null, "PropertyValue can not be null."); - result = result && !propertyValue.Contains(Value, StringComparison.OrdinalIgnoreCase); - if (!result) + foreach (string propertyValue in multiValue) { - break; + RoslynDebug.Assert(propertyValue != null, "PropertyValue can not be null."); + result = result && !propertyValue.Contains(Value, StringComparison.OrdinalIgnoreCase); + if (!result) + { + break; + } } } } @@ -177,16 +206,18 @@ internal static Condition Parse(string? conditionString) ThrownFormatExceptionForInvalidCondition(conditionString); } - for (int index = 0; index < 3; index++) + // Property name (parts[0]) and operator (parts[1]) must not be empty + if (RoslynString.IsNullOrWhiteSpace(parts[0]) || RoslynString.IsNullOrWhiteSpace(parts[1])) { - if (RoslynString.IsNullOrWhiteSpace(parts[index])) - { - ThrownFormatExceptionForInvalidCondition(conditionString); - } - - parts[index] = parts[index].Trim(); + ThrownFormatExceptionForInvalidCondition(conditionString); } + // parts[2] (value) can be empty to support filtering for uncategorized tests + // Trim all parts + parts[0] = parts[0].Trim(); + parts[1] = parts[1].Trim(); + parts[2] = parts[2].Trim(); + Operation operation = GetOperator(parts[1]); Condition condition = new(parts[0], operation, FilterHelper.Unescape(parts[2])); return condition; diff --git a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/FastFilter.cs b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/FastFilter.cs index 1a8c3ebdb1..cef2a21ecd 100644 --- a/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/FastFilter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.VSTestBridge/ObjectModel/FastFilter.cs @@ -46,9 +46,18 @@ internal bool Evaluate(Func propertyValueProvider) bool matched = false; foreach (string name in FilterProperties.Keys) { - // If there is no value corresponding to given name, treat it as unmatched. + // Special case: if filter contains empty string, check if property is null/empty (uncategorized) + bool hasEmptyStringFilter = FilterProperties[name].Any(string.IsNullOrWhiteSpace); + + // If there is no value corresponding to given name, treat it as unmatched unless filtering for empty string. if (!TryGetPropertyValue(name, propertyValueProvider, out string? singleValue, out string[]? multiValues)) { + if (hasEmptyStringFilter) + { + matched = true; + break; + } + continue; } @@ -57,11 +66,16 @@ internal bool Evaluate(Func propertyValueProvider) string? value = PropertyValueRegex == null ? singleValue : ApplyRegex(singleValue); matched = value != null && FilterProperties[name].Contains(value); } - else + else if (multiValues is { Length: > 0 }) { IEnumerable? values = PropertyValueRegex == null ? multiValues : multiValues?.Select(ApplyRegex); matched = values?.Any(result => result != null && FilterProperties[name].Contains(result)) == true; } + else if (hasEmptyStringFilter) + { + // Empty array matches empty string filter + matched = true; + } if (matched) { diff --git a/src/TestFramework/TestFramework/Attributes/TestMethod/TestCategoryAttribute.cs b/src/TestFramework/TestFramework/Attributes/TestMethod/TestCategoryAttribute.cs index fd4633692e..7f28b2d172 100644 --- a/src/TestFramework/TestFramework/Attributes/TestMethod/TestCategoryAttribute.cs +++ b/src/TestFramework/TestFramework/Attributes/TestMethod/TestCategoryAttribute.cs @@ -18,11 +18,20 @@ public sealed class TestCategoryAttribute : TestCategoryBaseAttribute /// public TestCategoryAttribute(string testCategory) { - List categories = - [ - testCategory - ]; - TestCategories = categories; + // Treat empty or whitespace-only strings as null/uncategorized + // This ensures empty categories are not added to the list + if (string.IsNullOrWhiteSpace(testCategory)) + { + TestCategories = []; + } + else + { + List categories = + [ + testCategory + ]; + TestCategories = categories; + } } /// diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestFilterTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestFilterTests.cs index 6f67a60835..75b2057dd9 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestFilterTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestFilterTests.cs @@ -117,6 +117,91 @@ public async Task RunWithFilterFromRunsettings(string currentTfm) testHostResult.AssertExitCodeIs(ExitCodes.Success); } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task RunWithFilter_EmptyTestCategory_MatchesUncategorizedTests(string currentTfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, currentTfm); + + // Filter for uncategorized tests using empty string + TestHostResult testHostResult = await testHost.ExecuteAsync("--filter TestCategory=", cancellationToken: TestContext.CancellationToken); + + // Should match tests without any TestCategory + testHostResult.AssertOutputContains("Running test: NoCategoryTest"); + testHostResult.AssertOutputContains("Running test: EmptyCategoryTest"); + + // Should NOT match tests with categories + testHostResult.AssertOutputDoesNotContain("Running test: CategoryAOnly"); + testHostResult.AssertOutputDoesNotContain("Running test: CategoryBOnly"); + testHostResult.AssertOutputDoesNotContain("Running test: CategoryAAndB"); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task RunWithFilter_EmptyTestCategoryNotEqual_MatchesCategorizedTests(string currentTfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, currentTfm); + + // Filter for tests that ARE categorized (not equal to empty) + TestHostResult testHostResult = await testHost.ExecuteAsync("--filter TestCategory!=", cancellationToken: TestContext.CancellationToken); + + // Should match tests with categories + testHostResult.AssertOutputContains("Running test: CategoryAOnly"); + testHostResult.AssertOutputContains("Running test: CategoryBOnly"); + testHostResult.AssertOutputContains("Running test: CategoryAAndB"); + + // Should NOT match tests without categories + testHostResult.AssertOutputDoesNotContain("Running test: NoCategoryTest"); + testHostResult.AssertOutputDoesNotContain("Running test: EmptyCategoryTest"); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task RunWithFilter_EmptyTestCategoryOrSpecificCategory_MatchesBoth(string currentTfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, currentTfm); + + // Filter for uncategorized OR CategoryA + TestHostResult testHostResult = await testHost.ExecuteAsync("--filter TestCategory=|TestCategory=CategoryA", cancellationToken: TestContext.CancellationToken); + + // Should match uncategorized tests + testHostResult.AssertOutputContains("Running test: NoCategoryTest"); + testHostResult.AssertOutputContains("Running test: EmptyCategoryTest"); + + // Should match CategoryA tests + testHostResult.AssertOutputContains("Running test: CategoryAOnly"); + testHostResult.AssertOutputContains("Running test: CategoryAAndB"); + + // Should NOT match CategoryB only + testHostResult.AssertOutputDoesNotContain("Running test: CategoryBOnly"); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task RunWithFilter_EmptyTestCategoryContains_MatchesUncategorizedTests(string currentTfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, currentTfm); + + // Filter using contains operator with empty string + TestHostResult testHostResult = await testHost.ExecuteAsync("--filter TestCategory~", cancellationToken: TestContext.CancellationToken); + + // Should match tests without any TestCategory + testHostResult.AssertOutputContains("Running test: NoCategoryTest"); + testHostResult.AssertOutputContains("Running test: EmptyCategoryTest"); + + // Should NOT match tests with categories + testHostResult.AssertOutputDoesNotContain("Running test: CategoryAOnly"); + testHostResult.AssertOutputDoesNotContain("Running test: CategoryBOnly"); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) { public string TargetAssetPath => GetAssetPath(AssetName); @@ -219,6 +304,19 @@ public void CategoryAAndB() { Console.WriteLine($"Running test: {nameof(CategoryAAndB)}"); } + + [TestMethod] + public void NoCategoryTest() + { + Console.WriteLine($"Running test: {nameof(NoCategoryTest)}"); + } + + [TestMethod] + [TestCategory("")] + public void EmptyCategoryTest() + { + Console.WriteLine($"Running test: {nameof(EmptyCategoryTest)}"); + } } """; } diff --git a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/ObjectModel/ConditionTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/ObjectModel/ConditionTests.cs new file mode 100644 index 0000000000..a75454d774 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/ObjectModel/ConditionTests.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.VSTestBridge.ObjectModel; + +namespace Microsoft.Testing.Extensions.VSTestBridge.UnitTests.ObjectModel; + +[TestClass] +public class ConditionTests +{ + [TestMethod] + public void Evaluate_EmptyStringEqualsOperator_WithNullProperty_ShouldReturnTrue() + { + var condition = new Condition("TestCategory", Operation.Equal, string.Empty); + bool result = condition.Evaluate(propertyName => null); + + Assert.IsTrue(result); + } + + [TestMethod] + public void Evaluate_EmptyStringEqualsOperator_WithEmptyArray_ShouldReturnTrue() + { + var condition = new Condition("TestCategory", Operation.Equal, string.Empty); + bool result = condition.Evaluate(propertyName => Array.Empty()); + + Assert.IsTrue(result); + } + + [TestMethod] + public void Evaluate_EmptyStringEqualsOperator_WithNonEmptyArray_ShouldReturnFalse() + { + var condition = new Condition("TestCategory", Operation.Equal, string.Empty); + bool result = condition.Evaluate(propertyName => new[] { "CategoryA" }); + + Assert.IsFalse(result); + } + + [TestMethod] + public void Evaluate_EmptyStringNotEqualsOperator_WithNullProperty_ShouldReturnFalse() + { + var condition = new Condition("TestCategory", Operation.NotEqual, string.Empty); + bool result = condition.Evaluate(propertyName => null); + + Assert.IsFalse(result); + } + + [TestMethod] + public void Evaluate_EmptyStringNotEqualsOperator_WithEmptyArray_ShouldReturnFalse() + { + var condition = new Condition("TestCategory", Operation.NotEqual, string.Empty); + bool result = condition.Evaluate(propertyName => Array.Empty()); + + Assert.IsFalse(result); + } + + [TestMethod] + public void Evaluate_EmptyStringNotEqualsOperator_WithNonEmptyArray_ShouldReturnTrue() + { + var condition = new Condition("TestCategory", Operation.NotEqual, string.Empty); + bool result = condition.Evaluate(propertyName => new[] { "CategoryA" }); + + Assert.IsTrue(result); + } + + [TestMethod] + public void Evaluate_EmptyStringContainsOperator_WithNullProperty_ShouldReturnTrue() + { + var condition = new Condition("TestCategory", Operation.Contains, string.Empty); + bool result = condition.Evaluate(propertyName => null); + + Assert.IsTrue(result); + } + + [TestMethod] + public void Evaluate_EmptyStringContainsOperator_WithNonEmptyArray_ShouldReturnFalse() + { + var condition = new Condition("TestCategory", Operation.Contains, string.Empty); + bool result = condition.Evaluate(propertyName => new[] { "CategoryA" }); + + Assert.IsFalse(result); + } + + [TestMethod] + public void Evaluate_EmptyStringNotContainsOperator_WithNullProperty_ShouldReturnFalse() + { + var condition = new Condition("TestCategory", Operation.NotContains, string.Empty); + bool result = condition.Evaluate(propertyName => null); + + Assert.IsFalse(result); + } + + [TestMethod] + public void Evaluate_EmptyStringNotContainsOperator_WithNonEmptyArray_ShouldReturnTrue() + { + var condition = new Condition("TestCategory", Operation.NotContains, string.Empty); + bool result = condition.Evaluate(propertyName => new[] { "CategoryA" }); + + Assert.IsTrue(result); + } + + [TestMethod] + public void Evaluate_NonEmptyStringEqualsOperator_WithMatchingValue_ShouldReturnTrue() + { + var condition = new Condition("TestCategory", Operation.Equal, "CategoryA"); + bool result = condition.Evaluate(propertyName => new[] { "CategoryA" }); + + Assert.IsTrue(result); + } + + [TestMethod] + public void Evaluate_NonEmptyStringEqualsOperator_WithNonMatchingValue_ShouldReturnFalse() + { + var condition = new Condition("TestCategory", Operation.Equal, "CategoryA"); + bool result = condition.Evaluate(propertyName => new[] { "CategoryB" }); + + Assert.IsFalse(result); + } + + [TestMethod] + public void Parse_EmptyValueAfterEquals_ShouldCreateConditionWithEmptyValue() + { + Condition condition = Condition.Parse("TestCategory="); + + Assert.AreEqual("TestCategory", condition.Name); + Assert.AreEqual(Operation.Equal, condition.Operation); + Assert.AreEqual(string.Empty, condition.Value); + } + + [TestMethod] + public void Parse_WhitespaceValueAfterEquals_ShouldCreateConditionWithEmptyValue() + { + Condition condition = Condition.Parse("TestCategory= "); + + Assert.AreEqual("TestCategory", condition.Name); + Assert.AreEqual(Operation.Equal, condition.Operation); + Assert.AreEqual(string.Empty, condition.Value); + } + + [TestMethod] + public void Parse_EmptyValueWithNotEquals_ShouldCreateConditionWithEmptyValue() + { + Condition condition = Condition.Parse("TestCategory!="); + + Assert.AreEqual("TestCategory", condition.Name); + Assert.AreEqual(Operation.NotEqual, condition.Operation); + Assert.AreEqual(string.Empty, condition.Value); + } + + [TestMethod] + public void Parse_EmptyValueWithContains_ShouldCreateConditionWithEmptyValue() + { + Condition condition = Condition.Parse("TestCategory~"); + + Assert.AreEqual("TestCategory", condition.Name); + Assert.AreEqual(Operation.Contains, condition.Operation); + Assert.AreEqual(string.Empty, condition.Value); + } + + [TestMethod] + public void Parse_EmptyValueWithNotContains_ShouldCreateConditionWithEmptyValue() + { + Condition condition = Condition.Parse("TestCategory!~"); + + Assert.AreEqual("TestCategory", condition.Name); + Assert.AreEqual(Operation.NotContains, condition.Operation); + Assert.AreEqual(string.Empty, condition.Value); + } +} diff --git a/test/UnitTests/TestFramework.UnitTests/Attributes/TestCategoryAttributeTests.cs b/test/UnitTests/TestFramework.UnitTests/Attributes/TestCategoryAttributeTests.cs new file mode 100644 index 0000000000..e24de81409 --- /dev/null +++ b/test/UnitTests/TestFramework.UnitTests/Attributes/TestCategoryAttributeTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AwesomeAssertions; + +using TestFramework.ForTestingMSTest; + +namespace Microsoft.VisualStudio.TestPlatform.TestFramework.UnitTests.Attributes; + +public class TestCategoryAttributeTests : TestContainer +{ + public void ConstructorShouldSetCategoryPassed() + { + var category = new TestCategoryAttribute("UnitTest"); + + category.TestCategories.Should().BeEquivalentTo(new[] { "UnitTest" }); + } + + public void ConstructorWithEmptyStringShouldResultInEmptyList() + { + var category = new TestCategoryAttribute(""); + + category.TestCategories.Should().BeEmpty(); + } + + public void ConstructorWithWhitespaceStringShouldResultInEmptyList() + { + var category = new TestCategoryAttribute(" "); + + category.TestCategories.Should().BeEmpty(); + } + + public void ConstructorWithNullShouldResultInEmptyList() + { + var category = new TestCategoryAttribute(null!); + + category.TestCategories.Should().BeEmpty(); + } + + public void ConstructorWithValidCategoryShouldPreserveValue() + { + var category = new TestCategoryAttribute("Integration"); + + category.TestCategories.Should().HaveCount(1); + category.TestCategories[0].Should().Be("Integration"); + } +}