[GITHUB-2897] Not exception but warning if some (not all) of the give…

testng-core/src/main/java/org/testng/JarFileUtils.java

@@ -104,15 +104,20 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
       Collection<XmlSuite> parsedSuites = Parser.parse(suitePath, processor);
       delete(file);
       for (XmlSuite suite : parsedSuites) {
-        // If test names were specified, only run these test names
+        // If test names were specified, only run these test names. If any test names missed, then
+        // won't run any test names. (default legacy logic)
         if (testNames != null) {
           TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames);
-          testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
-          suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
+          boolean validationResult =
+              testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+          if (validationResult) {
+            suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
+          }
+          return validationResult;
         } else {
           suites.add(suite);
+          return true;
         }
-        return true;
       }
     }
     return false;

testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java

@@ -1,9 +1,9 @@
 package org.testng.xml.internal;
 
 import java.util.List;
-import org.testng.internal.RuntimeBehavior;
 import org.testng.TestNGException;
 import org.testng.collections.Lists;
+import org.testng.internal.RuntimeBehavior;
 import org.testng.log4testng.Logger;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
@@ -51,28 +51,41 @@ public List<XmlSuite> getSuitesMatchingTestNames() {
   }
 
   /**
-   * Do validation for testNames and notify users if any testNames are missed in suite.
+   * Do validation for testNames and notify users if any testNames are missed in suite. This method
+   * is also used to decide how to run test suite when test names are given. In legacy logic, if
+   * test names are given and exist in suite, then run them; if any of them do not exist in suite,
+   * then throw exception and exit. After ignoreMissedTestNames is introduced, if
+   * ignoreMissedTestNames is enabled, then any of the given test names exist in suite will be run,
+   * and print warning message to tell those test names do not exist in suite.
    *
    * @param ignoreMissedTestNames if true print warning message otherwise throw TestNGException for
    *     missed testNames.
+   * @return boolean if ignoreMissedTestNames disabled, then return true if no missed test names in
+   *     suite, otherwise throw TestNGException; if ignoreMissedTestNames enabled, then return true
+   *     if any test names exist in suite, otehrwise (all given test names are missed) false.
    */
-  public void validateMissMatchedTestNames(final boolean ignoreMissedTestNames) {
-    final List<String> tmpTestNames = getMissedTestNames();
-    if (!tmpTestNames.isEmpty()) {
-      final String errMsg = "The test(s) <" + tmpTestNames + "> cannot be found in suite.";
+  public boolean validateMissMatchedTestNames(final boolean ignoreMissedTestNames) {
+    final List<String> missedTestNames = getMissedTestNames();
+    if (!missedTestNames.isEmpty()) {
+      final String errMsg = "The test(s) <" + missedTestNames + "> cannot be found in suite.";
+
       if (ignoreMissedTestNames || RuntimeBehavior.ignoreMissedTestNames()) {
         LOGGER.warn(errMsg);
+        // as long as any test names match, then tell caller to run them.
+        return !matchedTestNames.isEmpty();
       } else {
+        // legacy, throw exception and exit execution
         throw new TestNGException(errMsg);
       }
     }
+    return missedTestNames.isEmpty();
   }
 
   public List<String> getMissedTestNames() {
-    List<String> tmpTestNames = Lists.newArrayList();
-    tmpTestNames.addAll(testNames);
-    tmpTestNames.removeIf(matchedTestNames::contains);
-    return tmpTestNames;
+    List<String> missedTestNames = Lists.newArrayList();
+    missedTestNames.addAll(testNames);
+    missedTestNames.removeIf(matchedTestNames::contains);
+    return missedTestNames;
   }
 
   public List<XmlTest> getMatchedTests() {

testng-core/src/test/java/org/testng/JarFileUtilsTest.java

@@ -14,6 +14,7 @@
 import java.util.List;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
+import org.testng.internal.RuntimeBehavior;
 import org.testng.testhelper.JarCreator;
 import org.testng.xml.IPostProcessor;
 import org.testng.xml.XmlClass;
@@ -65,6 +66,22 @@ public void testWithInvalidTestNames() throws MalformedURLException {
         new String[] {"org.testng.jarfileutils.org.testng.SampleTest1"});
   }
 
+  @Test(
+      description =
+          "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled by System property 'testng.ignore.missed.testnames'.")
+  public void testWithInvalidTestNamesNoExceptionIfIgnoreMissedTestNamesEnabledBySystemProperty()
+      throws MalformedURLException {
+    String oldIgnoreMissedTestNames =
+        System.getProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "false");
+    try {
+      System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "true");
+      JarFileUtils utils = newJarFileUtils(Collections.singletonList("testng-tests-child11"));
+      runTest(utils, 1, null, null, "Jar suite");
+    } finally {
+      System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, oldIgnoreMissedTestNames);
+    }
+  }
+
   @Test
   public void testWithInvalidXmlFile() throws MalformedURLException {
     JarFileUtils utils =
@@ -161,7 +178,9 @@ private static void runTest(
     if (expectedTestNames != null) {
       assertThat(testNames).containsExactly(expectedTestNames);
     }
-    assertThat(classNames).contains(expectedClassNames);
+    if (expectedClassNames != null) {
+      assertThat(classNames).contains(expectedClassNames);
+    }
   }
 
   public static class FakeProcessor implements IPostProcessor {

testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java

@@ -6,12 +6,12 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import org.testng.internal.RuntimeBehavior;
 import org.testng.TestNGException;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 import org.testng.collections.CollectionUtils;
 import org.testng.collections.Lists;
+import org.testng.internal.RuntimeBehavior;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 import test.SimpleBaseTest;
@@ -91,7 +91,7 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyWithoutMatch() {
     }
   }
 
-  @Test(description = "GITHUB-2897, No exception thrown when ignoreMissedTestNames enabled.")
+  @Test(description = "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled.")
   public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesEnabled() {
     final boolean ignoreMissedTestNames = true;
     XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
@@ -101,7 +101,8 @@ public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesEnabled() {
   }
 
   @Test(
-      description = "GITHUB-2897, Expected exception thrown when ignoreMissedTestNames disabled.",
+      description =
+          "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames disabled.",
       expectedExceptions = TestNGException.class,
       expectedExceptionsMessageRegExp =
           "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.")
@@ -115,29 +116,41 @@ public void testHaveExceptionFromValidateWhenIgnoreMissedTestNamesDisabled() {
 
   @Test(
       description =
-          "GITHUB-2897, No exception thrown when ignoreMissedTestNames enabled by System property 'testng.ignore.missed.testnames'.")
+          "GITHUB-2897, No TestNGException thrown when ignoreMissedTestNames enabled by System property 'testng.ignore.missed.testnames'.")
   public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesEnabledBySystemProperty() {
     final boolean ignoreMissedTestNames = false;
-    System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "true");
-    XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
-    TestNamesMatcher testNamesMatcher =
-        new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
-    testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+    String oldIgnoreMissedTestNames =
+        System.getProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "false");
+    try {
+      System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "true");
+      XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+      TestNamesMatcher testNamesMatcher =
+          new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
+      testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+    } finally {
+      System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, oldIgnoreMissedTestNames);
+    }
   }
 
   @Test(
       description =
-          "GITHUB-2897, Expected exception thrown when ignoreMissedTestNames disabled by System property 'testng.ignore.missed.testnames'.",
+          "GITHUB-2897, Expected TestNGException thrown when ignoreMissedTestNames disabled by System property 'testng.ignore.missed.testnames'.",
       expectedExceptions = TestNGException.class,
       expectedExceptionsMessageRegExp =
           "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.")
   public void testHaveExceptionFromValidateWhenIgnoreMissedTestNamesDisabledBySystemProperty() {
     final boolean ignoreMissedTestNames = false;
-    System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "false");
-    XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
-    TestNamesMatcher testNamesMatcher =
-        new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
-    testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+    String oldIgnoreMissedTestNames =
+        System.getProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "false");
+    try {
+      System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, "false");
+      XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+      TestNamesMatcher testNamesMatcher =
+          new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
+      testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+    } finally {
+      System.setProperty(RuntimeBehavior.TESTNG_IGNORE_MISSED_TESTNAMES, oldIgnoreMissedTestNames);
+    }
   }
 
   @Test(description = "GITHUB-2897, Missed test names are found as expected.")