[GITHUB-2897] Not exception but warning if some (not all) of the give…

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+New: GITHUB-2897: Not exception but warning if some (not all) of the given test names are not found in suite files. (Bruce Wen)
 Fixed: GITHUB-2888: Skipped Tests with DataProvider appear as failed (Joaquin Moreira)
 Fixed: GITHUB-2884: Discrepancies with DataProvider and Retry of failed tests (Krishnan Mahadevan)
 Fixed: GITHUB-2879: Test listeners specified in parent testng.xml file are not included in testng-failed.xml file (Krishnan Mahadevan)
@@ -1294,7 +1295,7 @@ Fixed: Wasn't parsing <selector-class-name> correctly
 Fixed: Annotation Transformers now work on class-level annotations
 Fixed: Some class-level @Test attributes were not always honored
 Added: Clean separation between @Test invocation events and @Configuration invocation events
-       (see also TESTNG-111)
+	   (see also TESTNG-111)
 Added: Test instances created by @Factory now run in multiple threads in parallel mode
 Fixed: @Before/@AfterGroups invocation order
 Fixed: TESTNG-27: Parameters are not used on <test> level anymore
@@ -1462,8 +1463,8 @@ Fixed: TestNGAntTask was hardcoding m_haltOnFSP to true
 Fixed: Passing a null parameter caused an NPE in the reports
 Added: "listener" to the ant task (and documentation)
 Added: if patch-testng-sourcedir.properties is found in the classpath
-       with a property "sourcedir" containing a ; separated list of
-       directories, this list will override -sourcedir.
+	   with a property "sourcedir" containing a ; separated list of
+	   directories, this list will override -sourcedir.
 
 ===========================================================================
 
@@ -1473,14 +1474,14 @@ Added: if patch-testng-sourcedir.properties is found in the classpath
 Added: Maven 2 plug-in
 Fixed: Message formattings in TestNG assertion utility class
 Fixed: @Factory methods were counted as @Test as well
-       https://jira.opensymphony.com/browse/TESTNG-51
+	   https://jira.opensymphony.com/browse/TESTNG-51
 Fixed: All DataProvider parameters were shown in the HTML report
 Fixed: Bug in testng-failed.xml generation
 Fixed: <packages> bug when using a jar file to load the test classes
 Added: alwaysRun for before @Configuration methods
-       https://jira.opensymphony.com/browse/TESTNG-35
+	   https://jira.opensymphony.com/browse/TESTNG-35
 Fixed: groupless @Configurations were not invoked if a method depends on a group
-       https://jira.opensymphony.com/browse/TESTNG-45
+	   https://jira.opensymphony.com/browse/TESTNG-45
 Added: beforeGroups/afterGroups to @Configuration
 
 Eclipse plugin:
@@ -1621,7 +1622,7 @@ Fixed: Bug in <package> when directories contain spaces in their names
 Fixed: Introduced a JDK5 dependency in the JDK1.4 build (getEnclosingClass())
 Fixed: Output directory in ant task was not honored if it didn't exist
 Fixed: Problem with timeout according to
-       https://forums.opensymphony.com/thread.jspa?threadID=6707
+	   https://forums.opensymphony.com/thread.jspa?threadID=6707
 
 Eclipse plug-in:
 
@@ -1631,7 +1632,7 @@ Added: Quick Fix for JUnit conversion (Annotations and JavaDoc)
 Fixed: Methods Run as TestNG test
 Added: Package level Run as TestNG test
 Fixed: Resources from the linked directories are using a wrong path when
-       passed to command line TestNG
+	   passed to command line TestNG
 
 IDEA plug-in:
 

testng-ant/src/main/java/org/testng/TestNGAntTask.java

@@ -161,6 +161,7 @@ public class TestNGAntTask extends Task {
   private String m_methods;
   private Mode mode = Mode.testng;
   private boolean forkJvm = true;
+  private boolean m_ignoreMissedTestNames;
 
   public enum Mode {
     // lower-case to better look in build scripts
@@ -360,6 +361,10 @@ public void setTestNames(String testNames) {
     m_testNames = testNames;
   }
 
+  public void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+    m_ignoreMissedTestNames = ignoreMissedTestNames;
+  }
+
   /**
    * Sets the suite runner class to invoke
    *
@@ -578,6 +583,7 @@ protected List<String> createArguments() {
     addStringIfNotBlank(argv, CommandLineArgs.SUITE_NAME, m_suiteName);
     addStringIfNotBlank(argv, CommandLineArgs.TEST_NAME, m_testName);
     addStringIfNotBlank(argv, CommandLineArgs.TEST_NAMES, m_testNames);
+    addBooleanIfTrue(argv, CommandLineArgs.IGNORE_MISSED_TEST_NAMES, m_ignoreMissedTestNames);
     addStringIfNotBlank(argv, CommandLineArgs.METHODS, m_methods);
     addReporterConfigs(argv);
     addIntegerIfNotNull(argv, CommandLineArgs.SUITE_THREAD_POOL_SIZE, m_suiteThreadPoolSize);

testng-core/src/main/java/org/testng/CommandLineArgs.java

@@ -142,6 +142,14 @@ public class CommandLineArgs {
   @Parameter(names = TEST_NAMES, description = "The list of test names to run")
   public String testNames;
 
+  public static final String IGNORE_MISSED_TEST_NAMES = "-ignoreMissedTestNames";
+
+  @Parameters(
+      names = IGNORE_MISSED_TEST_NAMES,
+      description =
+          "Ignore missed test names given by '-testnames' and continue to run existing tests, if any.")
+  public boolean ignoreMissedTestNames = false;
+
   public static final String TEST_JAR = "-testjar";
 
   @Parameter(names = TEST_JAR, description = "A jar file containing the tests")

testng-core/src/main/java/org/testng/JarFileUtils.java

@@ -10,9 +10,9 @@
 import java.util.Objects;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
+import jdk.internal.joptsimple.internal.Strings;
 import org.testng.collections.Lists;
 import org.testng.internal.Utils;
-import org.testng.util.Strings;
 import org.testng.xml.IPostProcessor;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.internal.Parser;
@@ -23,6 +23,7 @@
 class JarFileUtils {
   private final IPostProcessor processor;
   private final String xmlPathInJar;
+  private final boolean ignoreMissedTestNames;
   private final List<String> testNames;
   private final List<XmlSuite> suites = Lists.newLinkedList();
   private final XmlSuite.ParallelMode mode;
@@ -36,10 +37,20 @@ class JarFileUtils {
       String xmlPathInJar,
       List<String> testNames,
       XmlSuite.ParallelMode mode) {
+    this(processor, xmlPathInJar, testNames, mode, false);
+  }
+
+  JarFileUtils(
+      IPostProcessor processor,
+      String xmlPathInJar,
+      List<String> testNames,
+      XmlSuite.ParallelMode mode,
+      final boolean ignoreMissedTestNames) {
     this.processor = processor;
     this.xmlPathInJar = xmlPathInJar;
     this.testNames = testNames;
     this.mode = mode == null ? XmlSuite.ParallelMode.NONE : mode;
+    this.ignoreMissedTestNames = ignoreMissedTestNames;
   }
 
   List<XmlSuite> extractSuitesFrom(File jarFile) {
@@ -96,7 +107,7 @@ private boolean testngXmlExistsInJar(File jarFile, List<String> classes) throws
         // If test names were specified, only run these test names
         if (testNames != null) {
           TestNamesMatcher testNamesMatcher = new TestNamesMatcher(suite, testNames);
-          testNamesMatcher.validateMissMatchedTestNames();
+          testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
           suites.addAll(testNamesMatcher.getSuitesMatchingTestNames());
         } else {
           suites.add(suite);

testng-core/src/main/java/org/testng/TestNG.java

@@ -354,7 +354,7 @@ private Collection<XmlSuite> processCommandLineArgs(Collection<XmlSuite> allSuit
       }
       // If test names were specified, only run these test names
       TestNamesMatcher testNamesMatcher = new TestNamesMatcher(s, m_testNames);
-      testNamesMatcher.validateMissMatchedTestNames();
+      testNamesMatcher.validateMissMatchedTestNames(m_ignoreMissedTestNames);
       result.addAll(testNamesMatcher.getSuitesMatchingTestNames());
     }
 
@@ -799,6 +799,8 @@ public List<ISuiteListener> getSuiteListeners() {
   /** The list of test names to run from the given suite */
   private List<String> m_testNames;
 
+  private boolean m_ignoreMissedTestNames;
+
   private Integer m_suiteThreadPoolSize = CommandLineArgs.SUITE_THREAD_POOL_SIZE_DEFAULT;
 
   private boolean m_randomizeSuites = Boolean.FALSE;
@@ -1475,6 +1477,7 @@ protected void configure(CommandLineArgs cla) {
 
     if (cla.testNames != null) {
       setTestNames(Arrays.asList(cla.testNames.split(",")));
+      setIgnoreMissedTestNames(cla.ignoreMissedTestNames);
     }
 
     // Note: can't use a Boolean field here because we are allowing a boolean
@@ -1574,6 +1577,10 @@ protected void configure(CommandLineArgs cla) {
     alwaysRunListeners(cla.alwaysRunListeners);
   }
 
+  private void setIgnoreMissedTestNames(boolean ignoreMissedTestNames) {
+    m_ignoreMissedTestNames = ignoreMissedTestNames;
+  }
+
   public void setSuiteThreadPoolSize(Integer suiteThreadPoolSize) {
     m_suiteThreadPoolSize = suiteThreadPoolSize;
   }
@@ -1641,6 +1648,9 @@ public void configure(Map cmdLineArgs) {
     String testNames = (String) cmdLineArgs.get(CommandLineArgs.TEST_NAMES);
     if (testNames != null) {
       result.testNames = testNames;
+      result.ignoreMissedTestNames =
+          (Boolean)
+              cmdLineArgs.getOrDefault(CommandLineArgs.IGNORE_MISSED_TEST_NAMES, Boolean.FALSE);
     }
 
     String useDefaultListeners = (String) cmdLineArgs.get(CommandLineArgs.USE_DEFAULT_LISTENERS);

testng-core/src/main/java/org/testng/xml/internal/TestNamesMatcher.java

@@ -3,12 +3,15 @@
 import java.util.List;
 import org.testng.TestNGException;
 import org.testng.collections.Lists;
+import org.testng.log4testng.Logger;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 
 /** The class to work with "-testnames" */
 public final class TestNamesMatcher {
 
+  private static final Logger LOGGER = Logger.getLogger(TestNamesMatcher.class);
+
   private final List<XmlSuite> cloneSuites = Lists.newArrayList();
   private final List<String> matchedTestNames = Lists.newArrayList();
   private final List<XmlTest> matchedTests = Lists.newArrayList();
@@ -43,13 +46,29 @@ public List<XmlSuite> getSuitesMatchingTestNames() {
     return cloneSuites;
   }
 
-  public void validateMissMatchedTestNames() {
+  /**
+   * Do validation for testNames and notify users if any testNames are missed in suite.
+   *
+   * @param ignoreMissedTestNames if true print warning message otherwise throw TestNGException for
+   *     missed testNames.
+   */
+  public void validateMissMatchedTestNames(final boolean ignoreMissedTestNames) {
+    final List<String> tmpTestNames = getMissedTestNames();
+    if (!tmpTestNames.isEmpty()) {
+      final String errMsg = "The test(s) <" + tmpTestNames + "> cannot be found in suite.";
+      if (ignoreMissedTestNames) {
+        LOGGER.warn(errMsg);
+      } else {
+        throw new TestNGException(errMsg);
+      }
+    }
+  }
+
+  public List<String> getMissedTestNames() {
     List<String> tmpTestNames = Lists.newArrayList();
     tmpTestNames.addAll(testNames);
     tmpTestNames.removeIf(matchedTestNames::contains);
-    if (!tmpTestNames.isEmpty()) {
-      throw new TestNGException("The test(s) <" + tmpTestNames + "> cannot be found in suite.");
-    }
+    return tmpTestNames;
   }
 
   public List<XmlTest> getMatchedTests() {

testng-core/src/test/java/org/testng/xml/internal/TestNamesMatcherTest.java

@@ -90,6 +90,38 @@ public void testCloneIfContainsTestsWithNamesMatchingAnyWithoutMatch() {
     }
   }
 
+  @Test(description = "GITHUB-2897, No exception thrown when ignoreMissedTestNames enabled.")
+  public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesEnabled() {
+    final boolean ignoreMissedTestNames = true;
+    XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+    TestNamesMatcher testNamesMatcher =
+        new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
+    testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+  }
+
+  @Test(
+      description = "GITHUB-2897, Expected exception thrown when ignoreMissedTestNames disabled.",
+      expectedExceptions = TestNGException.class,
+      expectedExceptionsMessageRegExp =
+          "\nThe test\\(s\\) \\<\\[test3\\]\\> cannot be found in suite.")
+  public void testNoExceptionFromValidateWhenIgnoreMissedTestNamesDisabled() {
+    final boolean ignoreMissedTestNames = false;
+    XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+    TestNamesMatcher testNamesMatcher =
+        new TestNamesMatcher(xmlSuite, Collections.singletonList("test3"));
+    testNamesMatcher.validateMissMatchedTestNames(ignoreMissedTestNames);
+  }
+
+  @Test(description = "GITHUB-2897, Missed test names are found as expected.")
+  public void testMissedTestNamesFound() {
+    XmlSuite xmlSuite = createDummySuiteWithTestNamesAs("test1", "test2");
+    final String expectedMissedTestNames = "test3";
+    TestNamesMatcher testNamesMatcher =
+        new TestNamesMatcher(xmlSuite, Collections.singletonList(expectedMissedTestNames));
+    List<String> missedTestNames = testNamesMatcher.getMissedTestNames();
+    assertThat(missedTestNames).hasSameElementsAs(Arrays.asList(expectedMissedTestNames));
+  }
+
   @DataProvider(name = "getTestnames")
   public Object[][] getTestnameToSearchFor() {
     return new Object[][] {