HADOOP-17628. DistCp contract test speedup.

* Address review comments
* log IOStats after each test case.
  Important: as the cached FS retains statistics, the numbers
  get bigger over time.
* HDFS test is now reinstated, as we've identified that most
  of its long execution time is from the large file upload/download
  suites. Disable them and its execution time drops from 4m to 30s,
  which means it can then be used to make sure the contract suite
  is consistent between HDFS and the object stores.

Change-Id: I6d1cf5bf42916035a806fa9e78c003074f8f12b9

Author: steveloughran

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.contract.s3a;
 
-import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
@@ -74,14 +73,8 @@ public void testDistCpWithIterator() throws Exception {
          renames);
   }
 
-  @Override
-  public void testDirectWrite() throws Exception {
-    skip("Not needed as all tests are direct by default");
-  }
-
   @Override
   public void testNonDirectWrite() throws Exception {
-//    ContractTestUtils.skip("disabled for peformance reasons");
     final long renames = getRenameOperationCount();
     super.testNonDirectWrite();
     assertEquals("Expected 2 renames for a non-direct write distcp", 2L,

hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java

@@ -18,7 +18,9 @@
 
 package org.apache.hadoop.tools.contract;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel;
 import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_DISTCP_JOB_ID;
 
 import java.io.IOException;
@@ -170,12 +172,20 @@ public void setup() throws Exception {
     localDir =
         localFS.makeQualified(new Path(new Path(
         GenericTestUtils.getTestDir().toURI()), testSubDir + "/local"));
+    localFS.delete(localDir, true);
     mkdirs(localFS, localDir);
-    remoteDir = path(testSubDir + "/remote");
+    Path testSubPath = path(testSubDir);
+    remoteDir = new Path(testSubPath, "remote");
     // test teardown does this, but IDE-based test debugging can skip
     // that teardown; this guarantees the initial state is clean
     remoteFS.delete(remoteDir, true);
-    localFS.delete(localDir, true);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    // if remote FS supports IOStatistics log it.
+    logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, getRemoteFS());
+    super.teardown();
   }
 
   /**
@@ -550,9 +560,9 @@ private Path distCpDeepDirectoryStructure(FileSystem srcFS,
   private void largeFiles(FileSystem srcFS, Path srcDir, FileSystem dstFS,
       Path dstDir) throws Exception {
     int fileSizeKb = conf.getInt(SCALE_TEST_DISTCP_FILE_SIZE_KB,
-        DEFAULT_DISTCP_SIZE_KB);
+        getDefaultDistCPSizeKb());
     if (fileSizeKb < 1) {
-      skip("File size in " + SCALE_TEST_DISTCP_FILE_SIZE_KB + " too small");
+      skip("File size in " + SCALE_TEST_DISTCP_FILE_SIZE_KB + " is zero");
     }
     initPathFields(srcDir, dstDir);
     Path largeFile1 = new Path(inputDir, "file1");
@@ -573,6 +583,18 @@ private void largeFiles(FileSystem srcFS, Path srcDir, FileSystem dstFS,
     verifyFileContents(dstFS, new Path(target, "inputDir/file3"), data3);
   }
 
+  /**
+   * Override point. What is the default distcp size
+   * for large files if not overridden by
+   * {@link #SCALE_TEST_DISTCP_FILE_SIZE_KB}.
+   * If 0 then, unless overridden in the configuration,
+   * the large file tests will not run.
+   * @return file size.
+   */
+  protected int getDefaultDistCPSizeKb() {
+    return DEFAULT_DISTCP_SIZE_KB;
+  }
+
   /**
    * Executes DistCp and asserts that the job finished successfully.
    * The choice of direct/indirect is based on the value of
@@ -629,14 +651,14 @@ private static void mkdirs(FileSystem fs, Path dir) throws Exception {
   @Test
   public void testDirectWrite() throws Exception {
     describe("copy file from local to remote using direct write option");
+    if (directWriteAlways()) {
+      skip("not needed as all other tests use the -direct option.");
+    }
     directWrite(localFS, localDir, remoteFS, remoteDir, true);
   }
 
   @Test
   public void testNonDirectWrite() throws Exception {
-    if (directWriteAlways()) {
-      skip("not needed");
-    }
     describe("copy file from local to remote without using direct write " +
         "option");
     directWrite(localFS, localDir, remoteFS, remoteDir, false);

…ract/OptionalTestHDFSContractDistCp.java …ols/contract/TestHDFSContractDistCp.javahadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/OptionalTestHDFSContractDistCp.java renamed to hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/TestHDFSContractDistCp.java hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/OptionalTestHDFSContractDistCp.java renamed to hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/TestHDFSContractDistCp.java

@@ -30,8 +30,9 @@
  * Verifies that the HDFS passes all the tests in
  * {@link AbstractContractDistCpTest}.
  * As such, it acts as an in-module validation of this contract test itself.
+ * It does skip the large file test cases for speed.
  */
-public class OptionalTestHDFSContractDistCp extends AbstractContractDistCpTest {
+public class TestHDFSContractDistCp extends AbstractContractDistCpTest {
 
   @BeforeClass
   public static void createCluster() throws IOException {
@@ -47,4 +48,14 @@ public static void teardownCluster() throws IOException {
   protected AbstractFSContract createContract(Configuration conf) {
     return new HDFSContract(conf);
   }
+
+  /**
+   * Turn off the large file tests as they are very slow and there
+   * are many other distcp to HDFS tests which verify such things.
+   * @return 0
+   */
+  @Override
+  protected int getDefaultDistCPSizeKb() {
+    return 0;
+  }
 }