HBASE-29802 NPE when shutting down mini cluster cause tests hang (#7604) (#7612)

Remove the 'shutdownHook' in SingleProcessHBaseCluster where we close
the FileSystem instance of a region server.
Set DistributedFileSystem cache to false when creating MiniMRCluster so
when shutting down MiniMRCluster it will not close the shared FileSystem
instance.
Also re-enable TestExportSnapshot.

(cherry picked from commit d11ee6a)

Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>
(cherry picked from commit 8841deb)

Author: Apache9

hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java

@@ -64,7 +64,6 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -80,7 +79,6 @@
 /**
  * Test Export Snapshot Tool
  */
-@Ignore // HBASE-24493
 @Category({ VerySlowMapReduceTests.class, LargeTests.class })
 public class TestExportSnapshot {
 

hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java

@@ -2832,10 +2832,14 @@ private void startMiniMapReduceCluster(final int servers) throws IOException {
         jvmOpts + " --add-opens java.base/java.lang=ALL-UNNAMED");
     }
 
+    // Disable fs cache for DistributedFileSystem, to avoid YARN RM close the shared FileSystem
+    // instance and cause trouble in HBase. See HBASE-29802 for more details.
+    JobConf mrClusterConf = new JobConf(conf);
+    mrClusterConf.setBoolean("fs.hdfs.impl.disable.cache", true);
     // Allow the user to override FS URI for this map-reduce cluster to use.
     mrCluster =
       new MiniMRCluster(servers, FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(),
-        1, null, null, new JobConf(this.conf));
+        1, null, null, mrClusterConf);
     JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster);
     if (jobConf == null) {
       jobConf = mrCluster.createJobConf();

hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java

@@ -24,7 +24,6 @@
 import java.util.List;
 import java.util.Set;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.master.HMaster;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -45,7 +44,6 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
 
 /**
  * This class creates a single process HBase cluster. each server. The master uses the 'default'
@@ -122,7 +120,7 @@ public Configuration getConfiguration() {
    * only, not All filesystems as the FileSystem system exit hook does.
    */
   public static class MiniHBaseClusterRegionServer extends HRegionServer {
-    private Thread shutdownThread = null;
+
     private User user = null;
     /**
      * List of RegionServers killed so far. ServerName also comprises startCode of a server, so any
@@ -137,20 +135,6 @@ public MiniHBaseClusterRegionServer(Configuration conf)
       this.user = User.getCurrent();
     }
 
-    /*
-     * @param currentfs We return this if we did not make a new one.
-     * @param uniqueName Same name used to help identify the created fs.
-     * @return A new fs instance if we are up on DistributeFileSystem.
-     */
-
-    @Override
-    protected void handleReportForDutyResponse(final RegionServerStartupResponse c)
-      throws IOException {
-      super.handleReportForDutyResponse(c);
-      // Run this thread to shutdown our filesystem on way out.
-      this.shutdownThread = new SingleFileSystemShutdownThread(getFileSystem());
-    }
-
     @Override
     public void run() {
       try {
@@ -163,12 +147,6 @@ public Object run() {
         });
       } catch (Throwable t) {
         LOG.error("Exception in run", t);
-      } finally {
-        // Run this on the way out.
-        if (this.shutdownThread != null) {
-          this.shutdownThread.start();
-          Threads.shutdown(this.shutdownThread, 30000);
-        }
       }
     }
 
@@ -198,29 +176,6 @@ private void abortRegionServer(String reason, Throwable cause) {
     }
   }
 
-  /**
-   * Alternate shutdown hook. Just shuts down the passed fs, not all as default filesystem hook
-   * does.
-   */
-  static class SingleFileSystemShutdownThread extends Thread {
-    private final FileSystem fs;
-
-    SingleFileSystemShutdownThread(final FileSystem fs) {
-      super("Shutdown of " + fs);
-      this.fs = fs;
-    }
-
-    @Override
-    public void run() {
-      try {
-        LOG.info("Hook closing fs=" + this.fs);
-        this.fs.close();
-      } catch (IOException e) {
-        LOG.warn("Running hook", e);
-      }
-    }
-  }
-
   private void init(final int nMasterNodes, final int numAlwaysStandByMasters,
     final int nRegionNodes, List<Integer> rsPorts, Class<? extends HMaster> masterClass,
     Class<? extends MiniHBaseCluster.MiniHBaseClusterRegionServer> regionserverClass)