HBASE-23956 Use less resources running tests

hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java

@@ -233,7 +233,7 @@ public void cleanup(IOException e) {
     this.connectionHeaderWithLength = baos.getBuffer();
 
     UserGroupInformation ticket = remoteId.ticket.getUGI();
-    this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to "
+    this.threadName = "BRPC Connection (" + this.rpcClient.socketFactory.hashCode() + ") to "
         + remoteId.getAddress().toString()
         + ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName()));
 

hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -54,12 +54,15 @@ public class NettyRpcClient extends AbstractRpcClient<NettyRpcConnection> {
   public NettyRpcClient(Configuration configuration, String clusterId, SocketAddress localAddress,
       MetricsConnection metrics) {
     super(configuration, clusterId, localAddress, metrics);
-    Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass = NettyRpcClientConfigHelper
-        .getEventLoopConfig(conf);
+    Pair<EventLoopGroup, Class<? extends Channel>> groupAndChannelClass =
+      NettyRpcClientConfigHelper.getEventLoopConfig(conf);
     if (groupAndChannelClass == null) {
       // Use our own EventLoopGroup.
-      this.group = new NioEventLoopGroup(0,
-          new DefaultThreadFactory("IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY));
+      int threadCount = conf.getInt(
+        NettyRpcClientConfigHelper.HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+      this.group = new NioEventLoopGroup(threadCount,
+          new DefaultThreadFactory("RPCClient(own)-NioEventLoopGroup", true,
+            Thread.NORM_PRIORITY));
       this.channelClass = NioSocketChannel.class;
       this.shutdownGroupWhenClose = true;
     } else {

hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,18 +17,18 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
-import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
-
-import org.apache.hbase.thirdparty.io.netty.channel.Channel;
-import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
-
 import java.util.HashMap;
 import java.util.Map;
-
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hbase.thirdparty.io.netty.channel.Channel;
+import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup;
+import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel;
+import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory;
 
 /**
  * Helper class for passing config to {@link NettyRpcClient}.
@@ -39,15 +39,27 @@
  * @since 2.0.0
  */
 @InterfaceAudience.Public
-public class NettyRpcClientConfigHelper {
+public final class NettyRpcClientConfigHelper {
 
   public static final String EVENT_LOOP_CONFIG = "hbase.rpc.client.event-loop.config";
 
+  /**
+   * Name of property to change netty rpc client eventloop thread count. Default is 0.
+   * Tests may set this down from unlimited.
+   */
+  public static final String HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY =
+    "hbase.netty.eventloop.rpcclient.thread.count";
+
   private static final String CONFIG_NAME = "global-event-loop";
 
   private static final Map<String, Pair<EventLoopGroup, Class<? extends Channel>>>
     EVENT_LOOP_CONFIG_MAP = new HashMap<>();
 
+  /**
+   * Shutdown constructor.
+   */
+  private NettyRpcClientConfigHelper() {}
+
   /**
    * Set the EventLoopGroup and channel class for {@code AsyncRpcClient}.
    */
@@ -71,12 +83,14 @@ public static void createEventLoopPerClient(Configuration conf) {
   static Pair<EventLoopGroup, Class<? extends Channel>> getEventLoopConfig(Configuration conf) {
     String name = conf.get(EVENT_LOOP_CONFIG);
     if (name == null) {
-      return DefaultNettyEventLoopConfig.GROUP_AND_CHANNEL_CLASS;
+      int threadCount = conf.getInt(HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0);
+      return new Pair<>(new NioEventLoopGroup(threadCount,
+        new DefaultThreadFactory("RPCClient-NioEventLoopGroup", true,
+          Thread.NORM_PRIORITY)), NioSocketChannel.class);
     }
     if (StringUtils.isBlank(name)) {
       return null;
     }
     return EVENT_LOOP_CONFIG_MAP.get(name);
   }
-
 }

hbase-client/src/test/resources/hbase-site.xml

@@ -21,6 +21,22 @@
  */
 -->
 <configuration>
+  <property>
+    <name>hbase.regionserver.msginterval</name>
+    <value>100</value>
+    <description>Interval between messages from the RegionServer to HMaster
+     in milliseconds.  Default is 15. Set this value low if you want unit
+     tests to be responsive.
+    </description>
+  </property>
+  <property>
+    <name>hbase.server.thread.wakefrequency</name>
+    <value>1000</value>
+    <value>100</value>
+    <description>Time to sleep in between searches for work (in milliseconds).
+     Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+    </description>
+  </property>
   <property>
     <name>hbase.defaults.for.version.skip</name>
     <value>true</value>
@@ -29,4 +45,119 @@
     <name>hbase.hconnection.threads.keepalivetime</name>
     <value>3</value>
   </property>
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>3</value>
+    <description>Default is 30</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.metahandler.count</name>
+    <value>3</value>
+    <description>Default is 20</description>
+   </property>
+  <property>
+    <name>hbase.netty.worker.count</name>
+    <value>3</value>
+    <description>Default is 0</description>
+  </property>
+  <property>
+    <name>hbase.hconnection.threads.max</name>
+    <value>6</value>
+    <description>Default is 256</description>
+  </property>
+  <property>
+    <name>hbase.htable.threads.max</name>
+    <value>3</value>
+    <description>Default is MAX_INTEGER</description>
+  </property>
+  <property>
+    <name>hbase.region.replica.replication.threads.max</name>
+    <value>7</value>
+    <description>Default is 256</description>
+  </property>
+  <property>
+    <name>hbase.rest.threads.max</name>
+    <value>5</value>
+    <description>Default is 100</description>
+  </property>
+  <property>
+    <name>hbase.replication.bulkload.copy.maxthreads</name>
+    <value>3</value>
+    <description>Default is 10</description>
+  </property>
+  <property>
+    <name>hbase.loadincremental.threads.max</name>
+    <value>1</value>
+    <description>Default is # of CPUs</description>
+  </property>
+  <property>
+    <name>hbase.hstore.flusher.count</name>
+    <value>1</value>
+    <description>Default is 2</description>
+  </property>
+  <property>
+    <name>hbase.oldwals.cleaner.thread.size</name>
+    <value>1</value>
+    <description>Default is 2</description>
+  </property>
+  <property>
+    <name>hbase.master.procedure.threads</name>
+    <value>5</value>
+    <description>Default is at least 16</description>
+  </property>
+  <property>
+    <name>hbase.procedure.remote.dispatcher.threadpool.size</name>
+    <value>3</value>
+    <description>Default is 128</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.executor.closeregion.threads</name>
+    <value>1</value>
+    <description>Default is 3</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.executor.openregion.threads</name>
+    <value>1</value>
+    <description>Default is 3</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.executor.openpriorityregion.threads</name>
+    <value>1</value>
+    <description>Default is 3</description>
+  </property>
+  <property>
+    <name>hbase.storescanner.parallel.seek.threads</name>
+    <value>3</value>
+    <description>Default is 10</description>
+  </property>
+  <property>
+    <name>hbase.hfile.compaction.discharger.thread.count</name>
+    <value>1</value>
+    <description>Default is 10</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.executor.refresh.peer.threads</name>
+    <value>1</value>
+    <description>Default is 2</description>
+  </property>
+  <property>
+    <name>hbase.hregion.open.and.init.threads.max</name>
+    <value>3</value>
+    <description>Default is 16 or # of Regions</description>
+  </property>
+  <property>
+    <name>hbase.master.handler.count</name>
+    <value>7</value>
+    <description>Default is 25</description>
+  </property>
+  <property>
+    <name>hbase.replication.source.maxthreads</name>
+    <value></value>
+    <description>Default is 10</description>
+  </property>
+  <property>
+    <name>hbase.hconnection.meta.lookup.threads.max</name>
+    <value>5</value>
+    <description>Default is 128</description>
+  </property>
 </configuration>

hbase-client/src/test/resources/hdfs-site.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+  <!-- hadoop-2.0.5+'s HDFS-4305 by default enforces a min blocks size
+       of 1024*1024.  Many unit tests that use the hlog use smaller
+       blocks.  Setting this config to 0 to have tests pass -->
+  <property>
+    <name>dfs.namenode.fs-limits.min-block-size</name>
+    <value>0</value>
+  </property>
+  <property>
+    <name>dfs.datanode.handler.count</name>
+    <value>5</value>
+    <description>Default is 10</description>
+  </property>
+  <property>
+    <name>dfs.namenode.handler.count</name>
+    <value>5</value>
+    <description>Default is 10</description>
+  </property>
+  <property>
+    <name>dfs.namenode.service.handler.count</name>
+    <value>5</value>
+    <description>Default is 10</description>
+  </property>
+  <!--
+   Constraining this config makes tests fail.
+  <property>
+    <name>dfs.datanode.max.transfer.threads</name>
+    <value>16</value>
+    <description>Default is 4096. If constrain this
+    too much, tests do not complete.</description>
+  </property>
+  -->
+</configuration>