diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java index 99708e31628a..46759a1d763a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/BlockingRpcConnection.java @@ -233,7 +233,7 @@ public void cleanup(IOException e) { this.connectionHeaderWithLength = baos.getBuffer(); UserGroupInformation ticket = remoteId.ticket.getUGI(); - this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to " + this.threadName = "BRPC Connection (" + this.rpcClient.socketFactory.hashCode() + ") to " + remoteId.getAddress().toString() + ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName())); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java deleted file mode 100644 index 87e554048f2f..000000000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/DefaultNettyEventLoopConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.ipc; - -import org.apache.hbase.thirdparty.io.netty.channel.Channel; -import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; -import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory; - -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.util.Pair; - -/** - * The default netty event loop config - */ -@InterfaceAudience.Private -class DefaultNettyEventLoopConfig { - - public static final Pair> GROUP_AND_CHANNEL_CLASS = Pair - .> newPair( - new NioEventLoopGroup(0, - new DefaultThreadFactory("Default-IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY)), - NioSocketChannel.class); -} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java index c4f70b05aaa1..f7a65e4d6fff 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClient.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -54,12 +54,15 @@ public class NettyRpcClient extends AbstractRpcClient { public NettyRpcClient(Configuration configuration, String clusterId, SocketAddress localAddress, MetricsConnection metrics) { super(configuration, clusterId, localAddress, metrics); - Pair> groupAndChannelClass = NettyRpcClientConfigHelper - .getEventLoopConfig(conf); + Pair> groupAndChannelClass = + NettyRpcClientConfigHelper.getEventLoopConfig(conf); if (groupAndChannelClass == null) { // Use our own EventLoopGroup. - this.group = new NioEventLoopGroup(0, - new DefaultThreadFactory("IPC-NioEventLoopGroup", true, Thread.MAX_PRIORITY)); + int threadCount = conf.getInt( + NettyRpcClientConfigHelper.HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0); + this.group = new NioEventLoopGroup(threadCount, + new DefaultThreadFactory("RPCClient(own)-NioEventLoopGroup", true, + Thread.NORM_PRIORITY)); this.channelClass = NioSocketChannel.class; this.shutdownGroupWhenClose = true; } else { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java index e7793399cf38..6107183dd460 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcClientConfigHelper.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,18 +17,18 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; - -import org.apache.hbase.thirdparty.io.netty.channel.Channel; -import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; - import java.util.HashMap; import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.util.Pair; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; +import org.apache.hbase.thirdparty.io.netty.channel.Channel; +import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; +import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory; /** * Helper class for passing config to {@link NettyRpcClient}. @@ -39,15 +39,27 @@ * @since 2.0.0 */ @InterfaceAudience.Public -public class NettyRpcClientConfigHelper { +public final class NettyRpcClientConfigHelper { public static final String EVENT_LOOP_CONFIG = "hbase.rpc.client.event-loop.config"; + /** + * Name of property to change netty rpc client eventloop thread count. Default is 0. + * Tests may set this down from unlimited. + */ + public static final String HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY = + "hbase.netty.eventloop.rpcclient.thread.count"; + private static final String CONFIG_NAME = "global-event-loop"; private static final Map>> EVENT_LOOP_CONFIG_MAP = new HashMap<>(); + /** + * Shutdown constructor. + */ + private NettyRpcClientConfigHelper() {} + /** * Set the EventLoopGroup and channel class for {@code AsyncRpcClient}. */ @@ -71,12 +83,14 @@ public static void createEventLoopPerClient(Configuration conf) { static Pair> getEventLoopConfig(Configuration conf) { String name = conf.get(EVENT_LOOP_CONFIG); if (name == null) { - return DefaultNettyEventLoopConfig.GROUP_AND_CHANNEL_CLASS; + int threadCount = conf.getInt(HBASE_NETTY_EVENTLOOP_RPCCLIENT_THREADCOUNT_KEY, 0); + return new Pair<>(new NioEventLoopGroup(threadCount, + new DefaultThreadFactory("RPCClient-NioEventLoopGroup", true, + Thread.NORM_PRIORITY)), NioSocketChannel.class); } if (StringUtils.isBlank(name)) { return null; } return EVENT_LOOP_CONFIG_MAP.get(name); } - } diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml index 99d2ab8d1fbb..3a2b0520de8b 100644 --- a/hbase-client/src/test/resources/hbase-site.xml +++ b/hbase-client/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.defaults.for.version.skip true @@ -29,4 +45,119 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-client/src/test/resources/hdfs-site.xml b/hbase-client/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-client/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-endpoint/src/test/resources/hbase-site.xml b/hbase-endpoint/src/test/resources/hbase-site.xml index 99d2ab8d1fbb..3a2b0520de8b 100644 --- a/hbase-endpoint/src/test/resources/hbase-site.xml +++ b/hbase-endpoint/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.defaults.for.version.skip true @@ -29,4 +45,119 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-endpoint/src/test/resources/hdfs-site.xml b/hbase-endpoint/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-endpoint/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java index 4ada5990cbcb..00d01a861d75 100644 --- a/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/client/example/HttpProxyExample.java @@ -229,6 +229,7 @@ public void start() throws InterruptedException, ExecutionException { channelGroup = new DefaultChannelGroup(GlobalEventExecutor.INSTANCE); serverChannel = new ServerBootstrap().group(bossGroup, workerGroup) .channel(NioServerSocketChannel.class).childOption(ChannelOption.TCP_NODELAY, true) + .childOption(ChannelOption.SO_REUSEADDR, true) .childHandler(new ChannelInitializer() { @Override diff --git a/hbase-examples/src/test/resources/hbase-site.xml b/hbase-examples/src/test/resources/hbase-site.xml index ab4d1cd0b733..6d6f4bf3c09e 100644 --- a/hbase-examples/src/test/resources/hbase-site.xml +++ b/hbase-examples/src/test/resources/hbase-site.xml @@ -21,8 +21,139 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.defaults.for.version.skip true + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-examples/src/test/resources/hdfs-site.xml b/hbase-examples/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-examples/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-mapreduce/src/test/resources/hbase-site.xml b/hbase-mapreduce/src/test/resources/hbase-site.xml index 64a196443589..a2aee0cc1998 100644 --- a/hbase-mapreduce/src/test/resources/hbase-site.xml +++ b/hbase-mapreduce/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.regionserver.msginterval 1000 @@ -158,4 +174,119 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-mapreduce/src/test/resources/hdfs-site.xml b/hbase-mapreduce/src/test/resources/hdfs-site.xml index 03be0c72c6d8..9230105a0fd4 100644 --- a/hbase-mapreduce/src/test/resources/hdfs-site.xml +++ b/hbase-mapreduce/src/test/resources/hdfs-site.xml @@ -29,4 +29,28 @@ dfs.namenode.fs-limits.min-block-size 0 - \ No newline at end of file + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-procedure/src/test/resources/hbase-site.xml b/hbase-procedure/src/test/resources/hbase-site.xml index 3709a719bae4..c938f285234b 100644 --- a/hbase-procedure/src/test/resources/hbase-site.xml +++ b/hbase-procedure/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.defaults.for.version.skip true @@ -45,4 +61,119 @@ WARNING: Doing so may expose you to additional risk of data loss! + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-procedure/src/test/resources/hdfs-site.xml b/hbase-procedure/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-procedure/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-rest/src/test/resources/hdfs-site.xml b/hbase-rest/src/test/resources/hdfs-site.xml index 03be0c72c6d8..9230105a0fd4 100644 --- a/hbase-rest/src/test/resources/hdfs-site.xml +++ b/hbase-rest/src/test/resources/hdfs-site.xml @@ -29,4 +29,28 @@ dfs.namenode.fs-limits.min-block-size 0 - \ No newline at end of file + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-rsgroup/src/test/resources/hbase-site.xml index 99d2ab8d1fbb..3a2b0520de8b 100644 --- a/hbase-rsgroup/src/test/resources/hbase-site.xml +++ b/hbase-rsgroup/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.defaults.for.version.skip true @@ -29,4 +45,119 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-rsgroup/src/test/resources/hdfs-site.xml b/hbase-rsgroup/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-rsgroup/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java index bba1bede118f..eab2a0ec85c7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/NettyRpcServer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,44 +17,42 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap; -import org.apache.hbase.thirdparty.io.netty.channel.Channel; -import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer; -import org.apache.hbase.thirdparty.io.netty.channel.ChannelOption; -import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline; -import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel; -import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup; -import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup; -import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; -import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel; -import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder; -import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory; -import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor; - import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; import java.util.List; import java.util.concurrent.CountDownLatch; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.Server; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.security.HBasePolicyProvider; +import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.protobuf.BlockingService; import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors.MethodDescriptor; import org.apache.hbase.thirdparty.com.google.protobuf.Message; -import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; +import org.apache.hbase.thirdparty.io.netty.bootstrap.ServerBootstrap; +import org.apache.hbase.thirdparty.io.netty.channel.Channel; +import org.apache.hbase.thirdparty.io.netty.channel.ChannelInitializer; +import org.apache.hbase.thirdparty.io.netty.channel.ChannelOption; +import org.apache.hbase.thirdparty.io.netty.channel.ChannelPipeline; +import org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup; +import org.apache.hbase.thirdparty.io.netty.channel.ServerChannel; +import org.apache.hbase.thirdparty.io.netty.channel.group.ChannelGroup; +import org.apache.hbase.thirdparty.io.netty.channel.group.DefaultChannelGroup; +import org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel; +import org.apache.hbase.thirdparty.io.netty.handler.codec.FixedLengthFrameDecoder; +import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory; +import org.apache.hbase.thirdparty.io.netty.util.concurrent.GlobalEventExecutor; /** * An RPC server with Netty4 implementation. @@ -62,9 +60,16 @@ */ @InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.CONFIG}) public class NettyRpcServer extends RpcServer { - public static final Logger LOG = LoggerFactory.getLogger(NettyRpcServer.class); + /** + * Name of property to change netty rpc server eventloop thread count. Default is 0. + * Tests may set this down from unlimited. + */ + public static final String HBASE_NETTY_EVENTLOOP_RPCSERVER_THREADCOUNT_KEY = + "hbase.netty.eventloop.rpcserver.thread.count"; + private static final int EVENTLOOP_THREADCOUNT_DEFAULT = 0; + private final InetSocketAddress bindAddress; private final CountDownLatch closed = new CountDownLatch(1); @@ -84,13 +89,17 @@ public NettyRpcServer(Server server, String name, List() { @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 37adbcad70d6..72eb3515b13b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1680,7 +1680,7 @@ private Map> doClose(boolean abort, MonitoredTask statu if (!stores.isEmpty()) { // initialize the thread pool for closing stores in parallel. ThreadPoolExecutor storeCloserThreadPool = - getStoreOpenAndCloseThreadPool("StoreCloserThread-" + + getStoreOpenAndCloseThreadPool("StoreCloser-" + getRegionInfo().getRegionNameAsString()); CompletionService>> completionService = new ExecutorCompletionService<>(storeCloserThreadPool); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 167bb3ee7430..7e5d88b3a1d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -569,7 +569,7 @@ private List openStoreFiles(Collection files, boolean } // initialize the thread pool for opening store files in parallel.. ThreadPoolExecutor storeFileOpenerThreadPool = - this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" + + this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpener-" + this.getColumnFamilyName()); CompletionService completionService = new ExecutorCompletionService<>(storeFileOpenerThreadPool); @@ -969,7 +969,7 @@ public ImmutableCollection close() throws IOException { if (!result.isEmpty()) { // initialize the thread pool for closing store files in parallel. ThreadPoolExecutor storeFileCloserThreadPool = this.region - .getStoreFileOpenAndCloseThreadPool("StoreFileCloserThread-" + .getStoreFileOpenAndCloseThreadPool("StoreFileCloser-" + this.getColumnFamilyName()); // close each store file in parallel diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java index 61f57ae87a31..ab709d453034 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java @@ -228,8 +228,10 @@ public AsyncFSWAL(FileSystem fs, Path rootDir, String logDir, String archiveDir, } } else { ThreadPoolExecutor threadPool = - new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), - new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d").setDaemon(true).build()); + new ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), + new ThreadFactoryBuilder().setNameFormat("AsyncFSWAL-%d-" + rootDir.toString()). + setDaemon(true).build()); hasConsumerTask = () -> threadPool.getQueue().peek() == consumer; this.consumeExecutor = threadPool; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java index 9bbc16d8597e..a81c5761c920 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java @@ -10,6 +10,7 @@ */ package org.apache.hadoop.hbase.replication.regionserver; +import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; @@ -27,7 +28,6 @@ import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -39,11 +39,11 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; -import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.LoadQueueItem; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.token.FsDelegationToken; +import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles; +import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.LoadQueueItem; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; @@ -51,16 +51,16 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * It is used for replicating HFile entries. It will first copy parallely all the hfiles to a local * staging directory and then it will use ({@link LoadIncrementalHFiles} to prepare a collection of * {@link LoadQueueItem} which will finally be loaded(replicated) into the table of this cluster. + * Call {@link #close()} when done. */ @InterfaceAudience.Private -public class HFileReplicator { +public class HFileReplicator implements Closeable { /** Maximum number of threads to allow in pool to copy hfiles during replication */ public static final String REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY = "hbase.replication.bulkload.copy.maxthreads"; @@ -109,7 +109,8 @@ public HFileReplicator(Configuration sourceClusterConf, REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT); this.exec = Threads.getBoundedCachedThreadPool(maxCopyThreads, 60, TimeUnit.SECONDS, new ThreadFactoryBuilder().setDaemon(true) - .setNameFormat("HFileReplicationCallable-%1$d").build()); + .setNameFormat("HFileReplicationCopier-%1$d-" + this.sourceBaseNamespaceDirPath). + build()); this.copiesPerThread = conf.getInt(REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY, REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT); @@ -117,6 +118,13 @@ public HFileReplicator(Configuration sourceClusterConf, sinkFs = FileSystem.get(conf); } + @Override + public void close() throws IOException { + if (this.exec != null) { + this.exec.shutdown(); + } + } + public Void replicate() throws IOException { // Copy all the hfiles to the local file system Map tableStagingDirsMap = copyHFilesToStagingDir(); @@ -132,8 +140,7 @@ public Void replicate() throws IOException { loadHFiles = new LoadIncrementalHFiles(conf); loadHFiles.setClusterIds(sourceClusterIds); } catch (Exception e) { - LOG.error("Failed to initialize LoadIncrementalHFiles for replicating bulk loaded" - + " data.", e); + LOG.error("Failed initialize LoadIncrementalHFiles for replicating bulk loaded data.", e); throw new IOException(e); } Configuration newConf = HBaseConfiguration.create(conf); @@ -148,19 +155,15 @@ public Void replicate() throws IOException { loadHFiles.prepareHFileQueue(stagingDir, table, queue, false); if (queue.isEmpty()) { - LOG.warn("Replication process did not find any files to replicate in directory " - + stagingDir.toUri()); + LOG.warn("Did not find any files to replicate in directory {}", stagingDir.toUri()); return null; } try (RegionLocator locator = connection.getRegionLocator(tableName)) { - fsDelegationToken.acquireDelegationToken(sinkFs); - // Set the staging directory which will be used by LoadIncrementalHFiles for loading the // data loadHFiles.setBulkToken(stagingDir.toString()); - doBulkLoad(loadHFiles, table, queue, locator, maxRetries); } finally { cleanup(stagingDir.toString(), table); @@ -177,13 +180,11 @@ private void doBulkLoad(LoadIncrementalHFiles loadHFiles, Table table, // need to reload split keys each iteration. startEndKeys = locator.getStartEndKeys(); if (count != 0) { - LOG.warn("Error occurred while replicating HFiles, retry attempt " + count + " with " - + queue.size() + " files still remaining to replicate."); + LOG.warn("Error replicating HFiles; retry={} with {} remaining.", count, queue.size()); } if (maxRetries != 0 && count >= maxRetries) { - throw new IOException("Retry attempted " + count - + " times without completing, bailing out."); + throw new IOException("Retry attempted " + count + " times without completing, bailing."); } count++; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java index ae0a732499e7..e910e530131d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java @@ -29,7 +29,6 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -257,18 +256,13 @@ public void replicateEntries(List entries, final CellScanner cells, List>>>> entry : bulkLoadsPerClusters.entrySet()) { Map>>> bulkLoadHFileMap = entry.getValue(); if (bulkLoadHFileMap != null && !bulkLoadHFileMap.isEmpty()) { - if(LOG.isDebugEnabled()) { - LOG.debug("Started replicating bulk loaded data from cluster ids: {}.", - entry.getKey().toString()); - } - HFileReplicator hFileReplicator = - new HFileReplicator(this.provider.getConf(this.conf, replicationClusterId), + LOG.debug("Replicating {} bulk loaded data", entry.getKey().toString()); + Configuration providerConf = this.provider.getConf(this.conf, replicationClusterId); + try (HFileReplicator hFileReplicator = new HFileReplicator(providerConf, sourceBaseNamespaceDirPath, sourceHFileArchiveDirPath, bulkLoadHFileMap, conf, - getConnection(), entry.getKey()); - hFileReplicator.replicate(); - if(LOG.isDebugEnabled()) { - LOG.debug("Finished replicating bulk loaded data from cluster id: {}", - entry.getKey().toString()); + getConnection(), entry.getKey())) { + hFileReplicator.replicate(); + LOG.debug("Finished replicating {} bulk loaded data", entry.getKey().toString()); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java index 683d17529e5f..c0189c83aae2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java @@ -110,7 +110,7 @@ public static List createRegions(final Configuration conf, final Pat if (newRegions == null) return null; int regionNumber = newRegions.length; ThreadPoolExecutor exec = getRegionOpenAndInitThreadPool(conf, - "RegionOpenAndInitThread-" + tableDescriptor.getTableName(), regionNumber); + "RegionOpenAndInit-" + tableDescriptor.getTableName(), regionNumber); try { return createRegions(exec, conf, rootDir, tableDescriptor, newRegions, task); } finally { @@ -230,8 +230,8 @@ static ThreadPoolExecutor getRegionOpenAndInitThreadPool(final Configuration con final String threadNamePrefix, int regionNumber) { int maxThreads = Math.min(regionNumber, conf.getInt( "hbase.hregion.open.and.init.threads.max", 16)); - ThreadPoolExecutor regionOpenAndInitThreadPool = Threads - .getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS, + ThreadPoolExecutor regionOpenAndInitThreadPool = Threads. + getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS, Threads.newDaemonThreadFactory(threadNamePrefix)); return regionOpenAndInitThreadPool; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java index 3e7b4882e319..3e247f3061ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/NettyEventLoopGroupConfig.java @@ -27,9 +27,7 @@ import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel; import org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; import org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultThreadFactory; - import java.util.concurrent.ThreadFactory; - import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; @@ -38,7 +36,6 @@ */ @InterfaceAudience.Private public class NettyEventLoopGroupConfig { - private final EventLoopGroup group; private final Class serverChannelClass; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java index a440cde5ff3e..beedd5e4590c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java @@ -77,8 +77,8 @@ public class TestReplicationBase { protected static Configuration CONF1 = UTIL1.getConfiguration(); protected static Configuration CONF2 = UTIL2.getConfiguration(); - protected static final int NUM_SLAVES1 = 2; - protected static final int NUM_SLAVES2 = 4; + protected static final int NUM_SLAVES1 = 1; + protected static final int NUM_SLAVES2 = 1; protected static final int NB_ROWS_IN_BATCH = 100; protected static final int NB_ROWS_IN_BIG_BATCH = NB_ROWS_IN_BATCH * 10; diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 64a196443589..a317376e77c7 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -23,7 +23,7 @@ hbase.regionserver.msginterval - 1000 + 100 Interval between messages from the RegionServer to HMaster in milliseconds. Default is 15. Set this value low if you want unit tests to be responsive. @@ -35,7 +35,7 @@ hbase.server.thread.wakefrequency - 1000 + 100 Time to sleep in between searches for work (in milliseconds). Used as sleep interval by service threads such as hbase:meta scanner and log roller. @@ -48,11 +48,13 @@ hbase.regionserver.handler.count - 5 + 3 + Default is 30 hbase.regionserver.metahandler.count - 6 + 3 + Default is 20 hbase.ipc.server.read.threadpool.size @@ -83,7 +85,6 @@ hbase.ipc.client.fallback-to-simple-auth-allowed true - hbase.regionserver.info.port -1 @@ -158,4 +159,119 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + + + hbase.netty.eventloop.rpclient.thread.count + 3 + Default is unbounded + + + hbase.netty.eventloop.rpcserver.thread.count + 3 + Default is unbounded + diff --git a/hbase-server/src/test/resources/hdfs-site.xml b/hbase-server/src/test/resources/hdfs-site.xml index 03be0c72c6d8..9230105a0fd4 100644 --- a/hbase-server/src/test/resources/hdfs-site.xml +++ b/hbase-server/src/test/resources/hdfs-site.xml @@ -29,4 +29,28 @@ dfs.namenode.fs-limits.min-block-size 0 - \ No newline at end of file + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-server/src/test/resources/log4j.properties b/hbase-server/src/test/resources/log4j.properties index 785371d9587d..8e590b177c92 100644 --- a/hbase-server/src/test/resources/log4j.properties +++ b/hbase-server/src/test/resources/log4j.properties @@ -66,6 +66,7 @@ log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN +log4j.logger.io.netty.channel=DEBUG # Enable this to get detailed connection error/retry logging. # log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE log4j.logger.org.apache.directory=WARN diff --git a/hbase-shell/src/test/resources/hbase-site.xml b/hbase-shell/src/test/resources/hbase-site.xml index 99d2ab8d1fbb..3a2b0520de8b 100644 --- a/hbase-shell/src/test/resources/hbase-site.xml +++ b/hbase-shell/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.defaults.for.version.skip true @@ -29,4 +45,119 @@ hbase.hconnection.threads.keepalivetime 3 + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-shell/src/test/resources/hdfs-site.xml b/hbase-shell/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-shell/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/hbase-thrift/src/test/resources/hbase-site.xml b/hbase-thrift/src/test/resources/hbase-site.xml index b3fb0d90c50c..a1e89ce70beb 100644 --- a/hbase-thrift/src/test/resources/hbase-site.xml +++ b/hbase-thrift/src/test/resources/hbase-site.xml @@ -21,6 +21,22 @@ */ --> + + hbase.regionserver.msginterval + 100 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.server.thread.wakefrequency + 1000 + 100 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + hbase.regionserver.msginterval 1000 @@ -154,4 +170,119 @@ Enable replay sanity checks on procedure tests. + + hbase.regionserver.handler.count + 3 + Default is 30 + + + hbase.regionserver.metahandler.count + 3 + Default is 20 + + + hbase.netty.worker.count + 3 + Default is 0 + + + hbase.hconnection.threads.max + 6 + Default is 256 + + + hbase.htable.threads.max + 3 + Default is MAX_INTEGER + + + hbase.region.replica.replication.threads.max + 7 + Default is 256 + + + hbase.rest.threads.max + 5 + Default is 100 + + + hbase.replication.bulkload.copy.maxthreads + 3 + Default is 10 + + + hbase.loadincremental.threads.max + 1 + Default is # of CPUs + + + hbase.hstore.flusher.count + 1 + Default is 2 + + + hbase.oldwals.cleaner.thread.size + 1 + Default is 2 + + + hbase.master.procedure.threads + 5 + Default is at least 16 + + + hbase.procedure.remote.dispatcher.threadpool.size + 3 + Default is 128 + + + hbase.regionserver.executor.closeregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openregion.threads + 1 + Default is 3 + + + hbase.regionserver.executor.openpriorityregion.threads + 1 + Default is 3 + + + hbase.storescanner.parallel.seek.threads + 3 + Default is 10 + + + hbase.hfile.compaction.discharger.thread.count + 1 + Default is 10 + + + hbase.regionserver.executor.refresh.peer.threads + 1 + Default is 2 + + + hbase.hregion.open.and.init.threads.max + 3 + Default is 16 or # of Regions + + + hbase.master.handler.count + 7 + Default is 25 + + + hbase.replication.source.maxthreads + + Default is 10 + + + hbase.hconnection.meta.lookup.threads.max + 5 + Default is 128 + diff --git a/hbase-thrift/src/test/resources/hdfs-site.xml b/hbase-thrift/src/test/resources/hdfs-site.xml new file mode 100644 index 000000000000..9230105a0fd4 --- /dev/null +++ b/hbase-thrift/src/test/resources/hdfs-site.xml @@ -0,0 +1,56 @@ + + + + + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + dfs.datanode.handler.count + 5 + Default is 10 + + + dfs.namenode.handler.count + 5 + Default is 10 + + + dfs.namenode.service.handler.count + 5 + Default is 10 + + + diff --git a/pom.xml b/pom.xml index 8886cc4c9307..0559f67a1333 100755 --- a/pom.xml +++ b/pom.xml @@ -584,8 +584,17 @@ ${surefire.testFailureIgnore} ${surefire.timeout} ${test.output.tofile} + ${test.build.classes} + + 3 -enableassertions -Dhbase.build.id=${build.id} -Xmx${surefire.Xmx} -Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Djdk.net.URLClassPath.disableClassPathURLCheck=true -Dorg.apache.hbase.thirdparty.io.netty.leakDetection.level=advanced + -Dio.netty.eventLoopThreads=3 -enableassertions -Xmx${surefire.cygwinXmx} -Djava.security.egd=file:/dev/./urandom -Djava.net.preferIPv4Stack=true