diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java index c69f17c5263f..3592128874a1 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java @@ -285,6 +285,7 @@ public String getRack(ServerName server) { serversPerHost[i] = new int[serversPerHostList.get(i).size()]; for (int j = 0; j < serversPerHost[i].length; j++) { serversPerHost[i][j] = serversPerHostList.get(i).get(j); + LOG.debug("server {} is on host {}",serversPerHostList.get(i).get(j), i); } if (serversPerHost[i].length > 1) { multiServersPerHost = true; @@ -295,6 +296,7 @@ public String getRack(ServerName server) { serversPerRack[i] = new int[serversPerRackList.get(i).size()]; for (int j = 0; j < serversPerRack[i].length; j++) { serversPerRack[i][j] = serversPerRackList.get(i).get(j); + LOG.info("server {} is on rack {}",serversPerRackList.get(i).get(j), i); } } diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index eaa923e74fcf..dcfa73b7abc0 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -350,7 +350,8 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { } if (idleRegionServerExist(cluster)){ - LOG.info("Running balancer because cluster has idle server(s)."); + LOG.info("Running balancer because cluster has idle server(s)."+ + " function cost={}", functionCost()); return true; } @@ -510,9 +511,9 @@ protected List balanceTable(TableName tableName, Map> serverMap, List balancedCluster = reconcile(list, plans, serverMap); // Print out the cluster loads to make debugging easier. - LOG.info("Mock Balance : " + printMock(balancedCluster)); + LOG.info("Mock after Balance : " + printMock(balancedCluster)); if (assertFullyBalanced) { assertClusterAsBalanced(balancedCluster); @@ -95,4 +95,40 @@ protected void testWithCluster(Map> serverMap, } } } + + protected void testWithClusterWithIteration(Map> serverMap, + RackManager rackManager, boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) { + List list = convertToList(serverMap); + LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list)); + + loadBalancer.setRackManager(rackManager); + // Run the balancer. + Map>> LoadOfAllTable = (Map) mockClusterServersWithTables(serverMap); + List plans = loadBalancer.balanceCluster(LoadOfAllTable); + assertNotNull("Initial cluster balance should produce plans.", plans); + + List balancedCluster = null; + // Run through iteration until done. Otherwise will be killed as test time out + while (plans != null && (assertFullyBalanced || assertFullyBalancedForReplicas)) { + // Apply the plan to the mock cluster. + balancedCluster = reconcile(list, plans, serverMap); + + // Print out the cluster loads to make debugging easier. + LOG.info("Mock after balance: " + printMock(balancedCluster)); + + LoadOfAllTable = (Map) mockClusterServersWithTables(serverMap); + plans = loadBalancer.balanceCluster(LoadOfAllTable); + } + + // Print out the cluster loads to make debugging easier. + LOG.info("Mock Final balance: " + printMock(balancedCluster)); + + if (assertFullyBalanced) { + assertNull("Given a requirement to be fully balanced, second attempt at plans should " + + "produce none.", plans); + } + if (assertFullyBalancedForReplicas) { + assertRegionReplicaPlacement(serverMap, rackManager); + } + } } diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaWithRacks.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaWithRacks.java index 3b2c847c5455..92013924cdfa 100644 --- a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaWithRacks.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaWithRacks.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.master.balancer; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -38,24 +39,32 @@ public class TestStochasticLoadBalancerRegionReplicaWithRacks extends Stochastic private static class ForTestRackManager extends RackManager { int numRacks; + Map serverIndexes = new HashMap(); + int numServers = 0; public ForTestRackManager(int numRacks) { this.numRacks = numRacks; } - @Override - public String getRack(ServerName server) { - return "rack_" + (server.hashCode() % numRacks); + @Override public String getRack(ServerName server) { + String key = server.getServerName(); + if (!serverIndexes.containsKey(key)) { + serverIndexes.put(key, numServers++); + } + return "rack_" + serverIndexes.get(key) % numRacks; } } @Test public void testRegionReplicationOnMidClusterWithRacks() { - conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L); + conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 100000000L); + conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f); conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec + // for full balance +// conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.001f); loadBalancer.onConfigurationChange(conf); - int numNodes = 4; + int numNodes = 5; int numRegions = numNodes * 1; int replication = 3; // 3 replicas per region int numRegionsPerServer = 1; @@ -65,6 +74,26 @@ public void testRegionReplicationOnMidClusterWithRacks() { createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables); RackManager rm = new ForTestRackManager(numRacks); - testWithCluster(serverMap, rm, false, true); + testWithClusterWithIteration(serverMap, rm, false, true); + } + + @Test + public void testRegionReplicationOnLargeClusterWithRacks() { + conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false); + conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 5000L); + conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f); + conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 10 * 1000); // 10 sec + loadBalancer.onConfigurationChange(conf); + int numNodes = 100; + int numRegions = numNodes * 30; + int replication = 3; // 3 replicas per region + int numRegionsPerServer = 28; + int numTables = 1; + int numRacks = 4; // all replicas should be on a different rack + Map> serverMap = + createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables); + RackManager rm = new ForTestRackManager(numRacks); + + testWithClusterWithIteration(serverMap, rm, true, true); } }