HBASE-26310 Repro balancer behavior during iterations

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java

@@ -285,6 +285,7 @@ public String getRack(ServerName server) {
       serversPerHost[i] = new int[serversPerHostList.get(i).size()];
       for (int j = 0; j < serversPerHost[i].length; j++) {
         serversPerHost[i][j] = serversPerHostList.get(i).get(j);
+        LOG.debug("server {} is on host {}",serversPerHostList.get(i).get(j), i);
       }
       if (serversPerHost[i].length > 1) {
         multiServersPerHost = true;
@@ -295,6 +296,7 @@ public String getRack(ServerName server) {
       serversPerRack[i] = new int[serversPerRackList.get(i).size()];
       for (int j = 0; j < serversPerRack[i].length; j++) {
         serversPerRack[i][j] = serversPerRackList.get(i).get(j);
+        LOG.info("server {} is on rack {}",serversPerRackList.get(i).get(j), i);
       }
     }
 

hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java

@@ -350,7 +350,8 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) {
     }
 
     if (idleRegionServerExist(cluster)){
-      LOG.info("Running balancer because cluster has idle server(s).");
+      LOG.info("Running balancer because cluster has idle server(s)."+
+        " function cost={}", functionCost());
       return true;
     }
 
@@ -510,9 +511,9 @@ protected List<RegionPlan> balanceTable(TableName tableName, Map<ServerName,
       LOG.info("Finished computing new moving plan. Computation took {} ms" +
           " to try {} different iterations.  Found a solution that moves " +
           "{} regions; Going from a computed imbalance of {}" +
-          " to a new imbalance of {}. ",
+          " to a new imbalance of {}. funtionCost={}",
         endTime - startTime, step, plans.size(),
-        initCost / sumMultiplier, currentCost / sumMultiplier);
+        initCost / sumMultiplier, currentCost / sumMultiplier, functionCost());
       sendRegionPlansToRingBuffer(plans, currentCost, initCost, initFunctionTotalCosts, step);
       return plans;
     }

hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/StochasticBalancerTestBase.java

@@ -80,7 +80,7 @@ protected void testWithCluster(Map<ServerName, List<RegionInfo>> serverMap,
       List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);
 
       // Print out the cluster loads to make debugging easier.
-      LOG.info("Mock Balance : " + printMock(balancedCluster));
+      LOG.info("Mock after Balance : " + printMock(balancedCluster));
 
       if (assertFullyBalanced) {
         assertClusterAsBalanced(balancedCluster);
@@ -95,4 +95,40 @@ protected void testWithCluster(Map<ServerName, List<RegionInfo>> serverMap,
       }
     }
   }
+
+  protected void testWithClusterWithIteration(Map<ServerName, List<RegionInfo>> serverMap,
+    RackManager rackManager, boolean assertFullyBalanced, boolean assertFullyBalancedForReplicas) {
+    List<ServerAndLoad> list = convertToList(serverMap);
+    LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
+
+    loadBalancer.setRackManager(rackManager);
+    // Run the balancer.
+    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable = (Map) mockClusterServersWithTables(serverMap);
+    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
+    assertNotNull("Initial cluster balance should produce plans.", plans);
+
+    List<ServerAndLoad> balancedCluster = null;
+    // Run through iteration until done. Otherwise will be killed as test time out
+    while (plans != null && (assertFullyBalanced || assertFullyBalancedForReplicas)) {
+      // Apply the plan to the mock cluster.
+      balancedCluster = reconcile(list, plans, serverMap);
+
+      // Print out the cluster loads to make debugging easier.
+      LOG.info("Mock after balance: " + printMock(balancedCluster));
+
+      LoadOfAllTable = (Map) mockClusterServersWithTables(serverMap);
+      plans = loadBalancer.balanceCluster(LoadOfAllTable);
+    }
+
+    // Print out the cluster loads to make debugging easier.
+    LOG.info("Mock Final balance: " + printMock(balancedCluster));
+
+    if (assertFullyBalanced) {
+      assertNull("Given a requirement to be fully balanced, second attempt at plans should " +
+        "produce none.", plans);
+    }
+    if (assertFullyBalancedForReplicas) {
+      assertRegionReplicaPlacement(serverMap, rackManager);
+    }
+  }
 }

hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancerRegionReplicaWithRacks.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.balancer;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
@@ -38,24 +39,32 @@ public class TestStochasticLoadBalancerRegionReplicaWithRacks extends Stochastic
 
   private static class ForTestRackManager extends RackManager {
     int numRacks;
+    Map<String, Integer> serverIndexes = new HashMap<String, Integer>();
+    int numServers = 0;
 
     public ForTestRackManager(int numRacks) {
       this.numRacks = numRacks;
     }
 
-    @Override
-    public String getRack(ServerName server) {
-      return "rack_" + (server.hashCode() % numRacks);
+    @Override public String getRack(ServerName server) {
+      String key = server.getServerName();
+      if (!serverIndexes.containsKey(key)) {
+        serverIndexes.put(key, numServers++);
+      }
+      return "rack_" + serverIndexes.get(key) % numRacks;
     }
   }
 
   @Test
   public void testRegionReplicationOnMidClusterWithRacks() {
-    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 10000000L);
+    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 100000000L);
+    conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true);
     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 120 * 1000); // 120 sec
+    // for full balance
+//    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.001f);
     loadBalancer.onConfigurationChange(conf);
-    int numNodes = 4;
+    int numNodes = 5;
     int numRegions = numNodes * 1;
     int replication = 3; // 3 replicas per region
     int numRegionsPerServer = 1;
@@ -65,6 +74,26 @@ public void testRegionReplicationOnMidClusterWithRacks() {
         createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
     RackManager rm = new ForTestRackManager(numRacks);
 
-    testWithCluster(serverMap, rm, false, true);
+    testWithClusterWithIteration(serverMap, rm, false, true);
+  }
+
+  @Test
+  public void testRegionReplicationOnLargeClusterWithRacks() {
+    conf.setBoolean("hbase.master.balancer.stochastic.runMaxSteps", false);
+    conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 5000L);
+    conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f);
+    conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 10 * 1000); // 10 sec
+    loadBalancer.onConfigurationChange(conf);
+    int numNodes = 100;
+    int numRegions = numNodes * 30;
+    int replication = 3; // 3 replicas per region
+    int numRegionsPerServer = 28;
+    int numTables = 1;
+    int numRacks = 4; // all replicas should be on a different rack
+    Map<ServerName, List<RegionInfo>> serverMap =
+      createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
+    RackManager rm = new ForTestRackManager(numRacks);
+
+    testWithClusterWithIteration(serverMap, rm, true, true);
   }
 }