HDFS-17641. Add badly distributed blocks metric

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -304,6 +304,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
 | `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat |
 | `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) |
 | `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 |
+| `BadlyDistributedBlocks` | Current number of blocks that are badly distributed across racks. |
 | `HighestPriorityLowRedundancyReplicatedBlocks` | Current number of non-corrupt, low redundancy replicated blocks with the highest risk of loss (have 0 or 1 replica). Will be recovered with the highest priority. |
 | `HighestPriorityLowRedundancyECBlocks` | Current number of non-corrupt, low redundancy EC blocks with the highest risk of loss. Will be recovered with the highest priority. |
 | `NumFilesUnderConstruction` | Current number of files under construction |

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java

@@ -38,24 +38,28 @@ public final class ECBlockGroupStats {
   private final long missingBlockGroups;
   private final long bytesInFutureBlockGroups;
   private final long pendingDeletionBlocks;
+  private final long badlyDistributedBlocks;
   private final Long highestPriorityLowRedundancyBlocks;
 
   public ECBlockGroupStats(long lowRedundancyBlockGroups,
       long corruptBlockGroups, long missingBlockGroups,
-      long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
+      long bytesInFutureBlockGroups, long pendingDeletionBlocks,
+      long badlyDistributedBlocks) {
     this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
-        bytesInFutureBlockGroups, pendingDeletionBlocks, null);
+        bytesInFutureBlockGroups, pendingDeletionBlocks,
+        badlyDistributedBlocks, null);
   }
 
   public ECBlockGroupStats(long lowRedundancyBlockGroups,
       long corruptBlockGroups, long missingBlockGroups,
       long bytesInFutureBlockGroups, long pendingDeletionBlocks,
-      Long highestPriorityLowRedundancyBlocks) {
+      long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
     this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
     this.corruptBlockGroups = corruptBlockGroups;
     this.missingBlockGroups = missingBlockGroups;
     this.bytesInFutureBlockGroups = bytesInFutureBlockGroups;
     this.pendingDeletionBlocks = pendingDeletionBlocks;
+    this.badlyDistributedBlocks = badlyDistributedBlocks;
     this.highestPriorityLowRedundancyBlocks
         = highestPriorityLowRedundancyBlocks;
   }
@@ -80,6 +84,10 @@ public long getPendingDeletionBlocks() {
     return pendingDeletionBlocks;
   }
 
+  public long getBadlyDistributedBlocks() {
+    return badlyDistributedBlocks;
+  }
+
   public boolean hasHighestPriorityLowRedundancyBlocks() {
     return getHighestPriorityLowRedundancyBlocks() != null;
   }
@@ -99,7 +107,8 @@ public String toString() {
         .append(", BytesInFutureBlockGroups=").append(
             getBytesInFutureBlockGroups())
         .append(", PendingDeletionBlocks=").append(
-            getPendingDeletionBlocks());
+            getPendingDeletionBlocks())
+        .append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
     if (hasHighestPriorityLowRedundancyBlocks()) {
       statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
           .append(getHighestPriorityLowRedundancyBlocks());
@@ -116,6 +125,7 @@ public int hashCode() {
         .append(missingBlockGroups)
         .append(bytesInFutureBlockGroups)
         .append(pendingDeletionBlocks)
+        .append(badlyDistributedBlocks)
         .append(highestPriorityLowRedundancyBlocks)
         .toHashCode();
   }
@@ -135,6 +145,7 @@ public boolean equals(Object o) {
         .append(missingBlockGroups, other.missingBlockGroups)
         .append(bytesInFutureBlockGroups, other.bytesInFutureBlockGroups)
         .append(pendingDeletionBlocks, other.pendingDeletionBlocks)
+        .append(badlyDistributedBlocks, other.badlyDistributedBlocks)
         .append(highestPriorityLowRedundancyBlocks,
             other.highestPriorityLowRedundancyBlocks)
         .isEquals();
@@ -151,6 +162,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
     long missingBlockGroups = 0;
     long bytesInFutureBlockGroups = 0;
     long pendingDeletionBlocks = 0;
+    long badlyDistributedBlocks = 0;
     long highestPriorityLowRedundancyBlocks = 0;
     boolean hasHighestPriorityLowRedundancyBlocks = false;
 
@@ -160,6 +172,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
       missingBlockGroups += stat.getMissingBlockGroups();
       bytesInFutureBlockGroups += stat.getBytesInFutureBlockGroups();
       pendingDeletionBlocks += stat.getPendingDeletionBlocks();
+      badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
       if (stat.hasHighestPriorityLowRedundancyBlocks()) {
         hasHighestPriorityLowRedundancyBlocks = true;
         highestPriorityLowRedundancyBlocks +=
@@ -169,9 +182,10 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
     if (hasHighestPriorityLowRedundancyBlocks) {
       return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
           missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
-          highestPriorityLowRedundancyBlocks);
+          badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
     }
     return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
-        missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
+        missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
+        badlyDistributedBlocks);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java

@@ -37,27 +37,30 @@ public final class ReplicatedBlockStats {
   private final long missingReplicationOneBlocks;
   private final long bytesInFutureBlocks;
   private final long pendingDeletionBlocks;
+  private final long badlyDistributedBlocks;
   private final Long highestPriorityLowRedundancyBlocks;
 
   public ReplicatedBlockStats(long lowRedundancyBlocks,
       long corruptBlocks, long missingBlocks,
       long missingReplicationOneBlocks, long bytesInFutureBlocks,
-      long pendingDeletionBlocks) {
+      long pendingDeletionBlocks, long badlyDistributedBlocks) {
     this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
         missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
-        null);
+        badlyDistributedBlocks, null);
   }
 
   public ReplicatedBlockStats(long lowRedundancyBlocks,
       long corruptBlocks, long missingBlocks,
       long missingReplicationOneBlocks, long bytesInFutureBlocks,
-      long pendingDeletionBlocks, Long highestPriorityLowRedundancyBlocks) {
+      long pendingDeletionBlocks, long badlyDistributedBlocks,
+      Long highestPriorityLowRedundancyBlocks) {
     this.lowRedundancyBlocks = lowRedundancyBlocks;
     this.corruptBlocks = corruptBlocks;
     this.missingBlocks = missingBlocks;
     this.missingReplicationOneBlocks = missingReplicationOneBlocks;
     this.bytesInFutureBlocks = bytesInFutureBlocks;
     this.pendingDeletionBlocks = pendingDeletionBlocks;
+    this.badlyDistributedBlocks = badlyDistributedBlocks;
     this.highestPriorityLowRedundancyBlocks
         = highestPriorityLowRedundancyBlocks;
   }
@@ -86,6 +89,10 @@ public long getPendingDeletionBlocks() {
     return pendingDeletionBlocks;
   }
 
+  public long getBadlyDistributedBlocks() {
+    return badlyDistributedBlocks;
+  }
+
   public boolean hasHighestPriorityLowRedundancyBlocks() {
     return getHighestPriorityLowRedundancyBlocks() != null;
   }
@@ -94,6 +101,7 @@ public Long getHighestPriorityLowRedundancyBlocks(){
     return highestPriorityLowRedundancyBlocks;
   }
 
+
   @Override
   public String toString() {
     StringBuilder statsBuilder = new StringBuilder();
@@ -105,7 +113,8 @@ public String toString() {
             getMissingReplicationOneBlocks())
         .append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
         .append(", PendingDeletionBlocks=").append(
-            getPendingDeletionBlocks());
+            getPendingDeletionBlocks())
+        .append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
     if (hasHighestPriorityLowRedundancyBlocks()) {
         statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
             getHighestPriorityLowRedundancyBlocks());
@@ -127,6 +136,7 @@ public static ReplicatedBlockStats merge(
     long missingReplicationOneBlocks = 0;
     long bytesInFutureBlocks = 0;
     long pendingDeletionBlocks = 0;
+    long badlyDistributedBlocks = 0;
     long highestPriorityLowRedundancyBlocks = 0;
     boolean hasHighestPriorityLowRedundancyBlocks = false;
 
@@ -138,6 +148,7 @@ public static ReplicatedBlockStats merge(
       missingReplicationOneBlocks += stat.getMissingReplicationOneBlocks();
       bytesInFutureBlocks += stat.getBytesInFutureBlocks();
       pendingDeletionBlocks += stat.getPendingDeletionBlocks();
+      badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
       if (stat.hasHighestPriorityLowRedundancyBlocks()) {
         hasHighestPriorityLowRedundancyBlocks = true;
         highestPriorityLowRedundancyBlocks +=
@@ -147,10 +158,10 @@ public static ReplicatedBlockStats merge(
     if (hasHighestPriorityLowRedundancyBlocks) {
       return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
           missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
-          pendingDeletionBlocks, highestPriorityLowRedundancyBlocks);
+          pendingDeletionBlocks, badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
     }
     return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
         missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
-        pendingDeletionBlocks);
+        pendingDeletionBlocks, badlyDistributedBlocks);
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -2037,13 +2037,13 @@ public static ReplicatedBlockStats convert(
       return new ReplicatedBlockStats(res.getLowRedundancy(),
           res.getCorruptBlocks(), res.getMissingBlocks(),
           res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
-          res.getPendingDeletionBlocks(),
+          res.getPendingDeletionBlocks(), res.getBadlyDistributedBlocks(),
           res.getHighestPrioLowRedundancyBlocks());
     }
     return new ReplicatedBlockStats(res.getLowRedundancy(),
         res.getCorruptBlocks(), res.getMissingBlocks(),
         res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
-        res.getPendingDeletionBlocks());
+        res.getBadlyDistributedBlocks(), res.getPendingDeletionBlocks());
   }
 
   public static ECBlockGroupStats convert(
@@ -2052,11 +2052,12 @@ public static ECBlockGroupStats convert(
       return new ECBlockGroupStats(res.getLowRedundancy(),
           res.getCorruptBlocks(), res.getMissingBlocks(),
           res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
-          res.getHighestPrioLowRedundancyBlocks());
+          res.getBadlyDistributedBlocks(), res.getHighestPrioLowRedundancyBlocks());
     }
     return new ECBlockGroupStats(res.getLowRedundancy(),
         res.getCorruptBlocks(), res.getMissingBlocks(),
-        res.getBlocksInFuture(), res.getPendingDeletionBlocks());
+        res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
+        res.getBadlyDistributedBlocks());
   }
 
   public static DatanodeReportTypeProto convert(DatanodeReportType t) {
@@ -2525,6 +2526,8 @@ public static GetFsReplicatedBlockStatsResponseProto convert(
         replicatedBlockStats.getBytesInFutureBlocks());
     result.setPendingDeletionBlocks(
         replicatedBlockStats.getPendingDeletionBlocks());
+    result.setBadlyDistributedBlocks(
+        replicatedBlockStats.getBadlyDistributedBlocks());
     if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
       result.setHighestPrioLowRedundancyBlocks(
           replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
@@ -2544,6 +2547,8 @@ public static GetFsECBlockGroupStatsResponseProto convert(
         ecBlockGroupStats.getBytesInFutureBlockGroups());
     result.setPendingDeletionBlocks(
         ecBlockGroupStats.getPendingDeletionBlocks());
+    result.setBadlyDistributedBlocks(
+        ecBlockGroupStats.getBadlyDistributedBlocks());
     if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
       result.setHighestPrioLowRedundancyBlocks(
           ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto

@@ -372,6 +372,7 @@ message GetFsReplicatedBlockStatsResponseProto {
   required uint64 blocks_in_future = 5;
   required uint64 pending_deletion_blocks = 6;
   optional uint64 highest_prio_low_redundancy_blocks = 7;
+  required uint64 badly_distributed_blocks = 8;
 
 }
 
@@ -385,6 +386,7 @@ message GetFsECBlockGroupStatsResponseProto {
   required uint64 blocks_in_future = 4;
   required uint64 pending_deletion_blocks = 5;
   optional uint64 highest_prio_low_redundancy_blocks = 6;
+  required uint64 badly_distributed_blocks = 7;
 }
 
 enum DatanodeReportTypeProto {  // type of the datanode report

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java

@@ -326,6 +326,13 @@ public interface FederationMBean {
    */
   long getNumberOfMissingBlocksWithReplicationFactorOne();
 
+  /**
+   * Gets the total number of badly distributed blocks.
+   *
+   * @return the total number of badly distrubted blocks.
+   */
+  long getNumberOfBadlyDistributedBlocks();
+
   /**
    * Gets the total number of replicated low redundancy blocks on the cluster
    * with the highest risk of loss.

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java

@@ -383,6 +383,16 @@ public long getNumberOfMissingBlocksWithReplicationFactorOne() {
     return 0;
   }
 
+  @Override
+  public long getNumberOfBadlyDistributedBlocks() {
+    try {
+      return getRBFMetrics().getNumberOfBadlyDistributedBlocks();
+    } catch (IOException e) {
+      LOG.debug("Failed to get number of badly distributed blocks", e);
+    }
+    return 0;
+  }
+
   @Override
   public long getHighestPriorityLowRedundancyReplicatedBlocks() {
     try {

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java

@@ -756,6 +756,12 @@ public long getHighestPriorityLowRedundancyReplicatedBlocks() {
         MembershipStats::getHighestPriorityLowRedundancyReplicatedBlocks);
   }
 
+  @Override
+  public long getNumberOfBadlyDistributedBlocks() {
+    return getNameserviceAggregatedLong(
+        MembershipStats::getNumberOfBadlyDistributedBlocks);
+  }
+
   @Override
   public long getHighestPriorityLowRedundancyECBlocks() {
     return getNameserviceAggregatedLong(

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java

@@ -363,6 +363,8 @@ public boolean registerNamenode(NamenodeStatusReport report)
           report.getScheduledReplicationBlocks());
       stats.setNumberOfMissingBlocksWithReplicationFactorOne(
           report.getNumberOfMissingBlocksWithReplicationFactorOne());
+      stats.setNumberOfBadlyDistributedBlocks(
+          report.getNumberOfBadlyDistributedBlocks());
       stats.setHighestPriorityLowRedundancyReplicatedBlocks(
           report.getHighestPriorityLowRedundancyReplicatedBlocks());
       stats.setHighestPriorityLowRedundancyECBlocks(

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/NamenodeStatusReport.java

@@ -73,6 +73,7 @@ public class NamenodeStatusReport {
   private int corruptFilesCount = -1;
   private long scheduledReplicationBlocks = -1;
   private long numberOfMissingBlocksWithReplicationFactorOne = -1;
+  private long numberOfBadlyDistributedBlocks = -1;
   private long highestPriorityLowRedundancyReplicatedBlocks = -1;
   private long highestPriorityLowRedundancyECBlocks = -1;
   private int pendingSPSPaths = -1;
@@ -394,18 +395,22 @@ public void setNamesystemInfo(long available, long total,
    * @param numCorruptFiles number of corrupt files.
    * @param numOfMissingBlocksWithReplicationFactorOne number of missing
    * blocks with rep one.
+   * @param numOfBadlyDistributedBlocks number of badly distributed blocks
    * @param highestPriorityLowRedundancyRepBlocks number of high priority low
    * redundancy rep blocks.
    * @param highPriorityLowRedundancyECBlocks number of high priority low
    * redundancy EC blocks.
    */
   public void setNamenodeInfo(int numCorruptFiles,
       long numOfMissingBlocksWithReplicationFactorOne,
+      long numOfBadlyDistributedBlocks,
       long highestPriorityLowRedundancyRepBlocks,
       long highPriorityLowRedundancyECBlocks) {
     this.corruptFilesCount = numCorruptFiles;
     this.numberOfMissingBlocksWithReplicationFactorOne =
         numOfMissingBlocksWithReplicationFactorOne;
+    this.numberOfBadlyDistributedBlocks =
+        numOfBadlyDistributedBlocks;
     this.highestPriorityLowRedundancyReplicatedBlocks =
         highestPriorityLowRedundancyRepBlocks;
     this.highestPriorityLowRedundancyECBlocks =
@@ -441,6 +446,16 @@ public long getNumberOfMissingBlocksWithReplicationFactorOne() {
     return this.numberOfMissingBlocksWithReplicationFactorOne;
   }
 
+  /**
+   * Gets the total number of badly distributed blocks.
+   *
+   * @return the total number of badly distrubted blocks.
+   */
+  public long getNumberOfBadlyDistributedBlocks() {
+    return this.numberOfBadlyDistributedBlocks;
+  }
+
+
   /**
    * Gets the total number of replicated low redundancy blocks on the cluster
    * with the highest risk of loss.

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java

@@ -543,7 +543,8 @@ private void populateNamenodeInfoMetrics(JSONArray aux, NamenodeStatusReport rep
                 .optLong("NumberOfMissingBlocksWithReplicationFactorOne"),
             jsonObject
                 .optLong("HighestPriorityLowRedundancyReplicatedBlocks"),
-            jsonObject.optLong("HighestPriorityLowRedundancyECBlocks"));
+            jsonObject.optLong("HighestPriorityLowRedundancyECBlocks"),
+            jsonObject.optLong("BadlyDistributedBlocks"));
       }
     }
   }
@@ -608,4 +609,4 @@ protected void serviceStop() throws Exception {
     }
     super.serviceStop();
   }
-}
+}

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipStats.java

@@ -122,6 +122,11 @@ public abstract void setNumberOfMissingBlocksWithReplicationFactorOne(
 
   public abstract long getNumberOfMissingBlocksWithReplicationFactorOne();
 
+  public abstract void setNumberOfBadlyDistributedBlocks(
+      long blocks);
+
+  public abstract long getNumberOfBadlyDistributedBlocks();
+
   public abstract void setHighestPriorityLowRedundancyReplicatedBlocks(
       long blocks);
 
@@ -171,4 +176,4 @@ public long getDateCreated() {
     // We don't store this record directly
     return 0;
   }
-}
+}