Skip to content

Commit 079f31a

Browse files
PrateekSanePrateekSane
PrateekSane
authored and
PrateekSane
committed
HDFS-17641. Change badly distributed blocks metric protobuf requirement to optional (#7274)
Note: This is a follow on pr from #7123 that walks back an incompatible change that was made.
1 parent e68c0ac commit 079f31a

5 files changed

Lines changed: 64 additions & 38 deletions

File tree

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,22 +38,21 @@ public final class ECBlockGroupStats {
3838
private final long missingBlockGroups;
3939
private final long bytesInFutureBlockGroups;
4040
private final long pendingDeletionBlocks;
41-
private final long badlyDistributedBlocks;
41+
private final Long badlyDistributedBlocks;
4242
private final Long highestPriorityLowRedundancyBlocks;
4343

4444
public ECBlockGroupStats(long lowRedundancyBlockGroups,
4545
long corruptBlockGroups, long missingBlockGroups,
46-
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
47-
long badlyDistributedBlocks) {
46+
long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
4847
this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
4948
bytesInFutureBlockGroups, pendingDeletionBlocks,
50-
badlyDistributedBlocks, null);
49+
null, null);
5150
}
5251

5352
public ECBlockGroupStats(long lowRedundancyBlockGroups,
5453
long corruptBlockGroups, long missingBlockGroups,
5554
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
56-
long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
55+
Long badlyDistributedBlocks, Long highestPriorityLowRedundancyBlocks) {
5756
this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
5857
this.corruptBlockGroups = corruptBlockGroups;
5958
this.missingBlockGroups = missingBlockGroups;
@@ -84,7 +83,11 @@ public long getPendingDeletionBlocks() {
8483
return pendingDeletionBlocks;
8584
}
8685

87-
public long getBadlyDistributedBlocks() {
86+
public boolean hasBadlyDistributedBlocks() {
87+
return getBadlyDistributedBlocks() != null;
88+
}
89+
90+
public Long getBadlyDistributedBlocks() {
8891
return badlyDistributedBlocks;
8992
}
9093

@@ -107,8 +110,11 @@ public String toString() {
107110
.append(", BytesInFutureBlockGroups=").append(
108111
getBytesInFutureBlockGroups())
109112
.append(", PendingDeletionBlocks=").append(
110-
getPendingDeletionBlocks())
111-
.append(" , BadlyDistributedBlocks=").append(getBadlyDistributedBlocks());
113+
getPendingDeletionBlocks());
114+
if(hasBadlyDistributedBlocks()) {
115+
statsBuilder.append(", BadlyDistributedBlocks=")
116+
.append(getBadlyDistributedBlocks());
117+
}
112118
if (hasHighestPriorityLowRedundancyBlocks()) {
113119
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
114120
.append(getHighestPriorityLowRedundancyBlocks());
@@ -163,6 +169,7 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
163169
long bytesInFutureBlockGroups = 0;
164170
long pendingDeletionBlocks = 0;
165171
long badlyDistributedBlocks = 0;
172+
boolean hasBadlyDistributedBlocks = false;
166173
long highestPriorityLowRedundancyBlocks = 0;
167174
boolean hasHighestPriorityLowRedundancyBlocks = false;
168175

@@ -172,20 +179,22 @@ public static ECBlockGroupStats merge(Collection<ECBlockGroupStats> stats) {
172179
missingBlockGroups += stat.getMissingBlockGroups();
173180
bytesInFutureBlockGroups += stat.getBytesInFutureBlockGroups();
174181
pendingDeletionBlocks += stat.getPendingDeletionBlocks();
175-
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
182+
if (stat.hasBadlyDistributedBlocks()) {
183+
hasBadlyDistributedBlocks = true;
184+
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
185+
}
176186
if (stat.hasHighestPriorityLowRedundancyBlocks()) {
177187
hasHighestPriorityLowRedundancyBlocks = true;
178188
highestPriorityLowRedundancyBlocks +=
179189
stat.getHighestPriorityLowRedundancyBlocks();
180190
}
181191
}
182-
if (hasHighestPriorityLowRedundancyBlocks) {
192+
if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
183193
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
184194
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
185195
badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
186196
}
187197
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
188-
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
189-
badlyDistributedBlocks);
198+
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
190199
}
191200
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -37,22 +37,22 @@ public final class ReplicatedBlockStats {
3737
private final long missingReplicationOneBlocks;
3838
private final long bytesInFutureBlocks;
3939
private final long pendingDeletionBlocks;
40-
private final long badlyDistributedBlocks;
40+
private final Long badlyDistributedBlocks;
4141
private final Long highestPriorityLowRedundancyBlocks;
4242

4343
public ReplicatedBlockStats(long lowRedundancyBlocks,
4444
long corruptBlocks, long missingBlocks,
4545
long missingReplicationOneBlocks, long bytesInFutureBlocks,
46-
long pendingDeletionBlocks, long badlyDistributedBlocks) {
46+
long pendingDeletionBlocks) {
4747
this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
4848
missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
49-
badlyDistributedBlocks, null);
49+
null, null);
5050
}
5151

5252
public ReplicatedBlockStats(long lowRedundancyBlocks,
5353
long corruptBlocks, long missingBlocks,
5454
long missingReplicationOneBlocks, long bytesInFutureBlocks,
55-
long pendingDeletionBlocks, long badlyDistributedBlocks,
55+
long pendingDeletionBlocks, Long badlyDistributedBlocks,
5656
Long highestPriorityLowRedundancyBlocks) {
5757
this.lowRedundancyBlocks = lowRedundancyBlocks;
5858
this.corruptBlocks = corruptBlocks;
@@ -89,7 +89,11 @@ public long getPendingDeletionBlocks() {
8989
return pendingDeletionBlocks;
9090
}
9191

92-
public long getBadlyDistributedBlocks() {
92+
public boolean hasBadlyDistributedBlocks() {
93+
return getBadlyDistributedBlocks() != null;
94+
}
95+
96+
public Long getBadlyDistributedBlocks() {
9397
return badlyDistributedBlocks;
9498
}
9599

@@ -113,8 +117,10 @@ public String toString() {
113117
getMissingReplicationOneBlocks())
114118
.append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
115119
.append(", PendingDeletionBlocks=").append(
116-
getPendingDeletionBlocks())
117-
.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
120+
getPendingDeletionBlocks());
121+
if (hasBadlyDistributedBlocks()) {
122+
statsBuilder.append(" , badlyDistributedBlocks=").append(getBadlyDistributedBlocks());
123+
}
118124
if (hasHighestPriorityLowRedundancyBlocks()) {
119125
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
120126
getHighestPriorityLowRedundancyBlocks());
@@ -137,6 +143,7 @@ public static ReplicatedBlockStats merge(
137143
long bytesInFutureBlocks = 0;
138144
long pendingDeletionBlocks = 0;
139145
long badlyDistributedBlocks = 0;
146+
boolean hasBadlyDistributedBlocks = false;
140147
long highestPriorityLowRedundancyBlocks = 0;
141148
boolean hasHighestPriorityLowRedundancyBlocks = false;
142149

@@ -148,20 +155,24 @@ public static ReplicatedBlockStats merge(
148155
missingReplicationOneBlocks += stat.getMissingReplicationOneBlocks();
149156
bytesInFutureBlocks += stat.getBytesInFutureBlocks();
150157
pendingDeletionBlocks += stat.getPendingDeletionBlocks();
151-
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
158+
159+
if (stat.hasBadlyDistributedBlocks()) {
160+
hasBadlyDistributedBlocks = true;
161+
badlyDistributedBlocks += stat.getBadlyDistributedBlocks();
162+
}
152163
if (stat.hasHighestPriorityLowRedundancyBlocks()) {
153164
hasHighestPriorityLowRedundancyBlocks = true;
154165
highestPriorityLowRedundancyBlocks +=
155166
stat.getHighestPriorityLowRedundancyBlocks();
156167
}
157168
}
158-
if (hasHighestPriorityLowRedundancyBlocks) {
169+
if (hasBadlyDistributedBlocks && hasHighestPriorityLowRedundancyBlocks) {
159170
return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
160171
missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
161172
pendingDeletionBlocks, badlyDistributedBlocks, highestPriorityLowRedundancyBlocks);
162173
}
163174
return new ReplicatedBlockStats(lowRedundancyBlocks, corruptBlocks,
164175
missingBlocks, missingReplicationOneBlocks, bytesInFutureBlocks,
165-
pendingDeletionBlocks, badlyDistributedBlocks);
176+
pendingDeletionBlocks);
166177
}
167178
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,7 +1971,7 @@ public static long[] convert(GetFsStatsResponseProto res) {
19711971

19721972
public static ReplicatedBlockStats convert(
19731973
GetFsReplicatedBlockStatsResponseProto res) {
1974-
if (res.hasHighestPrioLowRedundancyBlocks()) {
1974+
if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
19751975
return new ReplicatedBlockStats(res.getLowRedundancy(),
19761976
res.getCorruptBlocks(), res.getMissingBlocks(),
19771977
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
@@ -1981,21 +1981,20 @@ public static ReplicatedBlockStats convert(
19811981
return new ReplicatedBlockStats(res.getLowRedundancy(),
19821982
res.getCorruptBlocks(), res.getMissingBlocks(),
19831983
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
1984-
res.getBadlyDistributedBlocks(), res.getPendingDeletionBlocks());
1984+
res.getPendingDeletionBlocks());
19851985
}
19861986

19871987
public static ECBlockGroupStats convert(
19881988
GetFsECBlockGroupStatsResponseProto res) {
1989-
if (res.hasHighestPrioLowRedundancyBlocks()) {
1989+
if (res.hasBadlyDistributedBlocks() && res.hasHighestPrioLowRedundancyBlocks()) {
19901990
return new ECBlockGroupStats(res.getLowRedundancy(),
19911991
res.getCorruptBlocks(), res.getMissingBlocks(),
19921992
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
19931993
res.getBadlyDistributedBlocks(), res.getHighestPrioLowRedundancyBlocks());
19941994
}
19951995
return new ECBlockGroupStats(res.getLowRedundancy(),
19961996
res.getCorruptBlocks(), res.getMissingBlocks(),
1997-
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
1998-
res.getBadlyDistributedBlocks());
1997+
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
19991998
}
20001999

20012000
public static DatanodeReportTypeProto convert(DatanodeReportType t) {
@@ -2441,8 +2440,10 @@ public static GetFsReplicatedBlockStatsResponseProto convert(
24412440
replicatedBlockStats.getBytesInFutureBlocks());
24422441
result.setPendingDeletionBlocks(
24432442
replicatedBlockStats.getPendingDeletionBlocks());
2444-
result.setBadlyDistributedBlocks(
2445-
replicatedBlockStats.getBadlyDistributedBlocks());
2443+
if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
2444+
result.setBadlyDistributedBlocks(
2445+
replicatedBlockStats.getBadlyDistributedBlocks());
2446+
}
24462447
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
24472448
result.setHighestPrioLowRedundancyBlocks(
24482449
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
@@ -2462,8 +2463,10 @@ public static GetFsECBlockGroupStatsResponseProto convert(
24622463
ecBlockGroupStats.getBytesInFutureBlockGroups());
24632464
result.setPendingDeletionBlocks(
24642465
ecBlockGroupStats.getPendingDeletionBlocks());
2465-
result.setBadlyDistributedBlocks(
2466-
ecBlockGroupStats.getBadlyDistributedBlocks());
2466+
if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
2467+
result.setBadlyDistributedBlocks(
2468+
ecBlockGroupStats.getBadlyDistributedBlocks());
2469+
}
24672470
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
24682471
result.setHighestPrioLowRedundancyBlocks(
24692472
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,7 @@ message GetFsReplicatedBlockStatsResponseProto {
363363
required uint64 blocks_in_future = 5;
364364
required uint64 pending_deletion_blocks = 6;
365365
optional uint64 highest_prio_low_redundancy_blocks = 7;
366-
required uint64 badly_distributed_blocks = 8;
367-
366+
optional uint64 badly_distributed_blocks = 8;
368367
}
369368

370369
message GetFsECBlockGroupStatsRequestProto { // no input paramters
@@ -377,7 +376,7 @@ message GetFsECBlockGroupStatsResponseProto {
377376
required uint64 blocks_in_future = 4;
378377
required uint64 pending_deletion_blocks = 5;
379378
optional uint64 highest_prio_low_redundancy_blocks = 6;
380-
required uint64 badly_distributed_blocks = 7;
379+
optional uint64 badly_distributed_blocks = 7;
381380
}
382381

383382
enum DatanodeReportTypeProto { // type of the datanode report

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -547,8 +547,10 @@ public void report(String[] argv, int i) throws IOException {
547547
replicatedBlockStats.getMissingReplicaBlocks());
548548
System.out.println("\tMissing blocks (with replication factor 1): " +
549549
replicatedBlockStats.getMissingReplicationOneBlocks());
550-
System.out.println("\tBadly Distributed Blocks: " +
551-
replicatedBlockStats.getBadlyDistributedBlocks());
550+
if (replicatedBlockStats.hasBadlyDistributedBlocks()) {
551+
System.out.println("\tBadly Distributed Blocks: " +
552+
replicatedBlockStats.getBadlyDistributedBlocks());
553+
}
552554
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
553555
System.out.println("\tLow redundancy blocks with highest priority " +
554556
"to recover: " +
@@ -566,8 +568,10 @@ public void report(String[] argv, int i) throws IOException {
566568
ecBlockGroupStats.getCorruptBlockGroups());
567569
System.out.println("\tMissing block groups: " +
568570
ecBlockGroupStats.getMissingBlockGroups());
569-
System.out.println("\tBadly Distributed Blocks: " +
570-
ecBlockGroupStats.getBadlyDistributedBlocks());
571+
if (ecBlockGroupStats.hasBadlyDistributedBlocks()) {
572+
System.out.println("\tBadly Distributed Blocks: " +
573+
ecBlockGroupStats.getBadlyDistributedBlocks());
574+
}
571575
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
572576
System.out.println("\tLow redundancy blocks with highest priority " +
573577
"to recover: " +

0 commit comments

Comments
 (0)