From bfa607f9d1524a4784a500e191873464180dbc25 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sat, 7 Sep 2019 12:32:37 +0530 Subject: [PATCH 1/4] HBASE-22903 : Table to RegionStatesCount metrics --- .../apache/hadoop/hbase/ClusterMetrics.java | 15 +- .../hadoop/hbase/ClusterMetricsBuilder.java | 46 +++++- .../hbase/client/RegionStatesCount.java | 131 ++++++++++++++++++ .../hbase/shaded/protobuf/ProtobufUtil.java | 48 +++++++ .../src/main/protobuf/ClusterStatus.proto | 15 ++ .../apache/hadoop/hbase/master/HMaster.java | 19 +++ .../master/assignment/AssignmentManager.java | 38 +++++ hbase-shell/src/main/ruby/hbase/admin.rb | 13 +- 8 files changed, 315 insertions(+), 10 deletions(-) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionStatesCount.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java index acced3ea82b5..497ab938856b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetrics.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.master.RegionState; import org.apache.yetus.audience.InterfaceAudience; @@ -152,6 +153,14 @@ default double getAverageLoad() { return (double)getRegionCount() / (double)serverSize; } + /** + * Provide region states count for given table. + * e.g howmany regions of give table are opened/closed/rit etc + * + * @return map of table to region states count + */ + Map getTableRegionStatesCount(); + /** * Kinds of ClusterMetrics */ @@ -199,6 +208,10 @@ enum Option { /** * metrics about live region servers name */ - SERVERS_NAME + SERVERS_NAME, + /** + * metrics about table to no of regions status count + */ + TABLE_TO_REGIONS_COUNT, } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java index 30728ac4354f..64f5fcd34763 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterMetricsBuilder.java @@ -25,7 +25,10 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collector; import java.util.stream.Collectors; + +import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.master.RegionState; import org.apache.yetus.audience.InterfaceAudience; @@ -70,7 +73,17 @@ public static ClusterStatusProtos.ClusterStatus toClusterStatus(ClusterMetrics m .collect(Collectors.toList())) .setMasterInfoPort(metrics.getMasterInfoPort()) .addAllServersName(metrics.getServersName().stream().map(ProtobufUtil::toServerName) - .collect(Collectors.toList())); + .collect(Collectors.toList())) + .addAllTableRegionStatesCount(metrics.getTableRegionStatesCount() + .entrySet() + .stream() + .map(status -> + ClusterStatusProtos.TableRegionStatesCount + .newBuilder() + .setTableName(ProtobufUtil.toProtoTableName((status.getKey()))) + .setRegionStatesCount(ProtobufUtil.toTableRegionStatesCount(status.getValue())) + .build()) + .collect(Collectors.toList())); if (metrics.getMasterName() != null) { builder.setMaster(ProtobufUtil.toServerName((metrics.getMasterName()))); } @@ -108,7 +121,12 @@ public static ClusterMetrics toClusterMetrics( .map(HBaseProtos.Coprocessor::getName) .collect(Collectors.toList())) .setServerNames(proto.getServersNameList().stream().map(ProtobufUtil::toServerName) - .collect(Collectors.toList())); + .collect(Collectors.toList())) + .setTableRegionStatesCount( + proto.getTableRegionStatesCountList().stream() + .collect(Collectors.toMap( + e -> ProtobufUtil.toTableName(e.getTableName()), + e -> ProtobufUtil.toTableRegionStatesCount(e.getRegionStatesCount())))); if (proto.hasClusterId()) { builder.setClusterId(ClusterId.convert(proto.getClusterId()).toString()); } @@ -149,6 +167,7 @@ public static ClusterMetrics.Option toOption(ClusterStatusProtos.Option option) case BALANCER_ON: return ClusterMetrics.Option.BALANCER_ON; case SERVERS_NAME: return ClusterMetrics.Option.SERVERS_NAME; case MASTER_INFO_PORT: return ClusterMetrics.Option.MASTER_INFO_PORT; + case TABLE_TO_REGIONS_COUNT: return ClusterMetrics.Option.TABLE_TO_REGIONS_COUNT; // should not reach here default: throw new IllegalArgumentException("Invalid option: " + option); } @@ -172,6 +191,7 @@ public static ClusterStatusProtos.Option toOption(ClusterMetrics.Option option) case BALANCER_ON: return ClusterStatusProtos.Option.BALANCER_ON; case SERVERS_NAME: return Option.SERVERS_NAME; case MASTER_INFO_PORT: return ClusterStatusProtos.Option.MASTER_INFO_PORT; + case TABLE_TO_REGIONS_COUNT: return ClusterStatusProtos.Option.TABLE_TO_REGIONS_COUNT; // should not reach here default: throw new IllegalArgumentException("Invalid option: " + option); } @@ -214,6 +234,7 @@ public static ClusterMetricsBuilder newBuilder() { private Boolean balancerOn; private int masterInfoPort; private List serversName = Collections.emptyList(); + private Map tableRegionStatesCount = Collections.emptyMap(); private ClusterMetricsBuilder() { } @@ -263,6 +284,13 @@ public ClusterMetricsBuilder setServerNames(List serversName) { this.serversName = serversName; return this; } + + public ClusterMetricsBuilder setTableRegionStatesCount( + Map tableRegionStatesCount) { + this.tableRegionStatesCount = tableRegionStatesCount; + return this; + } + public ClusterMetrics build() { return new ClusterMetricsImpl( hbaseVersion, @@ -275,7 +303,9 @@ public ClusterMetrics build() { masterCoprocessorNames, balancerOn, masterInfoPort, - serversName); + serversName, + tableRegionStatesCount + ); } private static class ClusterMetricsImpl implements ClusterMetrics { @Nullable @@ -293,6 +323,7 @@ private static class ClusterMetricsImpl implements ClusterMetrics { private final Boolean balancerOn; private final int masterInfoPort; private final List serversName; + private final Map tableRegionStatesCount; ClusterMetricsImpl(String hbaseVersion, List deadServerNames, Map liveServerMetrics, @@ -303,7 +334,8 @@ private static class ClusterMetricsImpl implements ClusterMetrics { List masterCoprocessorNames, Boolean balancerOn, int masterInfoPort, - List serversName) { + List serversName, + Map tableRegionStatesCount) { this.hbaseVersion = hbaseVersion; this.deadServerNames = Preconditions.checkNotNull(deadServerNames); this.liveServerMetrics = Preconditions.checkNotNull(liveServerMetrics); @@ -315,6 +347,7 @@ private static class ClusterMetricsImpl implements ClusterMetrics { this.balancerOn = balancerOn; this.masterInfoPort = masterInfoPort; this.serversName = serversName; + this.tableRegionStatesCount = Preconditions.checkNotNull(tableRegionStatesCount); } @Override @@ -372,6 +405,11 @@ public List getServersName() { return Collections.unmodifiableList(serversName); } + @Override + public Map getTableRegionStatesCount() { + return Collections.unmodifiableMap(tableRegionStatesCount); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(1024); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionStatesCount.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionStatesCount.java new file mode 100644 index 000000000000..bce3e9dd6579 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionStatesCount.java @@ -0,0 +1,131 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public final class RegionStatesCount { + + private int openRegions; + private int splitRegions; + private int closedRegions; + private int regionsInTransition; + private int totalRegions; + + private RegionStatesCount() { + } + + public int getClosedRegions() { + return closedRegions; + } + + private void setClosedRegions(int closedRegions) { + this.closedRegions = closedRegions; + } + + public int getOpenRegions() { + return openRegions; + } + + public int getSplitRegions() { + return splitRegions; + } + + public int getRegionsInTransition() { + return regionsInTransition; + } + + public int getTotalRegions() { + return totalRegions; + } + + private void setOpenRegions(int openRegions) { + this.openRegions = openRegions; + } + + private void setSplitRegions(int splitRegions) { + this.splitRegions = splitRegions; + } + + private void setRegionsInTransition(int regionsInTransition) { + this.regionsInTransition = regionsInTransition; + } + + private void setTotalRegions(int totalRegions) { + this.totalRegions = totalRegions; + } + + public static class RegionStatesCountBuilder { + private int openRegions; + private int splitRegions; + private int closedRegions; + private int regionsInTransition; + private int totalRegions; + + public RegionStatesCountBuilder setOpenRegions(int openRegions) { + this.openRegions = openRegions; + return this; + } + + public RegionStatesCountBuilder setSplitRegions(int splitRegions) { + this.splitRegions = splitRegions; + return this; + } + + public RegionStatesCountBuilder setClosedRegions(int closedRegions) { + this.closedRegions = closedRegions; + return this; + } + + public RegionStatesCountBuilder setRegionsInTransition(int regionsInTransition) { + this.regionsInTransition = regionsInTransition; + return this; + } + + public RegionStatesCountBuilder setTotalRegions(int totalRegions) { + this.totalRegions = totalRegions; + return this; + } + + public RegionStatesCount build() { + RegionStatesCount regionStatesCount=new RegionStatesCount(); + regionStatesCount.setOpenRegions(openRegions); + regionStatesCount.setClosedRegions(closedRegions); + regionStatesCount.setRegionsInTransition(regionsInTransition); + regionStatesCount.setSplitRegions(splitRegions); + regionStatesCount.setTotalRegions(totalRegions); + return regionStatesCount; + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("RegionStatesCount{"); + sb.append("openRegions=").append(openRegions); + sb.append(", splitRegions=").append(splitRegions); + sb.append(", closedRegions=").append(closedRegions); + sb.append(", regionsInTransition=").append(regionsInTransition); + sb.append(", totalRegions=").append(totalRegions); + sb.append('}'); + return sb.toString(); + } + +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java index 9ccfad8e8a16..52e3bf1bfbe0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java @@ -79,6 +79,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionLoadStats; +import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.SnapshotDescription; @@ -3297,4 +3298,51 @@ public static Set toCompactedStoreFiles(byte[] bytes) throws IOException } return Collections.emptySet(); } + + public static ClusterStatusProtos.RegionStatesCount toTableRegionStatesCount( + RegionStatesCount regionStatesCount) { + int openRegions = 0; + int splitRegions = 0; + int closedRegions = 0; + int regionsInTransition = 0; + int totalRegions = 0; + if (regionStatesCount != null) { + openRegions = regionStatesCount.getOpenRegions(); + splitRegions = regionStatesCount.getSplitRegions(); + closedRegions = regionStatesCount.getClosedRegions(); + regionsInTransition = regionStatesCount.getRegionsInTransition(); + totalRegions = regionStatesCount.getTotalRegions(); + } + return ClusterStatusProtos.RegionStatesCount.newBuilder() + .setOpenRegions(openRegions) + .setSplitRegions(splitRegions) + .setClosedRegions(closedRegions) + .setRegionsInTransition(regionsInTransition) + .setTotalRegions(totalRegions) + .build(); + } + + public static RegionStatesCount toTableRegionStatesCount( + ClusterStatusProtos.RegionStatesCount regionStatesCount) { + int openRegions = 0; + int splitRegions = 0; + int closedRegions = 0; + int regionsInTransition = 0; + int totalRegions = 0; + if (regionStatesCount != null) { + closedRegions = regionStatesCount.getClosedRegions(); + regionsInTransition = regionStatesCount.getRegionsInTransition(); + openRegions = regionStatesCount.getOpenRegions(); + splitRegions = regionStatesCount.getSplitRegions(); + totalRegions = regionStatesCount.getTotalRegions(); + } + return new RegionStatesCount.RegionStatesCountBuilder() + .setOpenRegions(openRegions) + .setSplitRegions(splitRegions) + .setClosedRegions(closedRegions) + .setRegionsInTransition(regionsInTransition) + .setTotalRegions(totalRegions) + .build(); + } + } diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto index 088054948902..ef16ecbac13a 100644 --- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto @@ -231,6 +231,19 @@ message LiveServerInfo { required ServerLoad server_load = 2; } +message RegionStatesCount { + required uint32 open_regions = 1; + required uint32 split_regions = 2; + required uint32 closed_regions = 3; + required uint32 regions_in_transition = 4; + required uint32 total_regions = 5; +} + +message TableRegionStatesCount { + required TableName table_name = 1; + required RegionStatesCount region_states_count = 2; +} + message ClusterStatus { optional HBaseVersionFileContent hbase_version = 1; repeated LiveServerInfo live_servers = 2; @@ -243,6 +256,7 @@ message ClusterStatus { optional bool balancer_on = 9; optional int32 master_info_port = 10 [default = -1]; repeated ServerName servers_name = 11; + repeated TableRegionStatesCount table_region_states_count = 12; } enum Option { @@ -257,4 +271,5 @@ enum Option { BALANCER_ON = 8; MASTER_INFO_PORT = 9; SERVERS_NAME = 10; + TABLE_TO_REGIONS_COUNT = 11; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 5939053aec84..557949a25820 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -83,6 +83,7 @@ import org.apache.hadoop.hbase.client.MasterSwitchType; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.RegionStatesCount; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; @@ -2600,6 +2601,24 @@ public ClusterMetrics getClusterMetricsWithoutCoprocessor(EnumSet