HBASE-28209: Create a jmx metrics to expose the oldWALs directory size

hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -153,6 +153,13 @@ public enum OperationStatusCode {
   /** Default value for the balancer period */
   public static final int DEFAULT_HBASE_BALANCER_PERIOD = 300000;
 
+  /** Config for the oldWALs directory size updater period */
+  public static final String HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD =
+    "hbase.master.oldwals.dir.updater.period";
+
+  /** Default value for the oldWALs directory size updater period */
+  public static final int DEFAULT_HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD = 300000;
+
   /**
    * Config key for enable/disable automatically separate child regions to different region servers
    * in the procedure of split regions. One child will be kept to the server where parent region is

hbase-common/src/main/resources/hbase-default.xml

@@ -606,6 +606,12 @@ possible configurations would overwhelm and obscure the important.
     <description>Period at which the region balancer runs in the Master, in
       milliseconds.</description>
   </property>
+  <property>
+    <name>hbase.master.oldwals.dir.updater.period</name>
+    <value>300000</value>
+    <description>Period at which the oldWALs directory size calculator/updater will run in the
+      Master, in milliseconds.</description>
+  </property>
   <property>
     <name>hbase.regions.slop</name>
     <value>0.2</value>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSource.java

@@ -70,6 +70,7 @@ public interface MetricsMasterSource extends BaseSource {
   String CLUSTER_REQUESTS_NAME = "clusterRequests";
   String CLUSTER_READ_REQUESTS_NAME = "clusterReadRequests";
   String CLUSTER_WRITE_REQUESTS_NAME = "clusterWriteRequests";
+  String OLD_WAL_DIR_SIZE_NAME = "oldWALsDirSize";
   String MASTER_ACTIVE_TIME_DESC = "Master Active Time";
   String MASTER_START_TIME_DESC = "Master Start Time";
   String MASTER_FINISHED_INITIALIZATION_TIME_DESC =
@@ -91,6 +92,7 @@ public interface MetricsMasterSource extends BaseSource {
   String OFFLINE_REGION_COUNT_DESC = "Number of Offline Regions";
 
   String SERVER_CRASH_METRIC_PREFIX = "serverCrash";
+  String OLD_WAL_DIR_SIZE_DESC = "size of old WALs directory in bytes";
 
   /**
    * Increment the number of requests the cluster has seen.

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java

@@ -129,7 +129,9 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
         .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName())
         .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId())
         .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC),
-          String.valueOf(masterWrapper.getIsActiveMaster()));
+          String.valueOf(masterWrapper.getIsActiveMaster()))
+        .addGauge(Interns.info(OLD_WAL_DIR_SIZE_NAME, OLD_WAL_DIR_SIZE_DESC),
+          masterWrapper.getOldWALsDirSize());
     }
 
     metricsRegistry.snapshot(metricsRecordBuilder, all);

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java

@@ -153,4 +153,9 @@ public interface MetricsMasterWrapper {
    * @return pair of count for online regions and offline regions
    */
   PairOfSameType<Integer> getRegionCounts();
+
+  /**
+   * Get the size of old WALs directory in bytes.
+   */
+  long getOldWALsDirSize();
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -458,6 +458,7 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
   private SpaceQuotaSnapshotNotifier spaceQuotaSnapshotNotifier;
   private QuotaObserverChore quotaObserverChore;
   private SnapshotQuotaObserverChore snapshotQuotaChore;
+  private OldWALsDirSizeUpdaterChore oldWALsDirSizeUpdaterChore;
 
   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
   private ProcedureStore procedureStore;
@@ -1362,6 +1363,10 @@ private void finishActiveMasterInitialization() throws IOException, InterruptedE
 
     this.rollingUpgradeChore = new RollingUpgradeChore(this);
     getChoreService().scheduleChore(rollingUpgradeChore);
+
+    this.oldWALsDirSizeUpdaterChore = new OldWALsDirSizeUpdaterChore(this);
+    getChoreService().scheduleChore(this.oldWALsDirSizeUpdaterChore);
+
     status.markComplete("Progress after master initialized complete");
   }
 
@@ -1894,6 +1899,7 @@ protected void stopChores() {
     shutdownChore(hbckChore);
     shutdownChore(regionsRecoveryChore);
     shutdownChore(rollingUpgradeChore);
+    shutdownChore(oldWALsDirSizeUpdaterChore);
   }
 
   /** Returns Get remote side's InetAddress */

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java

@@ -89,6 +89,12 @@ public boolean accept(Path p) {
   // create the split log lock
   private final Lock splitLogLock = new ReentrantLock();
 
+  // old WALs directory size in bytes
+  private long oldWALsDirSize;
+
+  // old WALs directory size calculation interval
+  private final int OLD_WAL_DIR_UPDATE_INTERVAL = 5 * 60 * 1000; // 5 mins
+
   /**
    * Superceded by {@link SplitWALManager}; i.e. procedure-based WAL splitting rather than 'classic'
    * zk-coordinated WAL splitting.
@@ -114,6 +120,7 @@ public MasterWalManager(Configuration conf, FileSystem fs, Path rootDir, MasterS
     this.services = services;
     this.splitLogManager = new SplitLogManager(services, conf);
     this.oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+    this.oldWALsDirSize = 0;
   }
 
   public void stop() {
@@ -134,6 +141,14 @@ Path getOldLogDir() {
     return this.oldLogDir;
   }
 
+  public void updateOldWALsDirSize() throws IOException {
+    this.oldWALsDirSize = fs.getContentSummary(this.oldLogDir).getLength();
+  }
+
+  public long getOldWALsDirSize() {
+    return this.oldWALsDirSize;
+  }
+
   public FileSystem getFileSystem() {
     return this.fs;
   }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java

@@ -238,4 +238,12 @@ public PairOfSameType<Integer> getRegionCounts() {
       return new PairOfSameType<>(0, 0);
     }
   }
+
+  @Override
+  public long getOldWALsDirSize() {
+    if (master == null || !master.isInitialized()) {
+      return 0;
+    }
+    return master.getMasterWalManager().getOldWALsDirSize();
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/master/OldWALsDirSizeUpdaterChore.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ScheduledChore;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This chore is used to update the 'oldWALsDirSize' variable in {@link MasterWalManager} through
+ * the {@link MasterWalManager#updateOldWALsDirSize()} method.
+ */
+@InterfaceAudience.Private
+public class OldWALsDirSizeUpdaterChore extends ScheduledChore {
+  private static final Logger LOG = LoggerFactory.getLogger(OldWALsDirSizeUpdaterChore.class);
+
+  private final HMaster master;
+
+  public OldWALsDirSizeUpdaterChore(HMaster master) {
+    super(master.getServerName() + "-OldWALsDirSizeUpdaterChore", master,
+      master.getConfiguration().getInt(HConstants.HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD,
+        HConstants.DEFAULT_HBASE_OLDWAL_DIR_SIZE_UPDATER_PERIOD));
+    this.master = master;
+  }
+
+  @Override
+  protected void chore() {
+    try {
+      this.master.getMasterWalManager().updateOldWALsDirSize();
+    } catch (IOException e) {
+      LOG.error("Got exception while trying to update the old WALs Directory size counter: "
+        + e.getMessage(), e);
+    }
+  }
+}

src/main/asciidoc/_chapters/hbase-default.adoc

@@ -761,6 +761,17 @@ Period at which the region balancer runs in the Master.
 `300000`
 
 
+[[hbase.master.oldwals.dir.updater.period]]
+*`hbase.master.oldwals.dir.updater.period`*::
++
+.Description
+Period at which the oldWALs directory size calculator/updater will run in the Master.
+
++
+.Default
+`300000`
+
+
 [[hbase.regions.slop]]
 *`hbase.regions.slop`*::
 +