HBASE-26731 Add metrics for active and expired scanners (#4145)

Signed-off-by: Andrew Purtell <apurtell@apache.org>

Author: bbeaudreault

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -226,6 +226,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
    */
   void updateCompactionOutputSize(boolean isMajor, long bytes);
 
+  void incrScannerLeaseExpired();
+
   // Strings used for exporting to metrics system.
   String REGION_COUNT = "regionCount";
   String REGION_COUNT_DESC = "Number of regions";
@@ -598,4 +600,10 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT_DESC = "Total buffer count in ByteBuffAllocator";
   String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT = "ByteBuffAllocatorUsedBufferCount";
   String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC = "Used buffer count in ByteBuffAllocator";
+
+  String ACTIVE_SCANNERS = "activeScanners";
+  String ACTIVE_SCANNERS_DESC = "Gauge of currently active scanners";
+
+  String SCANNER_LEASE_EXPIRED_COUNT = "scannerLeaseExpiredCount";
+  String SCANNER_LEASE_EXPIRED_COUNT_DESC = "Count of scanners which were expired due to scanner lease timeout";
 }

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -91,6 +91,8 @@ public class MetricsRegionServerSourceImpl
   private final MetricHistogram pausesWithGc;
   private final MetricHistogram pausesWithoutGc;
 
+  private final MutableFastCounter scannerLeaseExpiredCount;
+
   public MetricsRegionServerSourceImpl(MetricsRegionServerWrapper rsWrap) {
     this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, rsWrap);
   }
@@ -179,6 +181,8 @@ public MetricsRegionServerSourceImpl(String metricsName,
       WARN_THRESHOLD_COUNT_DESC, 0L);
     pausesWithGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITH_GC_KEY);
     pausesWithoutGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITHOUT_GC_KEY);
+
+    scannerLeaseExpiredCount = getMetricsRegistry().newCounter(SCANNER_LEASE_EXPIRED_COUNT, SCANNER_LEASE_EXPIRED_COUNT_DESC, 0L);
   }
 
   @Override
@@ -322,6 +326,11 @@ public void updateCompactionOutputSize(boolean isMajor, long bytes) {
     }
   }
 
+  @Override
+  public void incrScannerLeaseExpired() {
+    scannerLeaseExpiredCount.incr();
+  }
+
   /**
    * Yes this is a get function that doesn't return anything.  Thanks Hadoop for breaking all
    * expectations of java programmers.  Instead of returning anything Hadoop metrics expects
@@ -586,7 +595,9 @@ private MetricsRecordBuilder addGaugesToMetricsRecordBuilder(MetricsRecordBuilde
                 rsWrap.getByteBuffAllocatorTotalBufferCount())
             .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT,
                 BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC),
-                rsWrap.getByteBuffAllocatorUsedBufferCount());
+                rsWrap.getByteBuffAllocatorUsedBufferCount())
+            .addGauge(Interns.info(ACTIVE_SCANNERS, ACTIVE_SCANNERS_DESC),
+                rsWrap.getActiveScanners());
   }
 
   @Override

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -569,4 +569,6 @@ public interface MetricsRegionServerWrapper {
   long getByteBuffAllocatorTotalBufferCount();
 
   long getByteBuffAllocatorUsedBufferCount();
+
+  int getActiveScanners();
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java

@@ -317,4 +317,8 @@ public void updateWriteQueryMeter(TableName tn) {
       serverWriteQueryMeter.mark();
     }
   }
+
+  public void incrScannerLeaseExpired() {
+    serverSource.incrScannerLeaseExpired();
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -672,6 +672,11 @@ public double getMobFileCacheHitPercent() {
     return mobFileCacheHitRatio * 100;
   }
 
+  @Override
+  public int getActiveScanners() {
+    return regionServer.getRpcServices().getScannersCount();
+  }
+
   /**
    * This is the runnable that will be executed on the executor every PERIOD number of seconds
    * It will take metrics/numbers from all of the regions and use them to compute point in

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

@@ -478,6 +478,7 @@ public void leaseExpired() {
         return;
       }
       LOG.info("Scanner lease {} expired {}", this.scannerName, rsh);
+      server.getMetrics().incrScannerLeaseExpired();
       RegionScanner s = rsh.s;
       HRegion region = null;
       try {

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -143,6 +143,11 @@ public long getByteBuffAllocatorUsedBufferCount() {
     return 0;
   }
 
+  @Override
+  public int getActiveScanners() {
+    return 0;
+  }
+
   @Override
   public long getReadRequestsCount() {
     return 997;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java

@@ -250,6 +250,14 @@ public void testPauseMonitor() {
     HELPER.assertCounter("pauseTimeWithGc_num_ops", 1, serverSource);
   }
 
+  @Test
+  public void testScannerMetrics() {
+    HELPER.assertCounter("scannerLeaseExpiredCount", 0, serverSource);
+    rsm.incrScannerLeaseExpired();
+    HELPER.assertCounter("scannerLeaseExpiredCount", 1, serverSource);
+    HELPER.assertGauge("activeScanners", 0, serverSource);
+  }
+
   @Test
   public void testTableQueryMeterSwitch() {
     TableName tn1 = TableName.valueOf("table1");