diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java index 4d9ebc28eb2b..b29e6a49dc62 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java @@ -160,4 +160,10 @@ default String getNameAsString() { * of this region */ int getMaxStoreFileRefCount(); + + /** + * @return the max reference count for any store file among all compacted stores files + * of this region + */ + int getMaxCompactedStoreFileRefCount(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java index 25d44a933a85..78fabc8d9272 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java @@ -67,6 +67,7 @@ public static RegionMetrics toRegionMetrics(ClusterStatusProtos.RegionLoad regio .setStoreFileCount(regionLoadPB.getStorefiles()) .setStoreRefCount(regionLoadPB.getStoreRefCount()) .setMaxStoreFileRefCount(regionLoadPB.getMaxStoreFileRefCount()) + .setMaxCompactedStoreFileRefCount(regionLoadPB.getMaxCompactedStoreFileRefCount()) .setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE)) .setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream() .collect(Collectors.toMap( @@ -115,6 +116,7 @@ public static ClusterStatusProtos.RegionLoad toRegionLoad(RegionMetrics regionMe .setStorefiles(regionMetrics.getStoreFileCount()) .setStoreRefCount(regionMetrics.getStoreRefCount()) .setMaxStoreFileRefCount(regionMetrics.getMaxStoreFileRefCount()) + .setMaxCompactedStoreFileRefCount(regionMetrics.getMaxCompactedStoreFileRefCount()) .setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE)) .addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId())) .setStoreUncompressedSizeMB( @@ -131,6 +133,7 @@ public static RegionMetricsBuilder newBuilder(byte[] name) { private int storeFileCount; private int storeRefCount; private int maxStoreFileRefCount; + private int maxCompactedStoreFileRefCount; private long compactingCellCount; private long compactedCellCount; private Size storeFileSize = Size.ZERO; @@ -168,6 +171,10 @@ public RegionMetricsBuilder setMaxStoreFileRefCount(int value) { this.maxStoreFileRefCount = value; return this; } + public RegionMetricsBuilder setMaxCompactedStoreFileRefCount(int value) { + this.maxCompactedStoreFileRefCount = value; + return this; + } public RegionMetricsBuilder setCompactingCellCount(long value) { this.compactingCellCount = value; return this; @@ -243,6 +250,7 @@ public RegionMetrics build() { storeFileCount, storeRefCount, maxStoreFileRefCount, + maxCompactedStoreFileRefCount, compactingCellCount, compactedCellCount, storeFileSize, @@ -268,6 +276,7 @@ private static class RegionMetricsImpl implements RegionMetrics { private final int storeFileCount; private final int storeRefCount; private final int maxStoreFileRefCount; + private final int maxCompactedStoreFileRefCount; private final long compactingCellCount; private final long compactedCellCount; private final Size storeFileSize; @@ -290,6 +299,7 @@ private static class RegionMetricsImpl implements RegionMetrics { int storeFileCount, int storeRefCount, int maxStoreFileRefCount, + int maxCompactedStoreFileRefCount, final long compactingCellCount, long compactedCellCount, Size storeFileSize, @@ -312,6 +322,7 @@ private static class RegionMetricsImpl implements RegionMetrics { this.storeFileCount = storeFileCount; this.storeRefCount = storeRefCount; this.maxStoreFileRefCount = maxStoreFileRefCount; + this.maxCompactedStoreFileRefCount = maxCompactedStoreFileRefCount; this.compactingCellCount = compactingCellCount; this.compactedCellCount = compactedCellCount; this.storeFileSize = Preconditions.checkNotNull(storeFileSize); @@ -356,6 +367,11 @@ public int getMaxStoreFileRefCount() { return maxStoreFileRefCount; } + @Override + public int getMaxCompactedStoreFileRefCount() { + return maxCompactedStoreFileRefCount; + } + @Override public Size getStoreFileSize() { return storeFileSize; @@ -451,6 +467,8 @@ public String toString() { this.getStoreRefCount()); Strings.appendKeyValue(sb, "maxStoreFileRefCount", this.getMaxStoreFileRefCount()); + Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount", + this.getMaxCompactedStoreFileRefCount()); Strings.appendKeyValue(sb, "uncompressedStoreFileSize", this.getUncompressedStoreFileSize()); Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp", diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java index b408f0268864..f9ed7745226f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java @@ -345,6 +345,7 @@ public String toString() { int storeFileCount = 0; int storeRefCount = 0; int maxStoreFileRefCount = 0; + int maxCompactedStoreFileRefCount = 0; long uncompressedStoreFileSizeMB = 0; long storeFileSizeMB = 0; long memStoreSizeMB = 0; @@ -363,6 +364,9 @@ public String toString() { storeRefCount += r.getStoreRefCount(); int currentMaxStoreFileRefCount = r.getMaxStoreFileRefCount(); maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount); + int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount(); + maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount, + currentMaxCompactedStoreFileRefCount); uncompressedStoreFileSizeMB += r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE); storeFileSizeMB += r.getStoreFileSize().get(Size.Unit.MEGABYTE); memStoreSizeMB += r.getMemStoreSize().get(Size.Unit.MEGABYTE); @@ -386,6 +390,8 @@ public String toString() { Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount); Strings.appendKeyValue(sb, "storeRefCount", storeRefCount); Strings.appendKeyValue(sb, "maxStoreFileRefCount", maxStoreFileRefCount); + Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount", + maxCompactedStoreFileRefCount); Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB); Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB); if (uncompressedStoreFileSizeMB != 0) { diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index f738f0500b6d..dc566fdc2cb7 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -1915,15 +1915,15 @@ possible configurations would overwhelm and obscure the important. hbase.regions.recovery.store.file.ref.count -1 - Very large ref count on a file indicates + Very large ref count on a compacted store file indicates that it is a ref leak on that object. Such files - can not be removed even after it is invalidated + can not be removed after it is invalidated via compaction. Only way to recover in such scenario is to reopen the region which can release all resources, like the refcount, leases, etc. This config represents Store files Ref Count threshold value considered for reopening regions. - Any region with store files ref count > this value + Any region with compacted store files ref count > this value would be eligible for reopening by master. Default value -1 indicates this feature is turned off. Only positive integer value should be provided to enable diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index f4fbf390ae75..eebe75491cca 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -234,6 +234,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String STORE_REF_COUNT = "storeRefCount"; String STORE_REF_COUNT_DESC = "Store reference count"; String MAX_STORE_FILE_REF_COUNT = "maxStoreFileRefCount"; + String MAX_COMPACTED_STORE_FILE_REF_COUNT = "maxCompactedStoreFileRefCount"; String MEMSTORE_SIZE = "memStoreSize"; String MEMSTORE_SIZE_DESC = "Size of the memstore"; String STOREFILE_SIZE = "storeFileSize"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index ccdc209575f9..ce890ff496c4 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -170,4 +170,10 @@ public interface MetricsRegionWrapper { * all store files that belong to this region */ long getMaxStoreFileRefCount(); + + /** + * @return the max number of references active on any store file among + * all compacted store files that belong to this region + */ + long getMaxCompactedStoreFileRefCount(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index bc8ecb930dc8..f4a565b08174 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -221,6 +221,11 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT, MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT), this.regionWrapper.getMaxStoreFileRefCount()); + mrb.addGauge(Interns.info( + regionNamePrefix + MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT, + MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT), + this.regionWrapper.getMaxCompactedStoreFileRefCount() + ); mrb.addGauge(Interns.info( regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE, MetricsRegionServerSource.MEMSTORE_SIZE_DESC), diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index bafa854feb39..9f42228a8998 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -104,6 +104,11 @@ public long getMaxStoreFileRefCount() { return 0; } + @Override + public long getMaxCompactedStoreFileRefCount() { + return 0; + } + @Override public long getMemStoreSize() { return 0; diff --git a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto index cdc9c06bbb4b..fafeb7e1ff77 100644 --- a/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto @@ -155,6 +155,12 @@ message RegionLoad { * that belong to given region */ optional int32 max_store_file_ref_count = 22 [default = 0]; + + /** + * The max number of references active on single store file among all compacted store files + * that belong to given region + */ + optional int32 max_compacted_store_file_ref_count = 23 [default = 0]; } /* Server-level protobufs */ diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto b/hbase-protocol/src/main/protobuf/ClusterStatus.proto index f920fff79f16..2e94fa94da3c 100644 --- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto @@ -151,6 +151,12 @@ message RegionLoad { * that belong to given region */ optional int32 max_store_file_ref_count = 22 [default = 0]; + + /** + * The max number of references active on single store file among all compacted store files + * that belong to given region + */ + optional int32 max_compacted_store_file_ref_count = 23 [default = 0]; } /* Server-level protobufs */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java index 9f204343bc08..6b5f7ba84efb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java @@ -130,12 +130,12 @@ private Map> getTableToRegionsByRefCount( for (ServerMetrics serverMetrics : serverMetricsMap.values()) { Map regionMetricsMap = serverMetrics.getRegionMetrics(); for (RegionMetrics regionMetrics : regionMetricsMap.values()) { - // For each region, each store file can have different ref counts - // We need to find maximum of all such ref counts and if that max count - // is beyond a threshold value, we should reopen the region. - // Here, we take max ref count of all store files and not the cumulative - // count of all store files - final int maxStoreFileRefCount = regionMetrics.getMaxStoreFileRefCount(); + // For each region, each compacted store file can have different ref counts + // We need to find maximum of all such ref counts and if that max count of compacted + // store files is beyond a threshold value, we should reopen the region. + // Here, we take max ref count of all compacted store files and not the cumulative + // count of all compacted store files + final int maxStoreFileRefCount = regionMetrics.getMaxCompactedStoreFileRefCount(); if (maxStoreFileRefCount > storeFileRefCountThreshold) { final byte[] regionName = regionMetrics.getRegionName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index f9a3ba01b0c1..17050a62bda9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1630,6 +1630,7 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, int storefiles = 0; int storeRefCount = 0; int maxStoreFileRefCount = 0; + int maxCompactedStoreFileRefCount = 0; int storeUncompressedSizeMB = 0; int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.getMemStoreDataSize() / 1024 / 1024); @@ -1647,6 +1648,9 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, storeRefCount += currentStoreRefCount; int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount(); maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount); + int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount(); + maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount, + currentMaxCompactedStoreFileRefCount); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024); //TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB? @@ -1676,6 +1680,7 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, .setStorefiles(storefiles) .setStoreRefCount(storeRefCount) .setMaxStoreFileRefCount(maxStoreFileRefCount) + .setMaxCompactedStoreFileRefCount(maxCompactedStoreFileRefCount) .setStoreUncompressedSizeMB(storeUncompressedSizeMB) .setStorefileSizeMB(storefileSizeMB) .setMemStoreSizeMB(memstoreSizeMB) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 9e105f7a7ee0..9a771a72347a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -2798,4 +2798,20 @@ public int getMaxStoreFileRefCount() { return maxStoreFileRefCount.isPresent() ? maxStoreFileRefCount.getAsInt() : 0; } + /** + * @return get maximum ref count of storeFile among all compacted HStore Files + * for the HStore + */ + public int getMaxCompactedStoreFileRefCount() { + OptionalInt maxCompactedStoreFileRefCount = this.storeEngine.getStoreFileManager() + .getCompactedfiles() + .stream() + .filter(sf -> sf.getReader() != null) + .filter(HStoreFile::isHFile) + .mapToInt(HStoreFile::getRefCount) + .max(); + return maxCompactedStoreFileRefCount.isPresent() + ? maxCompactedStoreFileRefCount.getAsInt() : 0; + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index e15623dff35e..ee60eeca8136 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -50,6 +50,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable private long numStoreFiles; private long storeRefCount; private long maxStoreFileRefCount; + private long maxCompactedStoreFileRefCount; private long memstoreSize; private long storeFileSize; private long maxStoreFileAge; @@ -131,6 +132,11 @@ public long getMaxStoreFileRefCount() { return maxStoreFileRefCount; } + @Override + public long getMaxCompactedStoreFileRefCount() { + return maxCompactedStoreFileRefCount; + } + @Override public long getReadRequestCount() { return this.region.getReadRequestsCount(); @@ -240,6 +246,7 @@ public void run() { long tempNumStoreFiles = 0; int tempStoreRefCount = 0; int tempMaxStoreFileRefCount = 0; + int tempMaxCompactedStoreFileRefCount = 0; long tempMemstoreSize = 0; long tempStoreFileSize = 0; long tempMaxStoreFileAge = 0; @@ -257,6 +264,9 @@ public void run() { int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount(); tempMaxStoreFileRefCount = Math.max(tempMaxStoreFileRefCount, currentMaxStoreFileRefCount); + int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount(); + tempMaxCompactedStoreFileRefCount = Math.max(tempMaxCompactedStoreFileRefCount, + currentMaxCompactedStoreFileRefCount); tempMemstoreSize += store.getMemStoreSize().getDataSize(); tempStoreFileSize += store.getStorefilesSize(); OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge(); @@ -285,6 +295,7 @@ public void run() { numStoreFiles = tempNumStoreFiles; storeRefCount = tempStoreRefCount; maxStoreFileRefCount = tempMaxStoreFileRefCount; + maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount; memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; maxStoreFileAge = tempMaxStoreFileAge; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java index 524d445a68c5..4b7f5e3a32af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java @@ -375,7 +375,7 @@ public long getLastReportTimestamp() { return serverMetrics; } - private static RegionMetrics getRegionMetrics(byte[] regionName, int storeRefCount) { + private static RegionMetrics getRegionMetrics(byte[] regionName, int compactedStoreRefCount) { RegionMetrics regionMetrics = new RegionMetrics() { @Override @@ -480,12 +480,17 @@ public long getLastMajorCompactionTimestamp() { @Override public int getStoreRefCount() { - return storeRefCount; + return compactedStoreRefCount; } @Override public int getMaxStoreFileRefCount() { - return storeRefCount; + return 0; + } + + @Override + public int getMaxCompactedStoreFileRefCount() { + return compactedStoreRefCount; } }; @@ -610,4 +615,4 @@ public boolean isStopped() { } -} \ No newline at end of file +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index 1e5cb3ed2e95..2255f9bd4ded 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -70,6 +70,11 @@ public long getMaxStoreFileRefCount() { return 0; } + @Override + public long getMaxCompactedStoreFileRefCount() { + return 0; + } + @Override public long getMemStoreSize() { return 103; diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc b/src/main/asciidoc/_chapters/hbase-default.adoc index 59a3a073c362..653ed37a2b89 100644 --- a/src/main/asciidoc/_chapters/hbase-default.adoc +++ b/src/main/asciidoc/_chapters/hbase-default.adoc @@ -2183,15 +2183,15 @@ The percent of region server RPC threads failed to abort RS. + .Description - Very large ref count on a file indicates + Very large ref count on a compacted store file indicates that it is a ref leak on that object. Such files - can not be removed even after it is invalidated + can not be removed after it is invalidated via compaction. Only way to recover in such scenario is to reopen the region which can release all resources, like the refcount, leases, etc. This config represents Store files Ref Count threshold value considered for reopening regions. - Any region with store files ref count > this value + Any region with compacted store files ref count > this value would be eligible for reopening by master. Default value -1 indicates this feature is turned off. Only positive integer value should be provided to enable