diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index 84323e5c6594..7b9ac9d50688 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -456,7 +456,7 @@ public static String createFromHFileLink(final Configuration conf, final FileSys * Create the back reference name */ // package-private for testing - static String createBackReferenceName(final String tableNameStr, final String regionName) { + public static String createBackReferenceName(final String tableNameStr, final String regionName) { return regionName + "." + tableNameStr.replace(TableName.NAMESPACE_DELIM, '='); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java index 337fde60cf7d..22d3c9ce2c0b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/Reference.java @@ -195,7 +195,7 @@ public static Reference convert(final FSProtos.Reference r) { * delimiter, pb reads to EOF which may not be what you want). * @return This instance serialized as a delimited protobuf w/ a magic pb prefix. */ - byte[] toByteArray() throws IOException { + public byte[] toByteArray() throws IOException { return ProtobufUtil.prependPBMagic(convert().toByteArray()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java index 7d4ec71d35b1..bc3d0feeebd4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeTableRegionsProcedure.java @@ -612,7 +612,7 @@ private List mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem reg // to read the hfiles. storeFileInfo.setConf(storeConfiguration); Path refFile = mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family, - new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), tracker); mergedFiles.add(refFile); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java index 2e2182b25d29..09703d9a8176 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java @@ -666,8 +666,9 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en // table dir. In case of failure, the proc would go through this again, already existing // region dirs and split files would just be ignored, new split files should get created. int nbFiles = 0; - final Map> files = - new HashMap>(htd.getColumnFamilyCount()); + final Map, StoreFileTracker>> files = + new HashMap, StoreFileTracker>>( + htd.getColumnFamilyCount()); for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) { String family = cfd.getNameAsString(); StoreFileTracker tracker = @@ -690,7 +691,7 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en } if (filteredSfis == null) { filteredSfis = new ArrayList(sfis.size()); - files.put(family, filteredSfis); + files.put(family, new Pair(filteredSfis, tracker)); } filteredSfis.add(sfi); nbFiles++; @@ -713,10 +714,11 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en final List>> futures = new ArrayList>>(nbFiles); // Split each store file. - for (Map.Entry> e : files.entrySet()) { + for (Map.Entry, StoreFileTracker>> e : files + .entrySet()) { byte[] familyName = Bytes.toBytes(e.getKey()); final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName); - final Collection storeFiles = e.getValue(); + final Collection storeFiles = e.getValue().getFirst(); if (storeFiles != null && storeFiles.size() > 0) { final Configuration storeConfiguration = StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd); @@ -727,8 +729,9 @@ private Pair, List> splitStoreFiles(final MasterProcedureEnv en // is running in a regionserver's Store context, or we might not be able // to read the hfiles. storeFileInfo.setConf(storeConfiguration); - StoreFileSplitter sfs = new StoreFileSplitter(regionFs, familyName, - new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); + StoreFileSplitter sfs = + new StoreFileSplitter(regionFs, e.getValue().getSecond(), familyName, + new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED)); futures.add(threadPool.submit(sfs)); } } @@ -794,8 +797,8 @@ private void assertSplitResultFilesCount(final FileSystem fs, } } - private Pair splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) - throws IOException { + private Pair splitStoreFile(HRegionFileSystem regionFs, StoreFileTracker tracker, + byte[] family, HStoreFile sf) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); @@ -803,10 +806,10 @@ private Pair splitStoreFile(HRegionFileSystem regionFs, byte[] famil final byte[] splitRow = getSplitRow(); final String familyName = Bytes.toString(family); - final Path path_first = - regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, false, splitPolicy); - final Path path_second = - regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, true, splitPolicy); + final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, + false, splitPolicy, tracker); + final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, + true, splitPolicy, tracker); if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); @@ -822,6 +825,7 @@ private class StoreFileSplitter implements Callable> { private final HRegionFileSystem regionFs; private final byte[] family; private final HStoreFile sf; + private final StoreFileTracker tracker; /** * Constructor that takes what it needs to split @@ -829,15 +833,17 @@ private class StoreFileSplitter implements Callable> { * @param family Family that contains the store file * @param sf which file */ - public StoreFileSplitter(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) { + public StoreFileSplitter(HRegionFileSystem regionFs, StoreFileTracker tracker, byte[] family, + HStoreFile sf) { this.regionFs = regionFs; this.sf = sf; this.family = family; + this.tracker = tracker; } @Override public Pair call() throws IOException { - return splitStoreFile(regionFs, family, sf); + return splitStoreFile(regionFs, tracker, family, sf); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java index df87000d9216..87c687e74ed3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; @@ -50,6 +51,8 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; @@ -422,7 +425,7 @@ private static Pair checkRegionReferences(MasterServices servi try { HRegionFileSystem regionFs = HRegionFileSystem .openRegionFromFileSystem(services.getConfiguration(), fs, tabledir, region, true); - boolean references = regionFs.hasReferences(tableDescriptor); + boolean references = hasReferences(services.getConfiguration(), regionFs, tableDescriptor); return new Pair<>(Boolean.TRUE, references); } catch (IOException e) { LOG.error("Error trying to determine if region {} has references, assuming it does", @@ -431,6 +434,17 @@ private static Pair checkRegionReferences(MasterServices servi } } + private static boolean hasReferences(Configuration conf, HRegionFileSystem regionFs, + TableDescriptor htd) throws IOException { + for (ColumnFamilyDescriptor family : htd.getColumnFamilies()) { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, family, regionFs, false); + if (sft.hasReferences(family.getNameAsString())) { + return true; + } + } + return false; + } + private void updateAssignmentManagerMetrics() { services.getAssignmentManager().getAssignmentManagerMetrics() .updateHoles(lastReport.getHoles().size()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java index 1c1145a2f482..f0ccd131e91a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/CachedMobFile.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.yetus.audience.InterfaceAudience; /** @@ -41,10 +42,12 @@ public CachedMobFile(HStoreFile sf) { } public static CachedMobFile create(FileSystem fs, Path path, Configuration conf, - CacheConfig cacheConf) throws IOException { - // XXX: primaryReplica is only used for constructing the key of block cache so it is not a - // critical problem if we pass the wrong value, so here we always pass true. Need to fix later. - HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); + CacheConfig cacheConf, StoreFileTracker sft) throws IOException { + // XXX: primaryReplica is only used for constructing the key of block cache so + // it is not a + // critical problem if we pass the wrong value, so here we always pass true. + // Need to fix later. + HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft); return new CachedMobFile(sf); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java index dcedd02ad285..0fa61d67095a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/ExpiredMobFileCleaner.java @@ -56,17 +56,17 @@ public class ExpiredMobFileCleaner extends Configured implements Tool { * @param tableName The current table name. * @param family The current family. */ - public void cleanExpiredMobFiles(String tableName, ColumnFamilyDescriptor family) + public void cleanExpiredMobFiles(TableDescriptor htd, ColumnFamilyDescriptor family) throws IOException { Configuration conf = getConf(); - TableName tn = TableName.valueOf(tableName); + String tableName = htd.getTableName().getNameAsString(); FileSystem fs = FileSystem.get(conf); LOG.info("Cleaning the expired MOB files of " + family.getNameAsString() + " in " + tableName); // disable the block cache. Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); - MobUtils.cleanExpiredMobFiles(fs, conf, tn, family, cacheConfig, + MobUtils.cleanExpiredMobFiles(fs, conf, htd, family, cacheConfig, EnvironmentEdgeManager.currentTime()); } @@ -105,7 +105,7 @@ public int run(String[] args) throws Exception { throw new IOException( "The minVersions of the column family is not 0, could not be handled by this cleaner"); } - cleanExpiredMobFiles(tableName, family); + cleanExpiredMobFiles(htd, family); return 0; } finally { admin.close(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java index 3293208771ac..9a6352e03718 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFile.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.yetus.audience.InterfaceAudience; /** @@ -133,11 +134,13 @@ public void close() throws IOException { * @param cacheConf The CacheConfig. * @return An instance of the MobFile. */ - public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf) - throws IOException { - // XXX: primaryReplica is only used for constructing the key of block cache so it is not a - // critical problem if we pass the wrong value, so here we always pass true. Need to fix later. - HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); + public static MobFile create(FileSystem fs, Path path, Configuration conf, CacheConfig cacheConf, + StoreFileTracker sft) throws IOException { + // XXX: primaryReplica is only used for constructing the key of block cache so + // it is not a + // critical problem if we pass the wrong value, so here we always pass true. + // Need to fix later. + HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true, sft); return new MobFile(sf); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java index b353b53ffb71..e6ec4e89dfb7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCache.java @@ -33,6 +33,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.StoreContext; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.IdLock; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -198,9 +201,11 @@ public void evictFile(String fileName) { * @param cacheConf The current MobCacheConfig * @return A opened mob file. */ - public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws IOException { + public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf, + StoreContext storeContext) throws IOException { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); if (!isCacheEnabled) { - MobFile mobFile = MobFile.create(fs, path, conf, cacheConf); + MobFile mobFile = MobFile.create(fs, path, conf, cacheConf, sft); mobFile.open(); return mobFile; } else { @@ -214,7 +219,7 @@ public MobFile openFile(FileSystem fs, Path path, CacheConfig cacheConf) throws if (map.size() > mobFileMaxCacheSize) { evict(); } - cached = CachedMobFile.create(fs, path, conf, cacheConf); + cached = CachedMobFile.create(fs, path, conf, cacheConf, sft); cached.open(); map.put(fileName, cached); miss.increment(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java index fda9f1292eb6..3144b71f11e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanerChore.java @@ -87,7 +87,7 @@ protected void chore() { for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { if (hcd.isMobEnabled() && hcd.getMinVersions() == 0) { try { - cleaner.cleanExpiredMobFiles(htd.getTableName().getNameAsString(), hcd); + cleaner.cleanExpiredMobFiles(htd, hcd); } catch (IOException e) { LOG.error("Failed to clean the expired mob files table={} family={}", htd.getTableName().getNameAsString(), hcd.getNameAsString(), e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java index 049192624ef3..e7a9af6a8289 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCleanupUtil.java @@ -35,7 +35,11 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -90,7 +94,9 @@ public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, Set allActiveMobFileName = new HashSet(); for (Path regionPath : regionDirs) { regionNames.add(regionPath.getName()); + HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, tableDir, regionPath); for (ColumnFamilyDescriptor hcd : list) { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false); String family = hcd.getNameAsString(); Path storePath = new Path(regionPath, family); boolean succeed = false; @@ -102,26 +108,19 @@ public static void cleanupObsoleteMobFiles(Configuration conf, TableName table, + " execution, aborting MOB file cleaner chore.", storePath); throw new IOException(errMsg); } - RemoteIterator rit = fs.listLocatedStatus(storePath); - List storeFiles = new ArrayList(); - // Load list of store files first - while (rit.hasNext()) { - Path p = rit.next().getPath(); - if (fs.isFile(p)) { - storeFiles.add(p); - } - } - LOG.info("Found {} store files in: {}", storeFiles.size(), storePath); + List storeFileInfos = sft.load(); + LOG.info("Found {} store files in: {}", storeFileInfos.size(), storePath); Path currentPath = null; try { - for (Path pp : storeFiles) { + for (StoreFileInfo storeFileInfo : storeFileInfos) { + Path pp = storeFileInfo.getPath(); currentPath = pp; LOG.trace("Store file: {}", pp); HStoreFile sf = null; byte[] mobRefData = null; byte[] bulkloadMarkerData = null; try { - sf = new HStoreFile(fs, pp, conf, CacheConfig.DISABLED, BloomType.NONE, true); + sf = new HStoreFile(storeFileInfo, BloomType.NONE, CacheConfig.DISABLED); sf.initReader(); mobRefData = sf.getMetadataValue(HStoreFile.MOB_FILE_REFS); bulkloadMarkerData = sf.getMetadataValue(HStoreFile.BULKLOAD_TASK_KEY); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java index 60f0f126ab60..e4c1f0373b30 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java @@ -59,9 +59,12 @@ import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HStoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -266,7 +269,7 @@ public static void setCacheMobBlocks(Scan scan, boolean cacheBlocks) { * @param cacheConfig The cacheConfig that disables the block cache. * @param current The current time. */ - public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, + public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableDescriptor htd, ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current) throws IOException { long timeToLive = columnDescriptor.getTimeToLive(); @@ -287,7 +290,11 @@ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, Table LOG.info("MOB HFiles older than " + expireDate.toGMTString() + " will be deleted!"); FileStatus[] stats = null; + TableName tableName = htd.getTableName(); Path mobTableDir = CommonFSUtils.getTableDir(getMobHome(conf), tableName); + Path regionDir = getMobRegionPath(conf, tableName); + HRegionFileSystem regionFS = HRegionFileSystem.create(conf, fs, mobTableDir, regionDir); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, columnDescriptor, regionFS); Path path = getMobFamilyPath(conf, tableName, columnDescriptor.getNameAsString()); try { stats = fs.listStatus(path); @@ -318,7 +325,7 @@ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, Table LOG.debug("{} is an expired file", fileName); } filesToClean - .add(new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true)); + .add(new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true, sft)); if ( filesToClean.size() >= conf.getInt(MOB_CLEANER_BATCH_SIZE_UPPER_BOUND, DEFAULT_MOB_CLEANER_BATCH_SIZE_UPPER_BOUND) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/RSMobFileCleanerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/RSMobFileCleanerChore.java index 06e349887337..91b85e3dbffd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/RSMobFileCleanerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/RSMobFileCleanerChore.java @@ -31,19 +31,23 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; +import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -112,6 +116,8 @@ protected void chore() { for (HRegion region : regions) { for (ColumnFamilyDescriptor hcd : list) { HStore store = region.getStore(hcd.getName()); + StoreFileTrackerFactory.create(region.getReadOnlyConfiguration(), false, + store.getStoreContext()); Collection sfs = store.getStorefiles(); Set regionMobs = new HashSet(); Path currentPath = null; @@ -191,11 +197,16 @@ protected void chore() { for (ColumnFamilyDescriptor hcd : list) { List toArchive = new ArrayList(); String family = hcd.getNameAsString(); - Path dir = MobUtils.getMobFamilyPath(rs.getConfiguration(), htd.getTableName(), family); - RemoteIterator rit = fs.listLocatedStatus(dir); - while (rit.hasNext()) { - LocatedFileStatus lfs = rit.next(); - Path p = lfs.getPath(); + RegionInfo regionInfo = MobUtils.getMobRegionInfo(htd.getTableName()); + HRegionFileSystem regionFS = HRegionFileSystem.create(rs.getConfiguration(), fs, + MobUtils.getMobRegionPath(new Path(rs.getConfiguration().get(HConstants.HBASE_DIR)), + htd.getTableName()), + regionInfo); + StoreFileTracker sft = + StoreFileTrackerFactory.create(rs.getConfiguration(), htd, hcd, regionFS); + List storeFileInfos = sft.load(); + for (StoreFileInfo storeFileInfo : storeFileInfos) { + Path p = storeFileInfo.getPath(); String[] mobParts = p.getName().split("_"); String regionName = mobParts[mobParts.length - 1]; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateHFileLinkParams.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateHFileLinkParams.java new file mode 100644 index 000000000000..f64a5ee60252 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CreateHFileLinkParams.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class CreateHFileLinkParams { + + private Configuration conf; + private FileSystem fs; + private Path dstFamilyPath; + private String familyName; + private String dstTableName; + private String dstRegionName; + private TableName linkedTable; + private String linkedRegion; + private String hfileName; + private boolean createBackRef; + + public CreateHFileLinkParams() { + + } + + public Configuration getConf() { + return conf; + } + + public CreateHFileLinkParams conf(Configuration conf) { + this.conf = conf; + return this; + } + + public FileSystem getFs() { + return fs; + } + + public CreateHFileLinkParams fileSystem(FileSystem fs) { + this.fs = fs; + return this; + } + + public Path getDstFamilyPath() { + return dstFamilyPath; + } + + public CreateHFileLinkParams dstFamilyPath(Path dstFamilyPath) { + this.dstFamilyPath = dstFamilyPath; + return this; + } + + public String getFamilyName() { + return familyName; + } + + public CreateHFileLinkParams familyName(String familyName) { + this.familyName = familyName; + return this; + } + + public String getDstTableName() { + return dstTableName; + } + + public CreateHFileLinkParams dstTableName(String dstTableName) { + this.dstTableName = dstTableName; + return this; + } + + public String getDstRegionName() { + return dstRegionName; + } + + public CreateHFileLinkParams dstRegionName(String dstRegionName) { + this.dstRegionName = dstRegionName; + return this; + } + + public TableName getLinkedTable() { + return linkedTable; + } + + public CreateHFileLinkParams linkedTable(TableName linkedTable) { + this.linkedTable = linkedTable; + return this; + } + + public String getLinkedRegion() { + return linkedRegion; + } + + public CreateHFileLinkParams linkedRegion(String linkedRegion) { + this.linkedRegion = linkedRegion; + return this; + } + + public String getHfileName() { + return hfileName; + } + + public CreateHFileLinkParams hfileName(String hfileName) { + this.hfileName = hfileName; + return this; + } + + public boolean isCreateBackRef() { + return createBackRef; + } + + public CreateHFileLinkParams createBackRef(boolean createBackRef) { + this.createBackRef = createBackRef; + return this; + } + + public static CreateHFileLinkParams create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName) + throws IOException { + return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true); + } + + public static CreateHFileLinkParams create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName, + final boolean createBackRef) throws IOException { + TableName linkedTable = hfileRegionInfo.getTable(); + String linkedRegion = hfileRegionInfo.getEncodedName(); + return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef); + } + + public static CreateHFileLinkParams create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, + final String hfileName) throws IOException { + return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true); + } + + public static CreateHFileLinkParams create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, + final String hfileName, final boolean createBackRef) throws IOException { + String familyName = dstFamilyPath.getName(); + String regionName = dstFamilyPath.getParent().getName(); + String tableName = + CommonFSUtils.getTableName(dstFamilyPath.getParent().getParent()).getNameAsString(); + + return create(conf, fs, dstFamilyPath, familyName, tableName, regionName, linkedTable, + linkedRegion, hfileName, createBackRef); + } + + public static CreateHFileLinkParams create(final Configuration conf, final FileSystem fs, + final Path dstFamilyPath, final String familyName, final String dstTableName, + final String dstRegionName, final TableName linkedTable, final String linkedRegion, + final String hfileName, final boolean createBackRef) throws IOException { + return new CreateHFileLinkParams().conf(conf).fileSystem(fs).linkedTable(linkedTable) + .linkedRegion(linkedRegion).hfileName(hfileName).createBackRef(createBackRef) + .dstFamilyPath(dstFamilyPath).dstTableName(dstTableName).dstRegionName(dstRegionName); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java index 6a07a6c9a088..ae93ae0011f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HMobStore.java @@ -56,6 +56,8 @@ import org.apache.hadoop.hbase.mob.MobFileName; import org.apache.hadoop.hbase.mob.MobStoreEngine; import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.IdLock; import org.apache.yetus.audience.InterfaceAudience; @@ -280,8 +282,9 @@ public void commitFile(final Path sourceFile, Path targetPath) throws IOExceptio private void validateMobFile(Path path) throws IOException { HStoreFile storeFile = null; try { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, getStoreContext()); storeFile = new HStoreFile(getFileSystem(), path, conf, getCacheConfig(), BloomType.NONE, - isPrimaryReplicaStore()); + isPrimaryReplicaStore(), sft); storeFile.initReader(); } catch (IOException e) { LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e); @@ -405,7 +408,7 @@ private MobCell readCell(List locations, String fileName, Cell search, MobFile file = null; Path path = new Path(location, fileName); try { - file = mobFileCache.openFile(fs, path, getCacheConfig()); + file = mobFileCache.openFile(fs, path, getCacheConfig(), this.getStoreContext()); return readPt != -1 ? file.readCell(search, cacheMobBlocks, readPt) : file.readCell(search, cacheMobBlocks); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index ae4045b1216b..e2285630e346 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -152,6 +152,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.ForbidMajorCompactionChecker; import org.apache.hadoop.hbase.regionserver.metrics.MetricsTableRequests; import org.apache.hadoop.hbase.regionserver.regionreplication.RegionReplicationSink; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.CompactionThroughputControllerFactory; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; import org.apache.hadoop.hbase.regionserver.throttle.StoreHotnessProtector; @@ -1345,7 +1347,9 @@ public static HDFSBlocksDistribution computeHDFSBlocksDistribution(Configuration if (StoreFileInfo.isReference(p) || HFileLink.isHFileLink(p)) { // Only construct StoreFileInfo object if its not a hfile, save obj // creation - StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, status); + StoreFileTracker sft = + StoreFileTrackerFactory.create(conf, tableDescriptor, family, regionFs); + StoreFileInfo storeFileInfo = new StoreFileInfo(conf, fs, status.getPath(), sft); hdfsBlocksDistribution.add(storeFileInfo.computeHDFSBlocksDistribution(fs)); } else if (StoreFileInfo.isHFile(p)) { // If its a HFile, then lets just add to the block distribution @@ -5475,9 +5479,12 @@ long replayRecoveredEditsIfAny(Map maxSeqIdInStores, // column family. Have to fake out file type too by casting our recovered.edits as // storefiles String fakeFamilyName = WALSplitUtil.getRegionDirRecoveredEditsDir(regionWALDir).getName(); + StoreContext storeContext = + StoreContext.getBuilder().withRegionFileSystem(getRegionFileSystem()).build(); + StoreFileTracker sft = StoreFileTrackerFactory.create(this.conf, true, storeContext); Set fakeStoreFiles = new HashSet<>(files.size()); for (Path file : files) { - fakeStoreFiles.add(new HStoreFile(walFS, file, this.conf, null, null, true)); + fakeStoreFiles.add(new HStoreFile(walFS, file, this.conf, null, null, true, sft)); } getRegionWALFileSystem().archiveRecoveredEdits(fakeFamilyName, fakeStoreFiles); } else { @@ -6494,17 +6501,19 @@ void replayWALBulkLoadEventMarker(WALProtos.BulkLoadDescriptor bulkLoadEvent) th continue; } - List storeFiles = storeDescriptor.getStoreFileList(); - for (String storeFile : storeFiles) { - StoreFileInfo storeFileInfo = null; + StoreContext storeContext = store.getStoreContext(); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, + ServerRegionReplicaUtil.isDefaultReplica(getRegionInfo()), storeContext); + + List storeFiles = sft.load(); + for (StoreFileInfo storeFileInfo : storeFiles) { try { - storeFileInfo = fs.getStoreFileInfo(Bytes.toString(family), storeFile); store.bulkLoadHFile(storeFileInfo); } catch (FileNotFoundException ex) { LOG.warn(getRegionInfo().getEncodedName() + " : " + ((storeFileInfo != null) ? storeFileInfo.toString() - : (new Path(Bytes.toString(family), storeFile)).toString()) + : (new Path(Bytes.toString(family), storeFileInfo.toString())).toString()) + " doesn't exist any more. Skip loading the file"); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index 6fccccfc8203..89f9473ec357 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -255,51 +255,6 @@ public String getStoragePolicyName(String familyName) { return null; } - /** - * Returns the store files available for the family. This methods performs the filtering based on - * the valid store files. - * @param familyName Column Family Name - * @return a set of {@link StoreFileInfo} for the specified family. - */ - public List getStoreFiles(final String familyName) throws IOException { - return getStoreFiles(familyName, true); - } - - /** - * Returns the store files available for the family. This methods performs the filtering based on - * the valid store files. - * @param familyName Column Family Name - * @return a set of {@link StoreFileInfo} for the specified family. - */ - public List getStoreFiles(final String familyName, final boolean validate) - throws IOException { - Path familyDir = getStoreDir(familyName); - FileStatus[] files = CommonFSUtils.listStatus(this.fs, familyDir); - if (files == null) { - if (LOG.isTraceEnabled()) { - LOG.trace("No StoreFiles for: " + familyDir); - } - return null; - } - - ArrayList storeFiles = new ArrayList<>(files.length); - for (FileStatus status : files) { - if (validate && !StoreFileInfo.isValid(status)) { - // recovered.hfiles directory is expected inside CF path when hbase.wal.split.to.hfile to - // true, refer HBASE-23740 - if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) { - LOG.warn("Invalid StoreFile: {}", status.getPath()); - } - continue; - } - StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, - regionInfoForFs, familyName, status.getPath()); - storeFiles.add(info); - - } - return storeFiles; - } - /** * Returns the store files' LocatedFileStatus which available for the family. This methods * performs the filtering based on the valid store files. @@ -350,47 +305,11 @@ Path getStoreFilePath(final String familyName, final String fileName) { * @param fileName File Name * @return The {@link StoreFileInfo} for the specified family/file */ - StoreFileInfo getStoreFileInfo(final String familyName, final String fileName) - throws IOException { + StoreFileInfo getStoreFileInfo(final String familyName, final String fileName, + final StoreFileTracker tracker) throws IOException { Path familyDir = getStoreDir(familyName); return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs, - familyName, new Path(familyDir, fileName)); - } - - /** - * Returns true if the specified family has reference files - * @param familyName Column Family Name - * @return true if family contains reference files - */ - public boolean hasReferences(final String familyName) throws IOException { - Path storeDir = getStoreDir(familyName); - FileStatus[] files = CommonFSUtils.listStatus(fs, storeDir); - if (files != null) { - for (FileStatus stat : files) { - if (stat.isDirectory()) { - continue; - } - if (StoreFileInfo.isReference(stat.getPath())) { - LOG.trace("Reference {}", stat.getPath()); - return true; - } - } - } - return false; - } - - /** - * Check whether region has Reference file - * @param htd table desciptor of the region - * @return true if region has reference file - */ - public boolean hasReferences(final TableDescriptor htd) throws IOException { - for (ColumnFamilyDescriptor family : htd.getColumnFamilies()) { - if (hasReferences(family.getNameAsString())) { - return true; - } - } - return false; + familyName, new Path(familyDir, fileName), tracker); } /** Returns the set of families present on disk n */ @@ -628,7 +547,7 @@ private void insertRegionFilesIntoStoreTracker(List allFiles, MasterProced tblDesc.getColumnFamily(Bytes.toBytes(familyName)), regionFs)); fileInfoMap.computeIfAbsent(familyName, l -> new ArrayList<>()); List infos = fileInfoMap.get(familyName); - infos.add(new StoreFileInfo(conf, fs, file, true)); + infos.add(new StoreFileInfo(conf, fs, file, true, trackerMap.get(familyName))); } for (Map.Entry entry : trackerMap.entrySet()) { entry.getValue().add(fileInfoMap.get(entry.getKey())); @@ -672,7 +591,7 @@ public void createSplitsDir(RegionInfo daughterA, RegionInfo daughterB) throws I * @return Path to created reference. */ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte[] splitRow, - boolean top, RegionSplitPolicy splitPolicy) throws IOException { + boolean top, RegionSplitPolicy splitPolicy, StoreFileTracker tracker) throws IOException { Path splitDir = new Path(getSplitsDir(hri), familyName); // Add the referred-to regions name as a dot separated suffix. // See REF_NAME_REGEX regex above. The referred-to regions name is @@ -742,8 +661,10 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte hfileName = m.group(4); } // must create back reference here - HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(), - hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true); + CreateHFileLinkParams params = CreateHFileLinkParams.create(conf, fs, splitDir, familyName, + hri.getTable().getNameAsString(), hri.getEncodedName(), linkedTable, linkedRegion, + hfileName, true); + tracker.createHFileLink(params); Path path = new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName)); LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName() @@ -799,8 +720,8 @@ static boolean mkdirs(FileSystem fs, Configuration conf, Path dir) throws IOExce * @return Path to created reference. * @throws IOException if the merge write fails. */ - public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f) - throws IOException { + public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFile f, + StoreFileTracker tracker) throws IOException { Path referenceDir = new Path(getMergesDir(regionInfoForFs), familyName); // A whole reference to the store file. Reference r = Reference.createTopReference(mergingRegion.getStartKey()); @@ -812,7 +733,8 @@ public Path mergeStoreFile(RegionInfo mergingRegion, String familyName, HStoreFi // Write reference with same file id only with the other region name as // suffix and into the new region location (under same family). Path p = new Path(referenceDir, f.getPath().getName() + "." + mergingRegionName); - return r.write(fs, p); + tracker.createReference(r, p); + return p; } /** @@ -1197,4 +1119,14 @@ private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSl } Thread.sleep((long) baseSleepBeforeRetries * sleepMultiplier); } + + public static HRegionFileSystem create(final Configuration conf, final FileSystem fs, + final Path tableDir, final Path regionDir) throws IOException { + return new HRegionFileSystem(conf, fs, tableDir, loadRegionInfoFileContent(fs, regionDir)); + } + + public static HRegionFileSystem create(final Configuration conf, final FileSystem fs, + final Path tableDir, final RegionInfo regionInfo) throws IOException { + return new HRegionFileSystem(conf, fs, tableDir, regionInfo); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 43a63359961e..c7db2c1f9bf9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -87,6 +87,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl; import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController; import org.apache.hadoop.hbase.regionserver.wal.WALUtil; import org.apache.hadoop.hbase.security.EncryptionUtil; @@ -428,7 +430,7 @@ public static long determineTTLFromFamily(final ColumnFamilyDescriptor family) { return ttl; } - StoreContext getStoreContext() { + public StoreContext getStoreContext() { return storeContext; } @@ -1388,8 +1390,9 @@ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pick compactionOutputs.remove(sf.getPath().getName()); } for (String compactionOutput : compactionOutputs) { + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); StoreFileInfo storeFileInfo = - getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput); + getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), compactionOutput, sft); HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); outputStoreFiles.add(storeFile); } @@ -2032,8 +2035,9 @@ public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) List storeFiles = new ArrayList<>(fileNames.size()); for (String file : fileNames) { // open the file as a store file (hfile link, etc) + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true, storeContext); StoreFileInfo storeFileInfo = - getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file); + getRegionFileSystem().getStoreFileInfo(getColumnFamilyName(), file, sft); HStoreFile storeFile = storeEngine.createStoreFileAndReader(storeFileInfo); storeFiles.add(storeFile); HStore.this.storeSize.addAndGet(storeFile.getReader().length()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java index 5df02bfb26a8..484393f94047 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.ReaderContext; import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; @@ -215,8 +216,8 @@ public long getMaxMemStoreTS() { * @param primaryReplica true if this is a store file for primary replica, otherwise false. */ public HStoreFile(FileSystem fs, Path p, Configuration conf, CacheConfig cacheConf, - BloomType cfBloomType, boolean primaryReplica) throws IOException { - this(new StoreFileInfo(conf, fs, p, primaryReplica), cfBloomType, cacheConf); + BloomType cfBloomType, boolean primaryReplica, StoreFileTracker sft) throws IOException { + this(new StoreFileInfo(conf, fs, p, primaryReplica, sft), cfBloomType, cacheConf); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java index 34f882516bae..6a601d467f19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java @@ -215,7 +215,7 @@ public StoreFileWriter createWriter(CreateStoreFileWriterParams params) throws I public HStoreFile createStoreFileAndReader(Path p) throws IOException { StoreFileInfo info = new StoreFileInfo(conf, ctx.getRegionFileSystem().getFileSystem(), p, - ctx.isPrimaryReplicaStore()); + ctx.isPrimaryReplicaStore(), StoreFileTrackerFactory.create(conf, true, ctx)); return createStoreFileAndReader(info); } @@ -348,8 +348,8 @@ public void refreshStoreFiles() throws IOException { public void refreshStoreFiles(Collection newFiles) throws IOException { List storeFiles = new ArrayList<>(newFiles.size()); for (String file : newFiles) { - storeFiles - .add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), file)); + storeFiles.add(ctx.getRegionFileSystem().getStoreFileInfo(ctx.getFamily().getNameAsString(), + file, storeFileTracker)); } refreshStoreFilesInternal(storeFiles); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java index 052dd5112319..a49d78b8907b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder; import org.apache.hadoop.hbase.mob.MobUtils; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; @@ -119,12 +120,13 @@ public class StoreFileInfo implements Configurable { * @param primaryReplica true if this is a store file for primary replica, otherwise false. */ public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath, - final boolean primaryReplica) throws IOException { - this(conf, fs, null, initialPath, primaryReplica); + final boolean primaryReplica, final StoreFileTracker sft) throws IOException { + this(conf, fs, null, initialPath, primaryReplica, sft); } private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, - final Path initialPath, final boolean primaryReplica) throws IOException { + final Path initialPath, final boolean primaryReplica, final StoreFileTracker sft) + throws IOException { assert fs != null; assert initialPath != null; assert conf != null; @@ -142,7 +144,7 @@ private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileS this.link = HFileLink.buildFromHFileLinkPattern(conf, p); LOG.trace("{} is a link", p); } else if (isReference(p)) { - this.reference = Reference.read(fs, p); + this.reference = sft.readReference(p); Path referencePath = getReferredToFile(p); if (HFileLink.isHFileLink(referencePath)) { // HFileLink Reference @@ -175,9 +177,19 @@ private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileS * @param fs The current file system to use. * @param fileStatus The {@link FileStatus} of the file */ - public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus) - throws IOException { - this(conf, fs, fileStatus, fileStatus.getPath(), true); + public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus, + final StoreFileTracker sft) throws IOException { + this(conf, fs, fileStatus, fileStatus.getPath(), true, sft); + } + + public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath, + final StoreFileTracker sft) throws IOException { + this(conf, null, null, initialPath, true, sft); + } + + public StoreFileInfo(final Configuration conf, final StoreFileTracker sft, final FileSystem fs, + final Path initialPath, final boolean primaryReplica) throws IOException { + this(conf, null, null, initialPath, primaryReplica, sft); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java index 128537f10afe..c6043e399887 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java @@ -18,13 +18,21 @@ package org.apache.hadoop.hbase.regionserver.storefiletracker; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The default implementation for store file tracker, where we do not persist the store file list, @@ -37,6 +45,8 @@ public DefaultStoreFileTracker(Configuration conf, boolean isPrimaryReplica, Sto super(conf, isPrimaryReplica, ctx); } + private static final Logger LOG = LoggerFactory.getLogger(DefaultStoreFileTracker.class); + @Override public boolean requireWritingToTmpDirFirst() { return true; @@ -55,12 +65,49 @@ protected void doAddCompactionResults(Collection compactedFiles, @Override protected List doLoadStoreFiles(boolean readOnly) throws IOException { - List files = - ctx.getRegionFileSystem().getStoreFiles(ctx.getFamily().getNameAsString()); + List files = getStoreFiles(ctx.getFamily().getNameAsString(), true); return files != null ? files : Collections.emptyList(); } @Override protected void doSetStoreFiles(Collection files) throws IOException { } + + /** + * Returns the store files available for the family. This methods performs the filtering based on + * the valid store files. + * @param familyName Column Family Name + * @return a set of {@link StoreFileInfo} for the specified family. + */ + public List getStoreFiles(final String familyName, final boolean validate) + throws IOException { + Path familyDir = ctx.getRegionFileSystem().getStoreDir(familyName); + FileStatus[] files = + CommonFSUtils.listStatus(ctx.getRegionFileSystem().getFileSystem(), familyDir); + if (files == null) { + if (LOG.isTraceEnabled()) { + LOG.trace("No StoreFiles for: " + familyDir); + } + return null; + } + + ArrayList storeFiles = new ArrayList<>(files.length); + for (FileStatus status : files) { + if (validate && !StoreFileInfo.isValid(status)) { + // recovered.hfiles directory is expected inside CF path when + // hbase.wal.split.to.hfile to + // true, refer HBASE-23740 + if (!HConstants.RECOVERED_HFILES_DIR.equals(status.getPath().getName())) { + LOG.warn("Invalid StoreFile: {}", status.getPath()); + } + continue; + } + StoreFileInfo info = ServerRegionReplicaUtil.getStoreFileInfo(conf, + ctx.getRegionFileSystem().getFileSystem(), ctx.getRegionInfo(), + ctx.getRegionFileSystem().getRegionInfoForFS(), familyName, status.getPath(), this); + storeFiles.add(info); + + } + return storeFiles; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java index d3dfe21521d7..d4187c4af496 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/FileBasedStoreFileTracker.java @@ -77,7 +77,7 @@ protected List doLoadStoreFiles(boolean readOnly) throws IOExcept for (StoreFileEntry entry : list.getStoreFileList()) { infos.add(ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, ctx.getRegionInfo(), ctx.getRegionFileSystem().getRegionInfoForFS(), ctx.getFamily().getNameAsString(), - new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()))); + new Path(ctx.getFamilyStoreDirectoryPath(), entry.getName()), this)); } // In general, for primary replica, the load method should only be called once when // initialization, so we do not need synchronized here. And for secondary replicas, though the diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java index b0024b73786a..c7876f0ff1c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java @@ -20,7 +20,10 @@ import java.io.IOException; import java.util.Collection; import java.util.List; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.Reference; +import org.apache.hadoop.hbase.regionserver.CreateHFileLinkParams; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -94,4 +97,12 @@ void replace(Collection compactedFiles, Collection * does not allow broken store files under the actual data directory. */ boolean requireWritingToTmpDirFirst(); + + Reference createReference(Reference reference, Path path) throws IOException; + + String createHFileLink(CreateHFileLinkParams params) throws IOException; + + boolean hasReferences(final String familyName) throws IOException; + + Reference readReference(Path path) throws IOException; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java index bdf3b92db65d..81f752c6e8ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java @@ -19,29 +19,42 @@ import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL; +import java.io.BufferedInputStream; +import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.Collection; import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.HFileLink; +import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.CreateHFileLinkParams; import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams; import org.apache.hadoop.hbase.regionserver.StoreContext; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.regionserver.StoreUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; + /** * Base class for all store file tracker. *

@@ -189,6 +202,107 @@ public final StoreFileWriter createWriter(CreateStoreFileWriterParams params) th return builder.build(); } + @Override + public String createHFileLink(CreateHFileLinkParams params) throws IOException { + String name = HFileLink.createHFileLinkName(params.getLinkedTable(), params.getLinkedRegion(), + params.getHfileName()); + String refName = + HFileLink.createBackReferenceName(params.getDstTableName(), params.getDstRegionName()); + + // Make sure the destination directory exists + params.getFs().mkdirs(params.getDstFamilyPath()); + + // Make sure the FileLink reference directory exists + Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, params.getLinkedTable(), + params.getLinkedRegion(), params.getFamilyName()); + Path backRefPath = null; + if (params.isCreateBackRef()) { + Path backRefssDir = HFileLink.getBackReferencesDir(archiveStoreDir, params.getHfileName()); + params.getFs().mkdirs(backRefssDir); + + // Create the reference for the link + backRefPath = new Path(backRefssDir, refName); + params.getFs().createNewFile(backRefPath); + } + try { + // Create the link + if (params.getFs().createNewFile(new Path(params.getDstFamilyPath(), name))) { + return name; + } + } catch (IOException e) { + LOG.error("couldn't create the link=" + name + " for " + params.getDstFamilyPath(), e); + // Revert the reference if the link creation failed + if (params.isCreateBackRef()) { + params.getFs().delete(backRefPath, false); + } + throw e; + } + throw new IOException( + "File link=" + name + " already exists under " + params.getDstFamilyPath() + " folder."); + } + + @Override + public Reference createReference(Reference reference, Path path) throws IOException { + FSDataOutputStream out = ctx.getRegionFileSystem().getFileSystem().create(path, false); + try { + out.write(reference.toByteArray()); + } finally { + out.close(); + } + return reference; + } + + /** + * Returns true if the specified family has reference files + * @param familyName Column Family Name + * @return true if family contains reference files + */ + public boolean hasReferences(final String familyName) throws IOException { + Path storeDir = ctx.getRegionFileSystem().getStoreDir(familyName); + FileStatus[] files = + CommonFSUtils.listStatus(ctx.getRegionFileSystem().getFileSystem(), storeDir); + if (files != null) { + for (FileStatus stat : files) { + if (stat.isDirectory()) { + continue; + } + if (StoreFileInfo.isReference(stat.getPath())) { + LOG.trace("Reference {}", stat.getPath()); + return true; + } + } + } + return false; + } + + @Override + public Reference readReference(final Path p) throws IOException { + InputStream in = ctx.getRegionFileSystem().getFileSystem().open(p); + try { + // I need to be able to move back in the stream if this is not a pb serialization so I can + // do the Writable decoding instead. + in = in.markSupported() ? in : new BufferedInputStream(in); + int pblen = ProtobufUtil.lengthOfPBMagic(); + in.mark(pblen); + byte[] pbuf = new byte[pblen]; + IOUtils.readFully(in, pbuf, 0, pblen); + // WATCHOUT! Return in middle of function!!! + if (ProtobufUtil.isPBMagicPrefix(pbuf)) return Reference.convert( + org.apache.hadoop.hbase.shaded.protobuf.generated.FSProtos.Reference.parseFrom(in)); + // Else presume Writables. Need to reset the stream since it didn't start w/ pb. + // We won't bother rewriting thie Reference as a pb since Reference is transitory. + in.reset(); + Reference r = new Reference(); + DataInputStream dis = new DataInputStream(in); + // Set in = dis so it gets the close below in the finally on our way out. + in = dis; + r.readFields(dis); + return r; + } finally { + in.close(); + } + } + /** * For primary replica, we will call load once when opening a region, and the implementation could * choose to do some cleanup work. So here we use {@code readOnly} to indicate that whether you diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java index 0f487afd1cba..828f1974fca7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java @@ -129,10 +129,16 @@ public static StoreFileTracker create(Configuration conf, boolean isPrimaryRepli */ public static StoreFileTracker create(Configuration conf, TableDescriptor td, ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs) { + return create(conf, td, cfd, regionFs, true); + } + + public static StoreFileTracker create(Configuration conf, TableDescriptor td, + ColumnFamilyDescriptor cfd, HRegionFileSystem regionFs, boolean isPrimaryReplica) { StoreContext ctx = StoreContext.getBuilder().withColumnFamilyDescriptor(cfd).withRegionFileSystem(regionFs) .withFamilyStoreDirectoryPath(regionFs.getStoreDir(cfd.getNameAsString())).build(); - return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), true, ctx); + return StoreFileTrackerFactory.create(mergeConfigurations(conf, td, cfd), isPrimaryReplica, + ctx); } private static Configuration mergeConfigurations(Configuration global, TableDescriptor table, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 8395456cd76e..39300538ef1a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.hbase.regionserver.CreateHFileLinkParams; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreContext; @@ -499,7 +500,10 @@ private void restoreRegion(final RegionInfo regionInfo, for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) { byte[] family = Bytes.toBytes(familyDir.getName()); - Set familyFiles = getTableRegionFamilyFiles(familyDir); + StoreFileTracker tracker = StoreFileTrackerFactory.create(conf, true, + StoreContext.getBuilder().withColumnFamilyDescriptor(tableDesc.getColumnFamily(family)) + .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); + Set familyFiles = getTableRegionFamilyFiles(tracker); List snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); List filesToTrack = new ArrayList<>(); @@ -510,8 +514,8 @@ private void restoreRegion(final RegionInfo regionInfo, // HFile already present familyFiles.remove(storeFile.getName()); // no need to restore already present files, but we need to add those to tracker - filesToTrack - .add(new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true)); + filesToTrack.add( + new StoreFileInfo(conf, fs, new Path(familyDir, storeFile.getName()), true, tracker)); } else { // HFile missing hfilesToAdd.add(storeFile); @@ -532,9 +536,11 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile : hfilesToAdd) { LOG.debug("Restoring missing HFileLink " + storeFile.getName() + " of snapshot=" + snapshotName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); - String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); + String fileName = + restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs, tracker); // mark the reference file to be added to tracker - filesToTrack.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); + filesToTrack + .add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true, tracker)); } } else { // Family doesn't exists in the snapshot @@ -544,10 +550,6 @@ private void restoreRegion(final RegionInfo regionInfo, fs.delete(familyDir, true); } - StoreFileTracker tracker = - StoreFileTrackerFactory.create(conf, true, StoreContext.getBuilder() - .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); - // simply reset list of tracked files with the matching files // and the extra one present in the snapshot tracker.set(filesToTrack); @@ -568,23 +570,26 @@ private void restoreRegion(final RegionInfo regionInfo, for (SnapshotRegionManifest.StoreFile storeFile : familyEntry.getValue()) { LOG.trace("Adding HFileLink (Not present in the table) " + storeFile.getName() + " of snapshot " + snapshotName + " to table=" + tableName); - String fileName = restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); - files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true)); + String fileName = + restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs, tracker); + files.add(new StoreFileInfo(conf, fs, new Path(familyDir, fileName), true, tracker)); } tracker.set(files); } } /** Returns The set of files in the specified family directory. */ - private Set getTableRegionFamilyFiles(final Path familyDir) throws IOException { - FileStatus[] hfiles = CommonFSUtils.listStatus(fs, familyDir); + private Set getTableRegionFamilyFiles(final StoreFileTracker storeFileTracker) + throws IOException { + List hfiles; + hfiles = storeFileTracker.load(); if (hfiles == null) { return Collections.emptySet(); } - Set familyFiles = new HashSet<>(hfiles.length); - for (int i = 0; i < hfiles.length; ++i) { - String hfileName = hfiles[i].getPath().getName(); + Set familyFiles = new HashSet<>(hfiles.size()); + for (int i = 0; i < hfiles.size(); ++i) { + String hfileName = hfiles.get(i).getPath().getName(); familyFiles.add(hfileName); } @@ -658,6 +663,16 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); List clonedFiles = new ArrayList<>(); + Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); + HRegionFileSystem regionFS = (fs.exists(regionPath)) + ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) + : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); + + Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, + tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray())); + StoreFileTracker tracker = + StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder() + .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " from cloned region " + "in snapshot " + snapshotName + " to table=" + tableName); @@ -668,24 +683,15 @@ private void cloneRegion(final RegionInfo newRegionInfo, final Path regionDir, if (fs.exists(mobPath)) { fs.delete(mobPath, true); } - restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); + restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); } else { - String file = restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); - clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true)); + String file = + restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs, tracker); + clonedFiles.add(new StoreFileInfo(conf, fs, new Path(familyDir, file), true, tracker)); } } // we don't need to track files under mobdir if (!MobUtils.isMobRegionInfo(newRegionInfo)) { - Path regionPath = new Path(tableDir, newRegionInfo.getEncodedName()); - HRegionFileSystem regionFS = (fs.exists(regionPath)) - ? HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, newRegionInfo, false) - : HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, newRegionInfo); - - Configuration sftConf = StoreUtils.createStoreConfiguration(conf, tableDesc, - tableDesc.getColumnFamily(familyFiles.getFamilyName().toByteArray())); - StoreFileTracker tracker = - StoreFileTrackerFactory.create(sftConf, true, StoreContext.getBuilder() - .withFamilyStoreDirectoryPath(familyDir).withRegionFileSystem(regionFS).build()); tracker.set(clonedFiles); } } @@ -719,15 +725,17 @@ private void cloneRegion(final HRegion region, final RegionInfo snapshotRegionIn * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ private String restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, - final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) - throws IOException { + final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef, + final StoreFileTracker tracker) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { - return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); + return tracker.createHFileLink( + CreateHFileLinkParams.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef)); } else if (StoreFileInfo.isReference(hfileName)) { - return restoreReferenceFile(familyDir, regionInfo, storeFile); + return restoreReferenceFile(familyDir, regionInfo, storeFile, tracker); } else { - return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); + return tracker.createHFileLink( + CreateHFileLinkParams.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef)); } } @@ -755,7 +763,8 @@ private String restoreStoreFile(final Path familyDir, final RegionInfo regionInf * @param storeFile reference file name */ private String restoreReferenceFile(final Path familyDir, final RegionInfo regionInfo, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + final SnapshotRegionManifest.StoreFile storeFile, final StoreFileTracker tracker) + throws IOException { String hfileName = storeFile.getName(); // Extract the referred information (hfile name and parent region) @@ -789,7 +798,7 @@ private String restoreReferenceFile(final Path familyDir, final RegionInfo regio // Create the new reference if (storeFile.hasReference()) { Reference reference = Reference.convert(storeFile.getReference()); - reference.write(fs, outPath); + tracker.createReference(reference, outPath); } else { InputStream in; if (linkPath != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index e578b9f9764e..7390891b4402 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -32,9 +32,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; @@ -207,7 +207,7 @@ protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws monitor.rethrowException(); Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString()); - List storeFiles = getStoreFiles(storePath); + List storeFiles = getStoreFiles(htd, hcd, regionInfo); if (storeFiles == null) { if (LOG.isDebugEnabled()) { LOG.debug("No mob files under family: " + hcd.getNameAsString()); @@ -341,15 +341,13 @@ protected void addRegion(Path tableDir, RegionInfo regionInfo, RegionVisitor vis } } - private List getStoreFiles(Path storeDir) throws IOException { - FileStatus[] stats = CommonFSUtils.listStatus(rootFs, storeDir); - if (stats == null) return null; - - ArrayList storeFiles = new ArrayList<>(stats.length); - for (int i = 0; i < stats.length; ++i) { - storeFiles.add(new StoreFileInfo(conf, rootFs, stats[i])); - } - return storeFiles; + private List getStoreFiles(TableDescriptor htd, ColumnFamilyDescriptor hcd, + RegionInfo regionInfo) throws IOException { + HRegionFileSystem regionFS = HRegionFileSystem.create(conf, rootFs, + MobUtils.getMobTableDir(new Path(conf.get(HConstants.HBASE_DIR)), htd.getTableName()), + regionInfo); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, hcd, regionFS, false); + return sft.load(); } private void addReferenceFiles(RegionVisitor visitor, Object regionData, Object familyData, @@ -385,7 +383,7 @@ private void load() throws IOException { ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader"); try { this.regionManifests = - SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc); + SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc, htd); } finally { tpool.shutdown(); } @@ -403,7 +401,7 @@ private void load() throws IOException { ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader"); try { v1Regions = - SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc); + SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc, htd); v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, rootFs, workingDir, desc, manifestSizeLimit); } catch (InvalidProtocolBufferException e) { @@ -502,7 +500,7 @@ private void convertToV2SingleManifest() throws IOException { setStatusMsg("Loading Region manifests for " + this.desc.getName()); try { v1Regions = - SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc); + SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc, htd); v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc, manifestSizeLimit); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java index 61c366de971a..d29f0d9845f5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java @@ -31,8 +31,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; @@ -119,7 +122,7 @@ public void storeFile(final HRegionFileSystem region, final Path familyDir, static List loadRegionManifests(final Configuration conf, final Executor executor, final FileSystem fs, final Path snapshotDir, - final SnapshotDescription desc) throws IOException { + final SnapshotDescription desc, final TableDescriptor htd) throws IOException { FileStatus[] regions = CommonFSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs)); if (regions == null) { @@ -134,7 +137,7 @@ static List loadRegionManifests(final Configuration conf @Override public SnapshotRegionManifest call() throws IOException { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, region.getPath()); - return buildManifestFromDisk(conf, fs, snapshotDir, hri); + return buildManifestFromDisk(conf, fs, snapshotDir, hri, htd); } }); } @@ -159,7 +162,8 @@ static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir, } static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, - final Path tableDir, final RegionInfo regionInfo) throws IOException { + final Path tableDir, final RegionInfo regionInfo, final TableDescriptor htd) + throws IOException { HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true); SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder(); @@ -179,7 +183,9 @@ static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, fi Collection familyNames = regionFs.getFamilies(); if (familyNames != null) { for (String familyName : familyNames) { - Collection storeFiles = regionFs.getStoreFiles(familyName, false); + StoreFileTracker sft = StoreFileTrackerFactory.create(conf, htd, + htd.getColumnFamily(familyName.getBytes()), regionFs, false); + List storeFiles = sft.load(); if (storeFiles == null) { LOG.debug("No files under family: " + familyName); continue; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java index c7e9166b54a8..57024102555e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java @@ -50,6 +50,8 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; +import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -680,7 +682,9 @@ static LinkedList> splitScan(LinkedList getStoresRequiringCompaction(Set requestedStores, long times boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts) throws IOException { // do we have any store files? - Collection storeFiles = fileSystem.getStoreFiles(family); + TableDescriptor htd = + connection.getTable(CommonFSUtils.getTableName(fileSystem.getTableDir())).getDescriptor(); + StoreFileTracker sft = StoreFileTrackerFactory.create(connection.getConfiguration(), htd, + htd.getColumnFamily(family.getBytes()), fileSystem, false); + List storeFiles = sft.load(); if (storeFiles == null) { LOG.info("Excluding store: " + family + " for compaction for region: " + fileSystem.getRegionInfo().getEncodedName(), " has no store files"); return false; } // check for reference files - if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) { + if (sft.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) { LOG.info("Including store: " + family + " with: " + storeFiles.size() + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName()); return true; diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp index b538cb7b6b4f..7214dd360640 100644 --- a/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/storeFile.jsp @@ -54,7 +54,7 @@ printer.setConf(conf); String[] options = {"-s"}; printer.parseOptions(options); - StoreFileInfo sfi = new StoreFileInfo(conf, fs, new Path(storeFile), true); + StoreFileInfo sfi = new StoreFileInfo(conf, fs, new Path(storeFile), true, null); printer.processFile(sfi.getFileStatus().getPath(), true); String text = byteStream.toString();%> <%=