HBASE-28969 Move HFileLink file creations to SFT (#6459)

Signed-off-by: Andrew Purtell <[email protected]>
Co-authored-by: Prathyusha Garre <[email protected]>

Author: gvprathyusha6

hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java

@@ -21,7 +21,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
@@ -290,175 +289,6 @@ public static String createHFileLinkName(final TableName tableName, final String
     return s;
   }
 
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf            {@link Configuration} to read for the archive directory name
-   * @param fs              {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath   - Destination path (table/region/cf/)
-   * @param hfileRegionInfo - Linked HFile Region Info
-   * @param hfileName       - Linked HFile name
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName)
-    throws IOException {
-    return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
-  }
-
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf            {@link Configuration} to read for the archive directory name
-   * @param fs              {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath   - Destination path (table/region/cf/)
-   * @param hfileRegionInfo - Linked HFile Region Info
-   * @param hfileName       - Linked HFile name
-   * @param createBackRef   - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName,
-    final boolean createBackRef) throws IOException {
-    TableName linkedTable = hfileRegionInfo.getTable();
-    String linkedRegion = hfileRegionInfo.getEncodedName();
-    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
-  }
-
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param linkedTable   - Linked Table Name
-   * @param linkedRegion  - Linked Region Name
-   * @param hfileName     - Linked HFile name
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
-    final String hfileName) throws IOException {
-    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
-  }
-
-  /**
-   * Create a new HFileLink. In the event of link creation failure, this method throws an
-   * IOException, so that the calling upper laying can decide on how to proceed with this.
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param linkedTable   - Linked Table Name
-   * @param linkedRegion  - Linked Region Name
-   * @param hfileName     - Linked HFile name
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
-    final String hfileName, final boolean createBackRef) throws IOException {
-    String familyName = dstFamilyPath.getName();
-    String regionName = dstFamilyPath.getParent().getName();
-    String tableName =
-      CommonFSUtils.getTableName(dstFamilyPath.getParent().getParent()).getNameAsString();
-
-    return create(conf, fs, dstFamilyPath, familyName, tableName, regionName, linkedTable,
-      linkedRegion, hfileName, createBackRef);
-  }
-
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param dstTableName  - Destination table name
-   * @param dstRegionName - Destination region name
-   * @param linkedTable   - Linked Table Name
-   * @param linkedRegion  - Linked Region Name
-   * @param hfileName     - Linked HFile name
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final String familyName, final String dstTableName,
-    final String dstRegionName, final TableName linkedTable, final String linkedRegion,
-    final String hfileName, final boolean createBackRef) throws IOException {
-    String name = createHFileLinkName(linkedTable, linkedRegion, hfileName);
-    String refName = createBackReferenceName(dstTableName, dstRegionName);
-
-    // Make sure the destination directory exists
-    fs.mkdirs(dstFamilyPath);
-
-    // Make sure the FileLink reference directory exists
-    Path archiveStoreDir =
-      HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, familyName);
-    Path backRefPath = null;
-    if (createBackRef) {
-      Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
-      fs.mkdirs(backRefssDir);
-
-      // Create the reference for the link
-      backRefPath = new Path(backRefssDir, refName);
-      fs.createNewFile(backRefPath);
-    }
-    try {
-      // Create the link
-      if (fs.createNewFile(new Path(dstFamilyPath, name))) {
-        return name;
-      }
-    } catch (IOException e) {
-      LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
-      // Revert the reference if the link creation failed
-      if (createBackRef) {
-        fs.delete(backRefPath, false);
-      }
-      throw e;
-    }
-    throw new IOException(
-      "File link=" + name + " already exists under " + dstFamilyPath + " folder.");
-  }
-
-  /**
-   * Create a new HFileLink starting from a hfileLink name
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String createFromHFileLink(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
-    throws IOException {
-    Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
-    if (!m.matches()) {
-      throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
-    }
-    return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), m.group(3),
-      m.group(4), createBackRef);
-  }
-
   /**
    * Create the back reference name
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/SplitTableRegionProcedure.java

@@ -701,9 +701,8 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
     // table dir. In case of failure, the proc would go through this again, already existing
     // region dirs and split files would just be ignored, new split files should get created.
     int nbFiles = 0;
-    final Map<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>> files =
-      new HashMap<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>>(
-        htd.getColumnFamilyCount());
+    final Map<String, Collection<StoreFileInfo>> files =
+      new HashMap<String, Collection<StoreFileInfo>>(htd.getColumnFamilyCount());
     for (ColumnFamilyDescriptor cfd : htd.getColumnFamilies()) {
       String family = cfd.getNameAsString();
       StoreFileTracker tracker =
@@ -726,7 +725,7 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
         }
         if (filteredSfis == null) {
           filteredSfis = new ArrayList<StoreFileInfo>(sfis.size());
-          files.put(family, new Pair(filteredSfis, tracker));
+          files.put(family, filteredSfis);
         }
         filteredSfis.add(sfi);
         nbFiles++;
@@ -749,12 +748,11 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
     final List<Future<Pair<Path, Path>>> futures = new ArrayList<Future<Pair<Path, Path>>>(nbFiles);
 
     // Split each store file.
-    for (Map.Entry<String, Pair<Collection<StoreFileInfo>, StoreFileTracker>> e : files
-      .entrySet()) {
+    for (Map.Entry<String, Collection<StoreFileInfo>> e : files.entrySet()) {
       byte[] familyName = Bytes.toBytes(e.getKey());
       final ColumnFamilyDescriptor hcd = htd.getColumnFamily(familyName);
-      Pair<Collection<StoreFileInfo>, StoreFileTracker> storeFilesAndTracker = e.getValue();
-      final Collection<StoreFileInfo> storeFiles = storeFilesAndTracker.getFirst();
+      Collection<StoreFileInfo> storeFileInfos = e.getValue();
+      final Collection<StoreFileInfo> storeFiles = storeFileInfos;
       if (storeFiles != null && storeFiles.size() > 0) {
         final Configuration storeConfiguration =
           StoreUtils.createStoreConfiguration(env.getMasterConfiguration(), htd, hcd);
@@ -765,9 +763,8 @@ private Pair<List<Path>, List<Path>> splitStoreFiles(final MasterProcedureEnv en
           // is running in a regionserver's Store context, or we might not be able
           // to read the hfiles.
           storeFileInfo.setConf(storeConfiguration);
-          StoreFileSplitter sfs =
-            new StoreFileSplitter(regionFs, storeFilesAndTracker.getSecond(), familyName,
-              new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
+          StoreFileSplitter sfs = new StoreFileSplitter(regionFs, htd, hcd,
+            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED));
           futures.add(threadPool.submit(sfs));
         }
       }
@@ -833,19 +830,27 @@ private void assertSplitResultFilesCount(final FileSystem fs,
     }
   }
 
-  private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, StoreFileTracker tracker,
-    byte[] family, HStoreFile sf) throws IOException {
+  private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, TableDescriptor htd,
+    ColumnFamilyDescriptor hcd, HStoreFile sf) throws IOException {
     if (LOG.isDebugEnabled()) {
       LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath()
         + " for region: " + getParentRegion().getShortNameToLog());
     }
 
     final byte[] splitRow = getSplitRow();
-    final String familyName = Bytes.toString(family);
+    final String familyName = hcd.getNameAsString();
+    StoreFileTracker daughterOneSft =
+      StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), htd, hcd,
+        HRegionFileSystem.create(regionFs.getFileSystem().getConf(), regionFs.getFileSystem(),
+          regionFs.getTableDir(), daughterOneRI));
+    StoreFileTracker daughterTwoSft =
+      StoreFileTrackerFactory.create(regionFs.getFileSystem().getConf(), htd, hcd,
+        HRegionFileSystem.create(regionFs.getFileSystem().getConf(), regionFs.getFileSystem(),
+          regionFs.getTableDir(), daughterTwoRI));
     final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
-      false, splitPolicy, tracker);
+      false, splitPolicy, daughterOneSft);
     final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
-      true, splitPolicy, tracker);
+      true, splitPolicy, daughterTwoSft);
     if (LOG.isDebugEnabled()) {
       LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath()
         + " for region: " + getParentRegion().getShortNameToLog());
@@ -859,27 +864,27 @@ private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, StoreFileTra
    */
   private class StoreFileSplitter implements Callable<Pair<Path, Path>> {
     private final HRegionFileSystem regionFs;
-    private final byte[] family;
+    private final ColumnFamilyDescriptor hcd;
     private final HStoreFile sf;
-    private final StoreFileTracker tracker;
+    private final TableDescriptor htd;
 
     /**
      * Constructor that takes what it needs to split
      * @param regionFs the file system
-     * @param family   Family that contains the store file
+     * @param hcd      Family that contains the store file
      * @param sf       which file
      */
-    public StoreFileSplitter(HRegionFileSystem regionFs, StoreFileTracker tracker, byte[] family,
-      HStoreFile sf) {
+    public StoreFileSplitter(HRegionFileSystem regionFs, TableDescriptor htd,
+      ColumnFamilyDescriptor hcd, HStoreFile sf) {
       this.regionFs = regionFs;
       this.sf = sf;
-      this.family = family;
-      this.tracker = tracker;
+      this.hcd = hcd;
+      this.htd = htd;
     }
 
     @Override
     public Pair<Path, Path> call() throws IOException {
-      return splitStoreFile(regionFs, tracker, family, sf);
+      return splitStoreFile(regionFs, htd, hcd, sf);
     }
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java

@@ -683,8 +683,7 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte
           hfileName = m.group(4);
         }
         // must create back reference here
-        HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(),
-          hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
+        tracker.createHFileLink(linkedTable, linkedRegion, hfileName, true);
         Path path =
           new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
         LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName()

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
@@ -119,4 +120,30 @@ StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPa
   StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica)
     throws IOException;
 
+  /**
+   * Create a new HFileLink
+   * <p>
+   * It also adds a back-reference to the hfile back-reference directory to simplify the
+   * reference-count and the cleaning process.
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
+   */
+  String createHFileLink(final TableName linkedTable, final String linkedRegion,
+    final String hfileName, final boolean createBackRef) throws IOException;
+
+  /**
+   * Create a new HFileLink starting from a hfileLink name
+   * <p>
+   * It also adds a back-reference to the hfile back-reference directory to simplify the
+   * reference-count and the cleaning process.
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
+   */
+  String createFromHFileLink(final String hfileName, final boolean createBackRef)
+    throws IOException;
+
 }