HBASE-27775 Use a separate WAL provider for hbase:replication table

Author: Apache9

hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java

@@ -102,4 +102,10 @@ public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn
       return ReflectionUtils.newInstance(clazz, conf, tableName);
     }
   }
+
+  public static boolean isReplicationQueueTable(Configuration conf, TableName tableName) {
+    TableName replicationQueueTableName = TableName.valueOf(conf.get(REPLICATION_QUEUE_TABLE_NAME,
+      REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString()));
+    return replicationQueueTableName.equals(tableName);
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -982,12 +982,12 @@ synchronized public void run() {
 
         lastRan = currentTime;
 
-        final WALProvider provider = regionServer.getWalFactory().getWALProvider();
-        final WALProvider metaProvider = regionServer.getWalFactory().getMetaWALProvider();
-        numWALFiles = (provider == null ? 0 : provider.getNumLogFiles())
-          + (metaProvider == null ? 0 : metaProvider.getNumLogFiles());
-        walFileSize = (provider == null ? 0 : provider.getLogFileSize())
-          + (metaProvider == null ? 0 : metaProvider.getLogFileSize());
+        List<WALProvider> providers = regionServer.getWalFactory().getAllWALProviders();
+        for (WALProvider provider : providers) {
+          numWALFiles += provider.getNumLogFiles();
+          walFileSize += provider.getLogFileSize();
+        }
+
         // Copy over computed values so that no thread sees half computed values.
         numStores = tempNumStores;
         numStoreFiles = tempNumStoreFiles;

hbase-server/src/main/java/org/apache/hadoop/hbase/wal/LazyInitializedWALProvider.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.wal;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
+import org.apache.hadoop.hbase.wal.WALFactory.Providers;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * A lazy initialized WAL provider for holding the WALProvider for some special tables, such as
+ * hbase:meta, hbase:replication, etc.
+ */
+@InterfaceAudience.Private
+class LazyInitializedWALProvider implements Closeable {
+
+  private final WALFactory factory;
+
+  private final String providerId;
+
+  private final String providerConfigName;
+
+  private final Abortable abortable;
+
+  private final AtomicReference<WALProvider> holder = new AtomicReference<>();
+
+  LazyInitializedWALProvider(WALFactory factory, String providerId, String providerConfigName,
+    Abortable abortable) {
+    this.factory = factory;
+    this.providerId = providerId;
+    this.providerConfigName = providerConfigName;
+    this.abortable = abortable;
+  }
+
+  WALProvider getProvider() throws IOException {
+    Configuration conf = factory.getConf();
+    for (;;) {
+      WALProvider provider = this.holder.get();
+      if (provider != null) {
+        return provider;
+      }
+      Class<? extends WALProvider> clz = null;
+      if (conf.get(providerConfigName) == null) {
+        try {
+          clz = conf.getClass(WALFactory.WAL_PROVIDER, Providers.defaultProvider.clazz,
+            WALProvider.class);
+        } catch (Throwable t) {
+          // the WAL provider should be an enum. Proceed
+        }
+      }
+      if (clz == null) {
+        clz = factory.getProviderClass(providerConfigName,
+          conf.get(WALFactory.WAL_PROVIDER, WALFactory.DEFAULT_WAL_PROVIDER));
+      }
+      provider = WALFactory.createProvider(clz);
+      provider.init(factory, conf, providerId, this.abortable);
+      provider.addWALActionsListener(new MetricsWAL());
+      if (this.holder.compareAndSet(null, provider)) {
+        return provider;
+      } else {
+        // someone is ahead of us, close and try again.
+        provider.close();
+      }
+    }
+  }
+
+  /**
+   * Get the provider if it already initialized, otherwise just return {@code null} instead of
+   * creating it.
+   */
+  WALProvider getProviderNoCreate() {
+    return holder.get();
+  }
+
+  @Override
+  public void close() throws IOException {
+    WALProvider provider = this.holder.get();
+    if (provider != null) {
+      provider.close();
+    }
+  }
+
+  void shutdown() throws IOException {
+    WALProvider provider = this.holder.get();
+    if (provider != null) {
+      provider.shutdown();
+    }
+  }
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java

@@ -20,6 +20,7 @@
 import com.google.errorprone.annotations.RestrictedApi;
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.hadoop.conf.Configuration;
@@ -28,10 +29,12 @@
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
 import org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager;
 import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
 import org.apache.hadoop.hbase.regionserver.wal.ProtobufWALStreamReader;
 import org.apache.hadoop.hbase.regionserver.wal.ProtobufWALTailingReader;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
 import org.apache.hadoop.hbase.util.CancelableProgressable;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.LeaseNotRecoveredException;
@@ -99,15 +102,22 @@ enum Providers {
 
   public static final String META_WAL_PROVIDER = "hbase.wal.meta_provider";
 
+  public static final String REPLICATION_WAL_PROVIDER = "hbase.wal.replication_provider";
+
   public static final String WAL_ENABLED = "hbase.regionserver.hlog.enabled";
 
+  static final String REPLICATION_WAL_PROVIDER_ID = "rep";
+
   final String factoryId;
   final Abortable abortable;
   private final WALProvider provider;
   // The meta updates are written to a different wal. If this
   // regionserver holds meta regions, then this ref will be non-null.
   // lazily intialized; most RegionServers don't deal with META
-  private final AtomicReference<WALProvider> metaProvider = new AtomicReference<>();
+  private final LazyInitializedWALProvider metaProvider;
+  // This is for avoid hbase:replication itself keeps trigger unnecessary updates to WAL file and
+  // generate a lot useless data, see HBASE-27775 for more details.
+  private final LazyInitializedWALProvider replicationProvider;
 
   /**
    * Configuration-specified WAL Reader used when a custom reader is requested
@@ -144,13 +154,15 @@ private WALFactory(Configuration conf) {
     factoryId = SINGLETON_ID;
     this.abortable = null;
     this.excludeDatanodeManager = new ExcludeDatanodeManager(conf);
+    this.metaProvider = null;
+    this.replicationProvider = null;
   }
 
   Providers getDefaultProvider() {
     return Providers.defaultProvider;
   }
 
-  public Class<? extends WALProvider> getProviderClass(String key, String defaultValue) {
+  Class<? extends WALProvider> getProviderClass(String key, String defaultValue) {
     try {
       Providers provider = Providers.valueOf(conf.get(key, defaultValue));
 
@@ -246,6 +258,10 @@ private WALFactory(Configuration conf, String factoryId, Abortable abortable,
     this.factoryId = factoryId;
     this.excludeDatanodeManager = new ExcludeDatanodeManager(conf);
     this.abortable = abortable;
+    this.metaProvider = new LazyInitializedWALProvider(this,
+      AbstractFSWALProvider.META_WAL_PROVIDER_ID, META_WAL_PROVIDER, this.abortable);
+    this.replicationProvider = new LazyInitializedWALProvider(this, REPLICATION_WAL_PROVIDER_ID,
+      REPLICATION_WAL_PROVIDER, this.abortable);
     // end required early initialization
     if (conf.getBoolean(WAL_ENABLED, true)) {
       WALProvider provider = createProvider(getProviderClass(WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
@@ -263,19 +279,45 @@ private WALFactory(Configuration conf, String factoryId, Abortable abortable,
     }
   }
 
+  public Configuration getConf() {
+    return conf;
+  }
+
   /**
    * Shutdown all WALs and clean up any underlying storage. Use only when you will not need to
    * replay and edits that have gone to any wals from this factory.
    */
   public void close() throws IOException {
-    final WALProvider metaProvider = this.metaProvider.get();
-    if (null != metaProvider) {
-      metaProvider.close();
+    List<IOException> ioes = new ArrayList<>();
+    // these fields could be null if the WALFactory is created only for being used in the
+    // getInstance method.
+    if (metaProvider != null) {
+      try {
+        metaProvider.close();
+      } catch (IOException e) {
+        ioes.add(e);
+      }
+    }
+    if (replicationProvider != null) {
+      try {
+        replicationProvider.close();
+      } catch (IOException e) {
+        ioes.add(e);
+      }
     }
-    // close is called on a WALFactory with null provider in the case of contention handling
-    // within the getInstance method.
-    if (null != provider) {
-      provider.close();
+    if (provider != null) {
+      try {
+        provider.close();
+      } catch (IOException e) {
+        ioes.add(e);
+      }
+    }
+    if (!ioes.isEmpty()) {
+      IOException ioe = new IOException("Failed to close WALFactory");
+      for (IOException e : ioes) {
+        ioe.addSuppressed(e);
+      }
+      throw ioe;
     }
   }
 
@@ -285,18 +327,36 @@ public void close() throws IOException {
    * if you can as it will try to leave things as tidy as possible.
    */
   public void shutdown() throws IOException {
-    IOException exception = null;
-    final WALProvider metaProvider = this.metaProvider.get();
-    if (null != metaProvider) {
+    List<IOException> ioes = new ArrayList<>();
+    // these fields could be null if the WALFactory is created only for being used in the
+    // getInstance method.
+    if (metaProvider != null) {
       try {
         metaProvider.shutdown();
-      } catch (IOException ioe) {
-        exception = ioe;
+      } catch (IOException e) {
+        ioes.add(e);
+      }
+    }
+    if (replicationProvider != null) {
+      try {
+        replicationProvider.shutdown();
+      } catch (IOException e) {
+        ioes.add(e);
+      }
+    }
+    if (provider != null) {
+      try {
+        provider.shutdown();
+      } catch (IOException e) {
+        ioes.add(e);
       }
     }
-    provider.shutdown();
-    if (null != exception) {
-      throw exception;
+    if (!ioes.isEmpty()) {
+      IOException ioe = new IOException("Failed to shutdown WALFactory");
+      for (IOException e : ioes) {
+        ioe.addSuppressed(e);
+      }
+      throw ioe;
     }
   }
 
@@ -309,48 +369,23 @@ public List<WAL> getWALs() {
    * creating the first hbase:meta WAL so we can register a listener.
    * @see #getMetaWALProvider()
    */
-  public WALProvider getMetaProvider() throws IOException {
-    for (;;) {
-      WALProvider provider = this.metaProvider.get();
-      if (provider != null) {
-        return provider;
-      }
-      Class<? extends WALProvider> clz = null;
-      if (conf.get(META_WAL_PROVIDER) == null) {
-        try {
-          clz = conf.getClass(WAL_PROVIDER, Providers.defaultProvider.clazz, WALProvider.class);
-        } catch (Throwable t) {
-          // the WAL provider should be an enum. Proceed
-        }
-      }
-      if (clz == null) {
-        clz = getProviderClass(META_WAL_PROVIDER, conf.get(WAL_PROVIDER, DEFAULT_WAL_PROVIDER));
-      }
-      provider = createProvider(clz);
-      provider.init(this, conf, AbstractFSWALProvider.META_WAL_PROVIDER_ID, this.abortable);
-      provider.addWALActionsListener(new MetricsWAL());
-      if (metaProvider.compareAndSet(null, provider)) {
-        return provider;
-      } else {
-        // someone is ahead of us, close and try again.
-        provider.close();
-      }
-    }
+  WALProvider getMetaProvider() throws IOException {
+    return metaProvider.getProvider();
   }
 
   /**
    * @param region the region which we want to get a WAL for. Could be null.
    */
   public WAL getWAL(RegionInfo region) throws IOException {
     // Use different WAL for hbase:meta. Instantiates the meta WALProvider if not already up.
-    if (
-      region != null && region.isMetaRegion()
-        && region.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID
-    ) {
-      return getMetaProvider().getWAL(region);
-    } else {
-      return provider.getWAL(region);
+    if (region != null && RegionReplicaUtil.isDefaultReplica(region)) {
+      if (region.isMetaRegion()) {
+        return metaProvider.getProvider().getWAL(region);
+      } else if (ReplicationStorageFactory.isReplicationQueueTable(conf, region.getTable())) {
+        return replicationProvider.getProvider().getWAL(region);
+      }
     }
+    return provider.getWAL(region);
   }
 
   public WALStreamReader createStreamReader(FileSystem fs, Path path) throws IOException {
@@ -532,11 +567,23 @@ public final WALProvider getWALProvider() {
   }
 
   /**
-   * @return Current metaProvider... may be null if not yet initialized.
-   * @see #getMetaProvider()
+   * Returns all the wal providers, for example, the default one, the one for hbase:meta and the one
+   * for hbase:replication.
    */
-  public final WALProvider getMetaWALProvider() {
-    return this.metaProvider.get();
+  public final List<WALProvider> getAllWALProviders() {
+    List<WALProvider> providers = new ArrayList<>();
+    if (provider != null) {
+      providers.add(provider);
+    }
+    WALProvider meta = metaProvider.getProviderNoCreate();
+    if (meta != null) {
+      providers.add(meta);
+    }
+    WALProvider replication = replicationProvider.getProviderNoCreate();
+    if (replication != null) {
+      providers.add(replication);
+    }
+    return providers;
   }
 
   public ExcludeDatanodeManager getExcludeDatanodeManager() {