Merge branch 'apache:trunk' into YARN-11446

Author: slfan1989

LICENSE-binary

@@ -363,7 +363,7 @@ org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.0.5
-org.yaml:snakeyaml:1.33
+org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
 
 

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -26,9 +26,9 @@ MYNAME="${BASH_SOURCE-$0}"
 function hadoop_usage
 {
   hadoop_add_option "buildpaths" "attempt to add class files from build tree"
-  hadoop_add_option "hostnames list[,of,host,names]" "hosts to use in slave mode"
+  hadoop_add_option "hostnames list[,of,host,names]" "hosts to use in worker mode"
   hadoop_add_option "loglevel level" "set the log4j level for this command"
-  hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
+  hadoop_add_option "hosts filename" "list of hosts to use in worker mode"
   hadoop_add_option "workers" "turn on worker mode"
 
   hadoop_add_subcommand "checknative" client "check native Hadoop and compression libraries availability"

hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 
-# Run a Hadoop command on all slave hosts.
+# Run a Hadoop command on all worker hosts.
 
 function hadoop_usage
 {

hadoop-common-project/hadoop-common/src/main/conf/log4j.properties

@@ -75,14 +75,6 @@ log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
 
-#
-# TaskLog Appender
-#
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
 #
 # HDFS block state change log from block manager
 #

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java

@@ -34,6 +34,8 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -44,6 +46,7 @@
 import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.GenericsUtil;
 import org.apache.hadoop.util.ServletUtil;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -338,14 +341,18 @@ public void doGet(HttpServletRequest request, HttpServletResponse response
         out.println(MARKER
             + "Submitted Class Name: <b>" + logName + "</b><br />");
 
-        Logger log = Logger.getLogger(logName);
+        org.slf4j.Logger log = LoggerFactory.getLogger(logName);
         out.println(MARKER
             + "Log Class: <b>" + log.getClass().getName() +"</b><br />");
         if (level != null) {
           out.println(MARKER + "Submitted Level: <b>" + level + "</b><br />");
         }
 
-        process(log, level, out);
+        if (GenericsUtil.isLog4jLogger(logName)) {
+          process(Logger.getLogger(logName), level, out);
+        } else {
+          out.println("Sorry, setting log level is only supported for log4j loggers.<br />");
+        }
       }
 
       out.println(FORMS);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java

@@ -20,6 +20,7 @@
 
 import java.lang.reflect.Array;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -33,6 +34,14 @@
 @InterfaceStability.Unstable
 public class GenericsUtil {
 
+  private static final String SLF4J_LOG4J_ADAPTER_CLASS = "org.slf4j.impl.Log4jLoggerAdapter";
+
+  /**
+   * Set to false only if log4j adapter class is not found in the classpath. Once set to false,
+   * the utility method should not bother re-loading class again.
+   */
+  private static final AtomicBoolean IS_LOG4J_LOGGER = new AtomicBoolean(true);
+
   /**
    * Returns the Class object (of type <code>Class&lt;T&gt;</code>) of the  
    * argument of type <code>T</code>. 
@@ -87,12 +96,27 @@ public static boolean isLog4jLogger(Class<?> clazz) {
     if (clazz == null) {
       return false;
     }
-    Logger log = LoggerFactory.getLogger(clazz);
+    return isLog4jLogger(clazz.getName());
+  }
+
+  /**
+   * Determine whether the log of the given logger is of Log4J implementation.
+   *
+   * @param logger the logger name, usually class name as string.
+   * @return true if the logger uses Log4J implementation.
+   */
+  public static boolean isLog4jLogger(String logger) {
+    if (logger == null || !IS_LOG4J_LOGGER.get()) {
+      return false;
+    }
+    Logger log = LoggerFactory.getLogger(logger);
     try {
-      Class log4jClass = Class.forName("org.slf4j.impl.Log4jLoggerAdapter");
+      Class<?> log4jClass = Class.forName(SLF4J_LOG4J_ADAPTER_CLASS);
       return log4jClass.isInstance(log);
     } catch (ClassNotFoundException e) {
+      IS_LOG4J_LOGGER.set(false);
       return false;
     }
   }
+
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericsUtil.java

@@ -140,7 +140,7 @@ public void testGetClass() {
 
   @Test
   public void testIsLog4jLogger() throws Exception {
-    assertFalse("False if clazz is null", GenericsUtil.isLog4jLogger(null));
+    assertFalse("False if clazz is null", GenericsUtil.isLog4jLogger((Class<?>) null));
     assertTrue("The implementation is Log4j",
         GenericsUtil.isLog4jLogger(TestGenericsUtil.class));
   }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java

@@ -317,11 +317,8 @@ private void updateState() {
       if (!resolver.registerNamenode(report)) {
         LOG.warn("Cannot register namenode {}", report);
       }
-    } catch (IOException e) {
-      LOG.info("Cannot register namenode in the State Store");
-    } catch (Exception ex) {
-      LOG.error("Unhandled exception updating NN registration for {}",
-          getNamenodeDesc(), ex);
+    } catch (Exception e) {
+      LOG.error("Cannot register namenode {} in the State Store", getNamenodeDesc(), e);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java

@@ -63,6 +63,7 @@
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.InvalidBlockReportLeaseException;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
@@ -791,6 +792,9 @@ private void offerService() throws Exception {
           shouldServiceRun = false;
           return;
         }
+        if (InvalidBlockReportLeaseException.class.getName().equals(reClass)) {
+          fullBlockReportLeaseId = 0;
+        }
         LOG.warn("RemoteException in offerService", re);
         sleepAfterException();
       } catch (IOException e) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java

@@ -172,6 +172,7 @@
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
 import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand;
 import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.InvalidBlockReportLeaseException;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
@@ -1651,6 +1652,8 @@ public DatanodeCommand blockReport(final DatanodeRegistration nodeReg,
             bm.processReport(nodeReg, reports[index].getStorage(),
                 blocks, context));
         }
+      } else {
+        throw new InvalidBlockReportLeaseException(context.getReportId(), context.getLeaseId());
       }
     } catch (UnregisteredNodeException une) {
       LOG.warn("Datanode {} is attempting to report but not register yet.",