Merge branch 'apache:trunk' into HADOOP-18861

Author: anmolanmol1234

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java

@@ -75,6 +75,7 @@ public enum OpType {
     GET_STORAGE_POLICIES("op_get_storage_policies"),
     GET_STORAGE_POLICY("op_get_storage_policy"),
     GET_TRASH_ROOT("op_get_trash_root"),
+    GET_TRASH_ROOTS("op_get_trash_roots"),
     GET_XATTR("op_get_xattr"),
     LIST_CACHE_DIRECTIVE("op_list_cache_directive"),
     LIST_CACHE_POOL("op_list_cache_pool"),

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -3612,6 +3612,8 @@ public Path getTrashRoot(Path path) {
    */
   @Override
   public Collection<FileStatus> getTrashRoots(boolean allUsers) {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOTS);
     Set<FileStatus> ret = new HashSet<>();
     // Get normal trash roots
     ret.addAll(super.getTrashRoots(allUsers));

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java

@@ -22,6 +22,7 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
 import org.apache.hadoop.fs.ContentSummary;
@@ -877,6 +878,40 @@ public static Map<String, String> getErasureCodeCodecs(Map<?, ?> json) {
     return map;
   }
 
+  public static Collection<FileStatus> getTrashRoots(Map<?, ?> json) {
+    List<?> objs = (List<?>) json.get("Paths");
+    if (objs != null) {
+      FileStatus[] trashRoots = new FileStatus[objs.size()];
+      for (int i = 0; i < objs.size(); i++) {
+        Map<?, ?> m = (Map<?, ?>) objs.get(i);
+        trashRoots[i] = toFileStatus(m);
+      }
+      return Arrays.asList(trashRoots);
+    }
+    return new ArrayList<FileStatus>(0);
+  }
+
+  public static FileStatus toFileStatus(Map<?, ?> json) {
+    Path path = new Path(getString(json, "path", ""));
+    long length = getLong(json, "length", 0);
+    boolean isdir = getBoolean(json, "isdir", false);
+    short replication = (short) getInt(json, "block_replication", -1);
+    long blockSize = getLong(json, "blocksize", 256);
+    long modificationTime = getLong(json, "modification_time", 0);
+    long accessTime = getLong(json, "access_time", 0);
+    String permString = getString(json, "permission", null);
+    FsPermission permission = toFsPermission(permString);
+    String owner = getString(json, "owner", null);
+    String group = getString(json, "group", null);
+    if (json.get("symlink") != null) {
+      Path symlink = new Path((String) json.get("symlink"));
+      return new FileStatus(length, isdir, replication, blockSize, modificationTime,
+          accessTime, permission, owner, group, symlink, path);
+    }
+    return new FileStatus(length, isdir, replication, blockSize, modificationTime,
+        accessTime, permission, owner, group, path);
+  }
+
   private static List<SnapshotDiffReport.DiffReportEntry> toDiffList(
       List<?> objs) {
     if (objs == null) {

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -1956,6 +1956,26 @@ String decodeResponse(Map<?, ?> json) throws IOException {
     }
   }
 
+  public Collection<FileStatus> getTrashRoots(boolean allUsers) {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOTS);
+
+    final HttpOpParam.Op op = GetOpParam.Op.GETTRASHROOTS;
+    try {
+      Collection<FileStatus> trashRoots =
+          new FsPathResponseRunner<Collection<FileStatus>>(op, null,
+              new AllUsersParam(allUsers)) {
+            @Override
+            Collection<FileStatus> decodeResponse(Map<?, ?> json) throws IOException {
+              return JsonUtilClient.getTrashRoots(json);
+            }
+          }.run();
+      return trashRoots;
+    } catch (IOException e) {
+      return super.getTrashRoots(allUsers);
+    }
+  }
+
   @Override
   public void access(final Path path, final FsAction mode) throws IOException {
     final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AllUsersParam.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** AllUsers parameter. */
+public class AllUsersParam extends BooleanParam {
+  /** Parameter name. */
+  public static final String NAME = "allusers";
+  /** Default parameter value. */
+  public static final String DEFAULT = FALSE;
+
+  private static final Domain DOMAIN = new Domain(NAME);
+
+  /**
+   * Constructor.
+   * @param value the parameter value.
+   */
+  public AllUsersParam(final Boolean value) {
+    super(DOMAIN, value);
+  }
+
+  /**
+   * Constructor.
+   * @param str a string representation of the parameter value.
+   */
+  public AllUsersParam(final String str) {
+    this(DOMAIN.parse(str));
+  }
+
+  @Override
+  public String getName() {
+    return NAME;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java

@@ -69,6 +69,7 @@ public enum Op implements HttpOpParam.Op {
     GETSTATUS(false, HttpURLConnection.HTTP_OK),
     GETECPOLICIES(false, HttpURLConnection.HTTP_OK),
     GETECCODECS(false, HttpURLConnection.HTTP_OK),
+    GETTRASHROOTS(false, HttpURLConnection.HTTP_OK),
     GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK);
 
     final boolean redirect;

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java

@@ -43,6 +43,7 @@
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
 import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
+import org.apache.hadoop.hdfs.web.resources.AllUsersParam;
 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
 import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
@@ -344,7 +345,8 @@ protected Response get(
       final TokenKindParam tokenKind,
       final TokenServiceParam tokenService,
       final NoRedirectParam noredirectParam,
-      final StartAfterParam startAfter
+      final StartAfterParam startAfter,
+      final AllUsersParam allUsers
   ) throws IOException, URISyntaxException {
     try {
       final Router router = getRouter();
@@ -393,7 +395,7 @@ protected Response get(
             offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
             excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
             snapshotDiffStartPath, snapshotDiffIndex,
-            tokenKind, tokenService, noredirectParam, startAfter);
+            tokenKind, tokenService, noredirectParam, startAfter, allUsers);
       }
       default:
         throw new UnsupportedOperationException(op + " is not supported");

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/StateStoreService.java

@@ -64,6 +64,12 @@
  * <li>File {@link
  * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl
  * StateStoreFileImpl}
+ * <li>FileSystem {@link
+ * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl
+ * StateStoreFileSystemImpl}
+ * <li>MySQL {@link
+ * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreMySQLImpl
+ * StateStoreMySQLImpl}
  * <li>ZooKeeper {@link
  * org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl
  * StateStoreZooKeeperImpl}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -2909,7 +2909,7 @@ public boolean processReport(final DatanodeID nodeID,
             + "discarded non-initial block report from {}"
             + " because namenode still in startup phase",
             strBlockReportId, fullBrLeaseId, nodeID);
-        blockReportLeaseManager.removeLease(node);
+        removeDNLeaseIfNeeded(node);
         return !node.hasStaleStorages();
       }
 
@@ -2957,6 +2957,23 @@ public boolean processReport(final DatanodeID nodeID,
     return !node.hasStaleStorages();
   }
 
+  /**
+   * Remove the DN lease only when we have received block reports,
+   * for all storages for a particular DN.
+   */
+  void removeDNLeaseIfNeeded(DatanodeDescriptor node) {
+    boolean needRemoveLease = true;
+    for (DatanodeStorageInfo sInfo : node.getStorageInfos()) {
+      if (sInfo.getBlockReportCount() == 0) {
+        needRemoveLease = false;
+        break;
+      }
+    }
+    if (needRemoveLease) {
+      blockReportLeaseManager.removeLease(node);
+    }
+  }
+
   public void removeBRLeaseIfNeeded(final DatanodeID nodeID,
       final BlockReportContext context) throws IOException {
     namesystem.writeLock();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java

@@ -30,6 +30,7 @@
 import java.security.PrivilegedExceptionAction;
 import java.util.Base64;
 import java.util.Base64.Encoder;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
@@ -56,6 +57,7 @@
 import javax.ws.rs.core.Response.ResponseBuilder;
 import javax.ws.rs.core.Response.Status;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.fs.QuotaUsage;
 import org.apache.hadoop.fs.StorageType;
@@ -1067,14 +1069,16 @@ public Response getRoot(
       @QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
           final NoRedirectParam noredirect,
       @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT)
-          final StartAfterParam startAfter
+          final StartAfterParam startAfter,
+      @QueryParam(AllUsersParam.NAME) @DefaultValue(AllUsersParam.DEFAULT)
+          final AllUsersParam allUsers
       ) throws IOException, InterruptedException {
     return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
         renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes,
         fsAction, snapshotName, oldSnapshotName,
         snapshotDiffStartPath, snapshotDiffIndex,
         tokenKind, tokenService,
-        noredirect, startAfter);
+        noredirect, startAfter, allUsers);
   }
 
   /** Handle HTTP GET request. */
@@ -1124,12 +1128,14 @@ public Response get(
       @QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
           final NoRedirectParam noredirect,
       @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT)
-          final StartAfterParam startAfter
+          final StartAfterParam startAfter,
+      @QueryParam(AllUsersParam.NAME) @DefaultValue(AllUsersParam.DEFAULT)
+          final AllUsersParam allUsers
       ) throws IOException, InterruptedException {
 
     init(ugi, delegation, username, doAsUser, path, op, offset, length,
         renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction,
-        snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter);
+        snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter, allUsers);
 
     return doAs(ugi, new PrivilegedExceptionAction<Response>() {
       @Override
@@ -1138,7 +1144,7 @@ public Response run() throws IOException, URISyntaxException {
             op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
             excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
             snapshotDiffStartPath, snapshotDiffIndex,
-            tokenKind, tokenService, noredirect, startAfter);
+            tokenKind, tokenService, noredirect, startAfter, allUsers);
       }
     });
   }
@@ -1172,7 +1178,8 @@ protected Response get(
       final TokenKindParam tokenKind,
       final TokenServiceParam tokenService,
       final NoRedirectParam noredirectParam,
-      final StartAfterParam startAfter
+      final StartAfterParam startAfter,
+      final AllUsersParam allUsers
       ) throws IOException, URISyntaxException {
     final Configuration conf = (Configuration) context
         .getAttribute(JspHelper.CURRENT_CONF);
@@ -1322,6 +1329,12 @@ protected Response get(
       final String jsonStr = JsonUtil.toJsonString("Path", trashPath);
       return Response.ok(jsonStr).type(MediaType.APPLICATION_JSON).build();
     }
+    case GETTRASHROOTS: {
+      Boolean value = allUsers.getValue();
+      final Collection<FileStatus> trashPaths = getTrashRoots(conf, value);
+      final String js = JsonUtil.toJsonString(trashPaths);
+      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+    }
     case LISTSTATUS_BATCH:
     {
       byte[] start = HdfsFileStatus.EMPTY_NAME;
@@ -1558,6 +1571,12 @@ public Void run() throws IOException {
     };
   }
 
+  private Collection<FileStatus> getTrashRoots(Configuration conf, boolean allUsers)
+      throws IOException {
+    FileSystem fs = FileSystem.get(conf != null ? conf : new Configuration());
+    return fs.getTrashRoots(allUsers);
+  }
+
 
   /** Handle HTTP DELETE request for the root. */
   @DELETE