HADOOP-18671 Add recoverLease(), setSafeMode(), isFileClosed() as interfaces to hadoop-common

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java

@@ -163,5 +163,11 @@ private CommonPathCapabilities() {
   public static final String ETAGS_PRESERVED_IN_RENAME =
       "fs.capability.etags.preserved.in.rename";
 
+  /**
+   * Does this Filesystem support lease recovery operations such as
+   * {@link LeaseRecoverable#recoverLease(Path)} and {@link LeaseRecoverable#isFileClosed(Path)}}?
+   * Value: {@value}.
+   */
+  public static final String LEASE_RECOVERABLE = "fs.capability.lease.recoverable";
 
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LeaseRecoverable.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+/**
+ * Whether the given Path of the FileSystem has the capability to perform lease recovery.
+ */
+public interface LeaseRecoverable {
+
+  /**
+   * Start the lease recovery of a file.
+   *
+   * @param file path to a file.
+   * @return true if the file is already closed, and it does not require lease recovery.
+   * @throws IOException if an error occurs during lease recovery.
+   * @throws UnsupportedOperationException if lease recovery is not supported by this filesystem.
+   */
+  boolean recoverLease(Path file) throws IOException;
+
+  /**
+   * Get the close status of a file.
+   * @param file The string representation of the path to the file
+   * @return return true if file is closed
+   * @throws IOException If an I/O error occurred
+   * @throws UnsupportedOperationException if isFileClosed is not supported by this filesystem.
+   */
+  boolean isFileClosed(Path file) throws IOException;
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SafeMode.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+/**
+ * Whether the given filesystem is in any status of safe mode.
+ */
+public interface SafeMode {
+
+  /**
+   * Enter, leave, or get safe mode.
+   *
+   * @param action One of {@link SafeModeAction} LEAVE, ENTER, GET, FORCE_EXIT.
+   * @throws IOException if set safe mode fails to proceed.
+   * @return true if the action is successfully accepted, otherwise false means rejected.
+   */
+  default boolean setSafeMode(SafeModeAction action) throws IOException {
+    return setSafeMode(action, false);
+  }
+
+  /**
+   * Enter, leave, or get safe mode.
+   *
+   * @param action    One of {@link SafeModeAction} LEAVE, ENTER, GET, FORCE_EXIT.
+   * @param isChecked If true check only for Active metadata node / NameNode's status,
+   *                  else check first metadata node / NameNode's status.
+   * @throws IOException if set safe mode fails to proceed.
+   * @return true if the action is successfully accepted, otherwise false means rejected.
+   */
+  boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException;
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SafeModeAction.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+/**
+ * An identical copy from org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction, that helps
+ * the other file system implementation to define {@link SafeMode}.
+ */
+public enum SafeModeAction {
+  /**
+   * Starting entering into safe mode.
+   */
+  ENTER,
+  /**
+   * Gracefully exit from safe mode.
+   */
+  LEAVE,
+  /**
+   * Force Exit from safe mode.
+   */
+  FORCE_EXIT,
+  /**
+   * Get the status of the safe mode.
+   */
+  GET;
+}

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md

@@ -41,4 +41,6 @@ HDFS as these are commonly expected by Hadoop client applications.
 2. [Extending the specification and its tests](extending.html)
 1. [Uploading a file using Multiple Parts](multipartuploader.html)
 1. [IOStatistics](iostatistics.html)
-1. [openFile()](openfile.html).
+1. [openFile()](openfile.html)
+1. [SafeMode](safemode.html)
+1. [LeaseRecoverable](leaserecoverable.html)

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/leaserecoverable.md

@@ -0,0 +1,52 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# <a name="LeaseRecoverable"></a> interface `LeaseRecoverable`
+
+The `LeaseRecoverable` interface tells whether a given path of current filesystem can perform lease
+recovery for open file that a lease is not explicitly renewed or the client holding it goes away.
+
+This interface should be implemented accordingly when necessary to any Filesystem that supports
+lease recovery, e.g. `DistributedFileSystem` (HDFS) and `ViewDistributedFileSystem`.
+
+```java
+public interface LeaseRecoverable {
+  boolean recoverLease(Path file) throws IOException;
+  boolean isFileClosed(Path file) throws IOException;
+}
+```
+
+There are two main functions of this interface, one performs lease recovery and another one
+verifies if a file has been closed.
+
+### boolean recoverLease(Path file)
+
+This function performs the lease recovery for the given file path, and it does not support
+directory path recovery.
+1. Return `true`, if the file has already closed, or does not require lease recovery.
+1. Return `false`, if the lease recovery is yet completed.
+1. Throw `IOException` if a directory path is given as input.
+
+### boolean isFileClosed(Path file)
+
+This function only checks if the give file path has been closed, and it does not support directory
+verification.
+1. Return `true`, if the file has been closed.
+1. Return `false`, if the file is still open.
+1. Throw `IOException` if a directory path is given as input.
+
+### Path Capabilities SHOULD BE declared
+
+If a filesystem supports `LeaseRecoverable`, it should return `true` to
+`PathCapabilities.hasPathCapability(path, "fs.capability.lease.recoverable")` for a given path.

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/safemode.md

@@ -0,0 +1,45 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# <a name="SafeMode"></a> interface `SafeMode`
+
+The `SafeMode` interface provides a way to perform safe mode actions and obtain the
+status after such actions performed to the `FileSystem`.
+
+This is admin only interface, should be implemented accordingly when necessary to
+Filesystem that support safe mode, e.g. `DistributedFileSystem` (HDFS) and
+`ViewDistributedFileSystem`.
+
+```java
+public interface SafeMode {
+  default boolean setSafeMode(SafeModeAction action) throws IOException {
+    return setSafeMode(action, false);
+  }
+  boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException;
+}
+```
+
+The goals of this interface is allow any file system implementation to share the
+same concept of safe mode with the following actions and states
+
+### Safe mode actions
+1. `GET`, get the safe mode status of the file system.
+1. `ENTER`, enter the safe mode for the file system.
+1. `LEAVE`, exit safe mode for the file system gracefully.
+1. `FORCE_EXIT`, exit safe mode for the file system even if there is any ongoing data process.
+
+### Safe mode states
+1. return true, when safe mode is on.
+1. return false, when safe mode is off, usually it's the result of safe mode actions
+with `GET`, `LEAVE`, `FORCE_EXIT`.

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractLeaseRecoveryTest.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LeaseRecoverable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.LEASE_RECOVERABLE;
+
+public abstract class AbstractContractLeaseRecoveryTest extends
+    AbstractFSContractTestBase {
+
+  @Test
+  public void testLeaseRecovery() throws Throwable {
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    ContractTestUtils.touch(fs, path);
+    LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
+
+    Assertions.assertThat(leaseRecoverableFs.recoverLease(path))
+        .describedAs("Issuing lease recovery on a closed file must be successful")
+        .isTrue();
+
+    Assertions.assertThat(leaseRecoverableFs.isFileClosed(path))
+        .describedAs("Get the isFileClose status on a closed file must be successful")
+        .isTrue();
+  }
+
+  @Test
+  public void testLeaseRecoveryFileNotExist() throws Throwable {
+    final Path path = new Path("notExist");
+    final FileSystem fs = getFileSystem();
+    LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "File does not exist",
+        () -> leaseRecoverableFs.recoverLease(path));
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "File does not exist",
+        () -> leaseRecoverableFs.isFileClosed(path));
+  }
+
+  @Test
+  public void testLeaseRecoveryFileOnDirectory() throws Throwable {
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
+    final Path parentDirectory = path.getParent();
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "Path is not a file",
+        () -> leaseRecoverableFs.recoverLease(parentDirectory));
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "Path is not a file",
+        () -> leaseRecoverableFs.isFileClosed(parentDirectory));
+  }
+
+  private LeaseRecoverable verifyAndGetLeaseRecoverableInstance(FileSystem fs, Path path)
+      throws IOException {
+    Assertions.assertThat(fs.hasPathCapability(path, LEASE_RECOVERABLE))
+        .describedAs("path capability %s of %s", LEASE_RECOVERABLE, path)
+        .isTrue();
+    Assertions.assertThat(fs)
+        .describedAs("filesystem %s", fs)
+        .isInstanceOf(LeaseRecoverable.class);
+    return (LeaseRecoverable) fs;
+  }
+}