HADOOP-18671 Add recoverLease(), setSafeMode(), isFileClosed() as interfaces to hadoop-common

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java

@@ -163,5 +163,11 @@ private CommonPathCapabilities() {
   public static final String ETAGS_PRESERVED_IN_RENAME =
       "fs.capability.etags.preserved.in.rename";
 
+  /**
+   * Does this Filesystem support lease recovery operations such as
+   * {@link LeaseRecoverable#recoverLease(Path)} and {@link LeaseRecoverable#isFileClosed(Path)}}?
+   * Value: {@value}.
+   */
+  public static final String LEASE_RECOVERABLE = "fs.capability.lease.recoverable";
 
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LeaseRecoverable.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+/**
+ * Whether the given Path of the FileSystem has the capability to performance
+ * lease recovery.
+ */
+public interface LeaseRecoverable {
+
+  /**
+   * Start the lease recovery of a file.
+   *
+   * @param file path to a file.
+   * @return true if the file is already closed, and it does not require lease recovery.
+   * @throws IOException if an error occurs during lease recovery.
+   * @throws UnsupportedOperationException if lease recovery is not supported by this filesystem.
+   */
+  boolean recoverLease(Path file) throws IOException;
+
+  /**
+   * Get the close status of a file.
+   * @param file The string representation of the path to the file
+   * @return return true if file is closed
+   * @throws IOException If an I/O error occurred
+   * @throws UnsupportedOperationException if isFileClosed is not supported by this filesystem.
+   */
+  boolean isFileClosed(Path file) throws IOException;
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SafeMode.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+/**
+ * Whether the given filesystem is in any status of safe mode.
+ */
+public interface SafeMode {
+
+  /**
+   * Enter, leave, or get safe mode.
+   *
+   * @param action One of {@link SafeModeAction} LEAVE, ENTER, GET, FORCE_EXIT.
+   * @throws IOException if set safe mode fails to proceed.
+   * @return true if the action is successfully accepted, otherwise false means rejected.
+   */
+  default boolean setSafeMode(SafeModeAction action) throws IOException {
+    return setSafeMode(action, false);
+  }
+
+  /**
+   * Enter, leave, or get safe mode.
+   *
+   * @param action    One of {@link SafeModeAction} LEAVE, ENTER, GET, FORCE_EXIT.
+   * @param isChecked If true check only for Active metadata node / NameNode's status,
+   *                  else check first metadata node / NameNode's status.
+   * @throws IOException if set safe mode fails to proceed.
+   * @return true if the action is successfully accepted, otherwise false means rejected.
+   */
+  boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException;
+
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SafeModeAction.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+/**
+ * An identical copy from org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction, that helps
+ * the other file system implementation to define {@link SafeMode}.
+ */
+public enum SafeModeAction {
+  /**
+   * Starting entering into safe mode.
+   */
+  ENTER,
+  /**
+   * Gracefully exit from safe mode.
+   */
+  LEAVE,
+  /**
+   * Force Exit from safe mode.
+   */
+  FORCE_EXIT,
+  /**
+   * Get the status of the safe mode.
+   */
+  GET;
+}

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md

@@ -41,4 +41,5 @@ HDFS as these are commonly expected by Hadoop client applications.
 2. [Extending the specification and its tests](extending.html)
 1. [Uploading a file using Multiple Parts](multipartuploader.html)
 1. [IOStatistics](iostatistics.html)
-1. [openFile()](openfile.html).
+1. [openFile()](openfile.html)
+1. [SafeMode](safemode.html)

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/safemode.md

@@ -0,0 +1,45 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# <a name="SafeMode"></a> interface `SafeMode`
+
+The `SafeMode` interface provides a way to perform safe mode actions and obtain the
+status after such actions performed to the `FileSystem`.
+
+This is admin only interface, should be implemented accordingly when necessary to
+Filesystem that support safe mode, e.g. `DistributedFileSystem` (HDFS) and
+`ViewDistributedFileSystem`.
+
+```java
+public interface SafeMode {
+  default boolean setSafeMode(SafeModeAction action) throws IOException {
+    return setSafeMode(action, false);
+  }
+  boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException;
+}
+```
+
+The goals of this interface is allow any file system implementation to share the
+same concept of safe mode with the following actions and states
+
+Safe mode actions
+1. `GET`, get the safe mode status of the file system.
+1. `ENTER`, enter the safe mode for the file system.
+1. `LEAVE`, exit safe mode for the file system gracefully.
+1. `FORCE_EXIT`, exit safe mode for the file system even if there is any ongoing data process.
+
+Safe mode states
+1. `ON`, when safe mode is on.
+1. `OFF`, when safe mode is off, usually it's the result of safe mode actions
+with `GET`, `LEAVE`, `FORCE_EXIT`

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractLeaseRecoveryTest.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LeaseRecoverable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.CommonPathCapabilities.LEASE_RECOVERABLE;
+
+public abstract class AbstractContractLeaseRecoveryTest extends
+    AbstractFSContractTestBase {
+
+  @Test
+  public void testLeaseRecovery() throws Throwable {
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    ContractTestUtils.touch(fs, path);
+    LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
+
+    Assertions.assertThat(leaseRecoverableFs.recoverLease(path))
+        .describedAs("Issuing lease recovery on a closed file must be successful")
+        .isTrue();
+
+    Assertions.assertThat(leaseRecoverableFs.isFileClosed(path))
+        .describedAs("Get the isFileClose status on a closed file must be successful")
+        .isTrue();
+  }
+
+  @Test
+  public void testLeaseRecoveryFileNotExist() throws Throwable {
+    final Path path = new Path("notExist");
+    final FileSystem fs = getFileSystem();
+    LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "File does not exist",
+        () -> leaseRecoverableFs.recoverLease(path));
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "File does not exist",
+        () -> leaseRecoverableFs.isFileClosed(path));
+  }
+
+  @Test
+  public void testLeaseRecoveryFileOnDirectory() throws Throwable {
+    final Path path = methodPath();
+    final FileSystem fs = getFileSystem();
+    LeaseRecoverable leaseRecoverableFs = verifyAndGetLeaseRecoverableInstance(fs, path);
+    final Path parentDirectory = path.getParent();
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "Path is not a file",
+        () -> leaseRecoverableFs.recoverLease(parentDirectory));
+
+    LambdaTestUtils.intercept(FileNotFoundException.class, "Path is not a file",
+        () -> leaseRecoverableFs.isFileClosed(parentDirectory));
+  }
+
+  private LeaseRecoverable verifyAndGetLeaseRecoverableInstance(FileSystem fs, Path path)
+      throws IOException {
+    Assertions.assertThat(fs.hasPathCapability(path, LEASE_RECOVERABLE))
+        .describedAs("path capability %s of %s", LEASE_RECOVERABLE, path)
+        .isTrue();
+    Assertions.assertThat(fs)
+        .describedAs("filesystem %s", fs)
+        .isInstanceOf(LeaseRecoverable.class);
+    return (LeaseRecoverable) fs;
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSafeModeTest.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.SafeMode;
+import org.apache.hadoop.fs.SafeModeAction;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+public abstract class AbstractContractSafeModeTest extends AbstractFSContractTestBase {
+
+  @Test
+  public void testSafeMode() throws Throwable {
+    final FileSystem fs = getFileSystem();
+    SafeMode fsWithSafeMode = verifyAndGetSafeModeInstance(fs);
+    Assertions.assertThat(fsWithSafeMode.setSafeMode(SafeModeAction.GET))
+        .describedAs("Getting the status of safe mode before entering should be off.").isFalse();
+    Assertions.assertThat(fsWithSafeMode.setSafeMode(SafeModeAction.ENTER))
+        .describedAs("Entering Safe mode and safe mode turns on.").isTrue();
+    Assertions.assertThat(fsWithSafeMode.setSafeMode(SafeModeAction.GET))
+        .describedAs("Getting the status of safe mode after entering, safe mode should be on.")
+        .isTrue();
+    Assertions.assertThat(fsWithSafeMode.setSafeMode(SafeModeAction.LEAVE))
+        .describedAs("Leaving safe mode, and safe mode switches off.").isFalse();
+    Assertions.assertThat(fsWithSafeMode.setSafeMode(SafeModeAction.FORCE_EXIT))
+        .describedAs("Force exist safe mode at any time, safe mode should always switches off.")
+        .isFalse();
+  }
+
+  private SafeMode verifyAndGetSafeModeInstance(FileSystem fs) {
+    Assertions.assertThat(fs)
+        .describedAs("File system %s must be an instance of %s", fs, SafeMode.class.getClass())
+        .isInstanceOf(SafeMode.class);
+    return (SafeMode) fs;
+  }
+}