HDFS-16463. Make dirent cross platform compatible (#4370)

* jnihelper.c in HDFS native client uses
  dirent.h. This header file isn't available
  on Windows.
* This PR provides a cross platform
  compatible implementation for dirent
  under the XPlatform library.

Author: GauthamBanasandra

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt

@@ -35,6 +35,8 @@ add_library(native_mini_dfs
     ../libhdfs/jni_helper.c
     ${OS_DIR}/mutexes.c
     ${OS_DIR}/thread_local_storage.c
+    $<TARGET_OBJECTS:x_platform_obj>
+    $<TARGET_OBJECTS:x_platform_obj_c_api>
 )
 
 add_executable(test_native_mini_dfs test_native_mini_dfs.c)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt

@@ -39,6 +39,8 @@ hadoop_add_dual_library(hdfs
     jclasses.c
     ${OS_DIR}/mutexes.c
     ${OS_DIR}/thread_local_storage.c
+    $<TARGET_OBJECTS:x_platform_obj>
+    $<TARGET_OBJECTS:x_platform_obj_c_api>
 )
 if(NEED_LINK_DL)
    set(LIB_DL dl)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c

@@ -23,10 +23,10 @@
 #include "platform.h"
 #include "os/mutexes.h"
 #include "os/thread_local_storage.h"
+#include "x-platform/c-api/dirent.h"
 #include "x-platform/types.h"
 
 #include <errno.h>
-#include <dirent.h>
 #include <stdio.h> 
 #include <string.h> 
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt

@@ -22,5 +22,6 @@ else()
   set(SYSCALL_SRC syscall_linux.cc)
 endif()
 
-add_library(x_platform_obj OBJECT ${SYSCALL_SRC} utils.cc)
-add_library(x_platform_obj_c_api OBJECT $<TARGET_OBJECTS:x_platform_obj> c-api/syscall.cc)
+add_library(x_platform_obj OBJECT ${SYSCALL_SRC} utils.cc dirent.cc)
+add_library(x_platform_obj_c_api OBJECT $<TARGET_OBJECTS:x_platform_obj> c-api/syscall.cc c-api/dirent.cc)
+target_compile_definitions(x_platform_obj_c_api PRIVATE USE_X_PLATFORM_DIRENT)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/core/dirent.h

@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_CORE_DIRENT_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_CORE_DIRENT_H
+
+/**
+ * DIR struct holds the pointer to XPlatform::Dirent instance. Since this will
+ * be used in C, we can't hold the pointer to XPlatform::Dirent. We're working
+ * around this by using a void pointer and casting it to XPlatform::Dirent when
+ * needed in C++.
+ */
+typedef struct DIR {
+  void *x_platform_dirent_ptr;
+} DIR;
+
+/**
+ * dirent struct contains the name of the file/folder while iterating through
+ * the directory's children.
+ */
+struct dirent {
+  char d_name[256];
+};
+
+/**
+ * Opens a directory for iteration. Internally, it instantiates DIR struct for
+ * the given path. closedir must be called on the returned pointer to DIR struct
+ * when done.
+ *
+ * @param dir_path The path to the directory to iterate through.
+ * @return A pointer to the DIR struct.
+ */
+DIR *opendir(const char *dir_path);
+
+/**
+ * For iterating through the children of the directory pointed to by the DIR
+ * struct pointer.
+ *
+ * @param dir The pointer to the DIR struct.
+ * @return A pointer to dirent struct containing the name of the current child
+ * file/folder.
+ */
+struct dirent *readdir(DIR *dir);
+
+/**
+ * De-allocates the XPlatform::Dirent instance pointed to by the DIR pointer.
+ *
+ * @param dir The pointer to DIR struct to close.
+ * @return 0 if successful.
+ */
+int closedir(DIR *dir);
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.cc

@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <cerrno>
+#include <iostream>
+#include <iterator>
+#include <system_error>
+#include <variant>
+
+#include "x-platform/c-api/dirent.h"
+#include "x-platform/dirent.h"
+
+DIR *opendir(const char *dir_path) {
+  const auto dir = new DIR;
+  dir->x_platform_dirent_ptr = new XPlatform::Dirent(dir_path);
+  return dir;
+}
+
+struct dirent *readdir(DIR *dir) {
+  /*
+   * We will use a static variable to hold the dirent, so that we align with the
+   * readdir's implementation in dirent.h header file in Linux.
+   */
+  static struct dirent static_dir_entry;
+
+  // Get the XPlatform::Dirent instance and move the iterator.
+  const auto x_platform_dirent =
+      static_cast<XPlatform::Dirent *>(dir->x_platform_dirent_ptr);
+  const auto dir_entry = x_platform_dirent->NextFile();
+
+  // End of iteration.
+  if (std::holds_alternative<std::monostate>(dir_entry)) {
+    return nullptr;
+  }
+
+  // Error in iteration.
+  if (std::holds_alternative<std::error_code>(dir_entry)) {
+    const auto err = std::get<std::error_code>(dir_entry);
+    errno = err.value();
+
+#ifdef X_PLATFORM_C_API_DIRENT_DEBUG
+    std::cerr << "Error in listing directory: " << err.message() << std::endl;
+#endif
+
+    return nullptr;
+  }
+
+  // Return the current child file/folder's name.
+  if (std::holds_alternative<std::filesystem::directory_entry>(dir_entry)) {
+    const auto entry = std::get<std::filesystem::directory_entry>(dir_entry);
+    const auto filename = entry.path().filename().string();
+
+    // The file name's length shouldn't exceed 256.
+    if (filename.length() >= 256) {
+      errno = 1;
+      return nullptr;
+    }
+
+    std::fill(std::begin(static_dir_entry.d_name),
+              std::end(static_dir_entry.d_name), '\0');
+    std::copy(filename.begin(), filename.end(),
+              std::begin(static_dir_entry.d_name));
+  }
+  return &static_dir_entry;
+}
+
+int closedir(DIR *dir) {
+  const auto x_platform_dirent =
+      static_cast<XPlatform::Dirent *>(dir->x_platform_dirent_ptr);
+  delete x_platform_dirent;
+  delete dir;
+
+  // We can't use the void return type for closedir since we want to align the
+  // closedir method's signature in dirent.h header file in Linux.
+  return 0;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/dirent.h

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_DIRENT_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_DIRENT_H
+
+#if !(defined(WIN32) || defined(USE_X_PLATFORM_DIRENT))
+
+/*
+ * For non-Windows environments, we use the dirent.h header itself.
+ */
+#include <dirent.h>
+
+#else
+
+/*
+ * If it's a Windows environment or if the macro USE_X_PLATFORM_DIRENT is
+ * defined, we switch to using dirent from the XPlatform library.
+ */
+#include "x-platform/c-api/extern/dirent.h"
+
+#endif
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/c-api/extern/dirent.h

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_EXTERN_DIRENT_H
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_C_API_EXTERN_DIRENT_H
+
+/*
+ * We will use extern "C" only on Windows.
+ */
+#if defined(WIN32) && defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "x-platform/c-api/core/dirent.h"
+
+#if defined(WIN32) && defined(__cplusplus)
+}
+#endif
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.cc

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <filesystem>
+#include <system_error>
+#include <variant>
+
+#include "dirent.h"
+
+std::variant<std::monostate, std::filesystem::directory_entry, std::error_code>
+XPlatform::Dirent::NextFile() {
+  if (dir_it_err_) {
+    return dir_it_err_;
+  }
+
+  if (dir_it_ == std::filesystem::end(dir_it_)) {
+    return std::monostate();
+  }
+
+  const std::filesystem::directory_entry dir_entry = *dir_it_;
+  dir_it_ = dir_it_.increment(dir_it_err_);
+  return dir_entry;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/dirent.h

@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_DIRENT
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_DIRENT
+
+#include <filesystem>
+#include <string>
+#include <system_error>
+#include <variant>
+
+namespace XPlatform {
+/**
+ * {@class XPlatform::Dirent} provides the functionality to perform a one-time
+ * iteration per {@link XPlatform::Dirent} through the child files or folders
+ * under a given path.
+ */
+class Dirent {
+public:
+  Dirent(const std::string &path)
+      : dir_it_{std::filesystem::path{path}, dir_it_err_} {}
+
+  // Abiding to the Rule of 5
+  Dirent(const Dirent &) = default;
+  Dirent(Dirent &&) = default;
+  Dirent &operator=(const Dirent &) = default;
+  Dirent &operator=(Dirent &&) = default;
+  ~Dirent() = default;
+
+  /**
+   * Advances the iterator {@link XPlatform::Dirent#dir_it_} to the next file in
+   * the given path.
+   *
+   * @return An {@link std::variant} comprising of any one of the following
+   * types:
+   * 1. {@link std::monostate} which indicates the end of iteration of all the
+   * files in the given path.
+   * 2. {@link std::filesystem::directory_entry} which is the directory entry of
+   * the current file.
+   * 3. {@link std::error_code} which corresponds to the error in retrieving the
+   * file.
+   */
+  std::variant<std::monostate, std::filesystem::directory_entry,
+               std::error_code>
+  NextFile();
+
+private:
+  /**
+   * Indicates the error corresponding to the most recent invocation of
+   * directory iteration by {@link XPlatform::Dirent#dir_it_}.
+   */
+  std::error_code dir_it_err_{};
+
+  /**
+   * The iterator used for iterating through the files or folders under the
+   * given path.
+   */
+  std::filesystem::directory_iterator dir_it_;
+};
+} // namespace XPlatform
+
+#endif