HADOOP-17542. IPV6 support in Netutils#createSocketAddress

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java

@@ -30,6 +30,7 @@
 import java.util.concurrent.CompletableFuture;
 
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -104,12 +105,18 @@ public FileSystem getRawFileSystem() {
 
   /** Return the name of the checksum file associated with a file.*/
   public Path getChecksumFile(Path file) {
+    if (StringUtils.countMatches(file.toString(), ":") > 2) {
+      return new Path(file.getParent(), "./" + file.getName() + ".crc");
+    }
     return new Path(file.getParent(), "." + file.getName() + ".crc");
   }
 
   /** Return true iff file is a checksum file name.*/
   public static boolean isChecksumFile(Path file) {
     String name = file.getName();
+    if (StringUtils.countMatches(file.toString(), ":") > 2) {
+      return name.startsWith("./") && name.endsWith(".crc");
+    }
     return name.startsWith(".") && name.endsWith(".crc");
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java

@@ -199,8 +199,15 @@ public Path(String pathString) throws IllegalArgumentException {
     int start = 0;
 
     // parse uri scheme, if any
-    int colon = pathString.indexOf(':');
-    int slash = pathString.indexOf('/');
+    int colon = -1;
+    int slash = -1;
+    if (StringUtils.countMatches(pathString, ":") > 2) {
+      colon = pathString.indexOf(":/");
+      slash = pathString.indexOf('/');
+    } else {
+      colon = pathString.indexOf(':');
+      slash = pathString.indexOf('/');
+    }
     if ((colon != -1) &&
         ((slash == -1) || (colon < slash))) {     // has a scheme
       scheme = pathString.substring(0, colon);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java

@@ -45,6 +45,7 @@
 
 import javax.net.SocketFactory;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
@@ -223,6 +224,22 @@ public static InetSocketAddress createSocketAddr(String target,
     }
     target = target.trim();
     boolean hasScheme = target.contains("://");
+    if (StringUtils.countMatches(target, ":") > 2) {
+      // if scheme exists in the target
+      // for example : https://ffff:ffff:ffff:ffff::1:XXXXX
+      // we have to form https://[ffff:ffff:ffff:ffff::1]:XXXXX
+      if (hasScheme) {
+        int i = target.lastIndexOf("/");
+        String scheme = target.substring(0, i + 1);
+        String ipAddrWithPort = target.substring(i + 1);
+        target = scheme + normalizeV6Address(ipAddrWithPort);
+      } else {
+        // if scheme does not exists in the target
+        // for example : ffff:ffff:ffff:ffff::1:XXXXX
+        // we have to form [ffff:ffff:ffff:ffff::1]:XXXXX
+        target = normalizeV6Address(target);
+      }
+    }
     URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent);
 
     String host = uri.getHost();
@@ -275,6 +292,24 @@ private static URI createURI(String target,
     return uri;
   }
 
+  public static String normalizeV6Address(String target) {
+    if (!target.startsWith("[")) {
+      if (target.contains("%")) {
+        int i = target.lastIndexOf('%');
+        target = target.trim();
+        String port = target.substring(target.lastIndexOf(":") + 1);
+        String addr = target.substring(0, i);
+        target = "[" + addr + "]" + ":" + port;
+      } else {
+        int i = target.lastIndexOf(':');
+        String port = target.substring(target.lastIndexOf(":") + 1);
+        String addr = target.substring(0, i);
+        target = "[" + addr + "]" + ":" + port;
+      }
+    }
+    return target;
+  }
+
   /**
    * Create a socket address with the given host and port.  The hostname
    * might be replaced with another host that was set via

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java

@@ -817,4 +817,40 @@ private <T> void assertBetterArrayEquals(T[] expect, T[]got) {
     String gotStr = StringUtils.join(got, ", ");
     assertEquals(expectStr, gotStr);
   }
+
+  @Test
+  public void testCreateSocketAddressWithIPV6() throws Throwable {
+    String IPV6_SAMPLE_ADDRESS = "2a03:2880:2130:cf05:face:b00c:0:1";
+    String IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS + ":12345";
+
+    InetSocketAddress addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT,
+        1000, "myconfig");
+    assertEquals("[" + IPV6_SAMPLE_ADDRESS + "]", addr.getHostName());
+    assertEquals(12345, addr.getPort());
+
+    String IPV6_SAMPLE_ADDRESS_WITHSCOPE = IPV6_SAMPLE_ADDRESS + "%2";
+    IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS_WITHSCOPE + ":12345";
+    addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, 1000, "myconfig");
+    assertEquals("[" + IPV6_SAMPLE_ADDRESS + "]", addr.getHostName());
+    assertEquals(12345, addr.getPort());
+
+    IPV6_SAMPLE_ADDRESS = "[2a03:2880:2130:cf05:face:b00c:0:1]";
+    IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS + ":12345";
+
+    addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, 1000, "myconfig");
+    assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName());
+    assertEquals(12345, addr.getPort());
+
+    String IPV6_ADDRESS_WITH_SCHEME = "https://2a03:2880:2130:cf05:face:b00c:0:1:12345";
+    addr = NetUtils.createSocketAddr(IPV6_ADDRESS_WITH_SCHEME, 1000,
+        "myconfig");
+    assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName());
+    assertEquals(12345, addr.getPort());
+
+    IPV6_ADDRESS_WITH_SCHEME = "https://[2a03:2880:2130:cf05:face:b00c:0:1]:12345";
+    addr = NetUtils.createSocketAddr(IPV6_ADDRESS_WITH_SCHEME, 1000,
+        "myconfig");
+    assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName());
+    assertEquals(12345, addr.getPort());
+  }
 }