Support List as a return type in Hive UDF

Author: maropu

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala

@@ -23,6 +23,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.{StructField => HiveStructF
 import org.apache.hadoop.hive.serde2.typeinfo.{DecimalTypeInfo, TypeInfoFactory}
 import org.apache.hadoop.hive.serde2.{io => hiveIo}
 import org.apache.hadoop.{io => hadoopIo}
+import org.apache.spark.Logging
+import org.apache.spark.annotation.DeveloperApi
 
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -172,7 +174,7 @@ import scala.collection.JavaConversions._
  *       e.g. date_add(printf("%s-%s-%s", a,b,c), 3)
  *       We don't need to unwrap the data for printf and wrap it again and passes in data_add
  */
-private[hive] trait HiveInspectors {
+private[hive] trait HiveInspectors extends Logging {
 
   def javaClassToDataType(clz: Class[_]): DataType = clz match {
     // writable
@@ -216,8 +218,16 @@ private[hive] trait HiveInspectors {
 
     case c: Class[_] if c.isArray => ArrayType(javaClassToDataType(c.getComponentType))
 
+    // list type
+    case c: Class[_] if c == classOf[java.util.List[java.lang.Object]] =>
+      logWarning("Failed to catch a correct component type in List<> because of type erasure," +
+        " so you need to handle it correctly by yourself")
+      ArrayType(ErasedType)
+
     // Hive seems to return this for struct types?
     case c: Class[_] if c == classOf[java.lang.Object] => NullType
+
+    case c => throw new HiveDataTypeException("Unknown java type: " + c)
   }
 
   /**
@@ -828,3 +838,18 @@ private[hive] trait HiveInspectors {
     }
   }
 }
+
+/**
+ * :: DeveloperApi ::
+ * This represents an erased type because of type erasure in JVM.
+ */
+@DeveloperApi
+class ErasedType private() extends DataType {
+  override def defaultSize: Int = 1
+  private[spark] override def asNullable: ErasedType = this
+}
+
+case object ErasedType extends ErasedType
+
+/** The exception thrown from the [[HiveInspectors]]. */
+private[hive] class HiveDataTypeException(message: String) extends Exception(message)

sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFToListInt.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class UDFToListInt extends UDF {
+    public List<Integer> evaluate(Object o) {
+        return Arrays.asList(1, 2, 3);
+    }
+}

sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFToListString.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class UDFToListString extends UDF {
+    public List<String> evaluate(Object o) {
+        return Arrays.asList("data1", "data2", "data3");
+    }
+}

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala

@@ -133,6 +133,32 @@ class HiveUDFSuite extends QueryTest {
     TestHive.reset()
   }
 
+  test("UDFToListString") {
+    val testData = TestHive.sparkContext.parallelize(StringCaseClass("") :: Nil).toDF()
+    testData.registerTempTable("inputTable")
+
+    sql(s"CREATE TEMPORARY FUNCTION testUDFToListString AS '${classOf[UDFToListString].getName}'")
+    checkAnswer(
+      sql("SELECT testUDFToListString(s) FROM inputTable"), //.collect(),
+      Seq(Row("data1" :: "data2" :: "data3" :: Nil)))
+    sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFToListString")
+
+    TestHive.reset()
+  }
+
+  test("UDFToListInt") {
+    val testData = TestHive.sparkContext.parallelize(StringCaseClass("") :: Nil).toDF()
+    testData.registerTempTable("inputTable")
+
+    sql(s"CREATE TEMPORARY FUNCTION testUDFToListInt AS '${classOf[UDFToListInt].getName}'")
+    checkAnswer(
+      sql("SELECT testUDFToListInt(s) FROM inputTable"), //.collect(),
+      Seq(Row(1 :: 2 :: 3 :: Nil)))
+    sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFToListInt")
+
+    TestHive.reset()
+  }
+
   test("UDFListListInt") {
     val testData = TestHive.sparkContext.parallelize(
       ListListIntCaseClass(Nil) ::