After type conflict resolution, if a NullType is found, StringType is used.

Author: yhuai

sql/core/src/main/scala/org/apache/spark/sql/json/JsonTable.scala

@@ -67,6 +67,7 @@ object JsonTable extends Serializable with Logging {
       json: RDD[String], sampleSchema: Option[Double] = None): LogicalPlan = {
     val schemaData = sampleSchema.map(json.sample(false, _, 1)).getOrElse(json)
     val allKeys = parseJson(schemaData).map(getAllKeysWithValueTypes).reduce(_ ++ _)
+
     // Resolve type conflicts
     val resolved = allKeys.groupBy {
       case (key, dataType) => key
@@ -79,7 +80,14 @@ object JsonTable extends Serializable with Logging {
           case (_, dataType) => dataType
         }.reduce((type1: DataType, type2: DataType) => getCompatibleType(type1, type2))
 
-        (fieldName, dataType)
+        // Finally, we replace all NullType to StringType. We do not need to take care
+        // StructType because all fields with a StructType are represented by a placeholder
+        // StructType(Nil).
+        dataType match {
+          case NullType => (fieldName, StringType)
+          case ArrayType(NullType) => (fieldName, ArrayType(StringType))
+          case other => (fieldName, other)
+        }
       }
     }
 
@@ -353,4 +361,4 @@ object JsonTable extends Serializable with Logging {
   protected def asAttributes(struct: StructType): Seq[AttributeReference] = {
     struct.fields.map(f => AttributeReference(f.name, f.dataType, nullable = true)())
   }
-}
+}

sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala

@@ -65,7 +65,7 @@ class JsonSuite extends QueryTest {
       AttributeReference("double", DoubleType, true)() ::
       AttributeReference("integer", IntegerType, true)() ::
       AttributeReference("long", LongType, true)() ::
-      AttributeReference("null", NullType, true)() ::
+      AttributeReference("null", StringType, true)() ::
       AttributeReference("string", StringType, true)() :: Nil
 
     checkSchema(expectedSchema, jsonSchemaRDD.logicalPlan.output)
@@ -95,7 +95,7 @@ class JsonSuite extends QueryTest {
       AttributeReference("arrayOfDouble", ArrayType(DoubleType), true)() ::
       AttributeReference("arrayOfInteger", ArrayType(IntegerType), true)() ::
       AttributeReference("arrayOfLong", ArrayType(LongType), true)() ::
-      AttributeReference("arrayOfNull", ArrayType(NullType), true)() ::
+      AttributeReference("arrayOfNull", ArrayType(StringType), true)() ::
       AttributeReference("arrayOfString", ArrayType(StringType), true)() ::
       AttributeReference("arrayOfStruct", ArrayType(
         StructType(StructField("field1", BooleanType, true) ::