Skip to content

Commit 921e6cb

Browse files
committed
address more comments
1 parent 2b286cd commit 921e6cb

3 files changed

Lines changed: 9 additions & 6 deletions

File tree

external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
9696
// For backward compatibility, if the Avro type is Long and it is not logical type,
9797
// the value is processed as timestamp type with millisecond precision.
9898
updater.setLong(ordinal, value.asInstanceOf[Long] * 1000)
99+
case other => throw new IncompatibleSchemaException(
100+
s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
99101
}
100102

101103
case (LONG, DateType) => (updater, ordinal, value) =>

external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,14 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
9393
(getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
9494
case DateType =>
9595
(getter, ordinal) => getter.getInt(ordinal) * DateTimeUtils.MILLIS_PER_DAY
96-
case TimestampType =>
97-
(getter, ordinal) => avroType.getLogicalType match {
98-
case _: TimestampMillis => getter.getLong(ordinal) / 1000
99-
case _: TimestampMicros => getter.getLong(ordinal)
96+
case TimestampType => avroType.getLogicalType match {
97+
case _: TimestampMillis => (getter, ordinal) => getter.getLong(ordinal) / 1000
98+
case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
10099
// For backward compatibility, if the Avro type is Long and it is not logical type,
101100
// output the timestamp value as with millisecond precision.
102-
case null => getter.getLong(ordinal) / 1000
101+
case null => (getter, ordinal) => getter.getLong(ordinal) / 1000
102+
case other => throw new IncompatibleSchemaException(
103+
s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
103104
}
104105

105106
case ArrayType(et, containsNull) =>

external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ import org.apache.spark.sql._
3636
import org.apache.spark.sql.execution.datasources.DataSource
3737
import org.apache.spark.sql.internal.SQLConf
3838
import org.apache.spark.sql.test.{SharedSQLContext, SQLTestUtils}
39-
import org.apache.spark.sql.types.{StructType, _}
39+
import org.apache.spark.sql.types._
4040

4141
class AvroSuite extends QueryTest with SharedSQLContext with SQLTestUtils {
4242
val episodesAvro = testFile("episodes.avro")

0 commit comments

Comments
 (0)