Skip to content

Commit de50e4b

Browse files
committed
address comments
1 parent 96a7269 commit de50e4b

4 files changed

Lines changed: 18 additions & 12 deletions

File tree

dev/deps/spark-deps-hadoop-2.7-hive-2.3

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -201,12 +201,12 @@ orc-shims/1.6.10//orc-shims-1.6.10.jar
201201
oro/2.0.8//oro-2.0.8.jar
202202
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
203203
paranamer/2.8//paranamer-2.8.jar
204-
parquet-column/1.12.0//parquet-column-1.12.0.jar
205-
parquet-common/1.12.0//parquet-common-1.12.0.jar
206-
parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar
207-
parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar
208-
parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar
209-
parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar
204+
parquet-column/1.12.1//parquet-column-1.12.1.jar
205+
parquet-common/1.12.1//parquet-common-1.12.1.jar
206+
parquet-encoding/1.12.1//parquet-encoding-1.12.1.jar
207+
parquet-format-structures/1.12.1//parquet-format-structures-1.12.1.jar
208+
parquet-hadoop/1.12.1//parquet-hadoop-1.12.1.jar
209+
parquet-jackson/1.12.1//parquet-jackson-1.12.1.jar
210210
protobuf-java/2.5.0//protobuf-java-2.5.0.jar
211211
py4j/0.10.9.2//py4j-0.10.9.2.jar
212212
pyrolite/4.30//pyrolite-4.30.jar

dev/deps/spark-deps-hadoop-3.2-hive-2.3

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,12 @@ orc-shims/1.6.10//orc-shims-1.6.10.jar
172172
oro/2.0.8//oro-2.0.8.jar
173173
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
174174
paranamer/2.8//paranamer-2.8.jar
175-
parquet-column/1.12.0//parquet-column-1.12.0.jar
176-
parquet-common/1.12.0//parquet-common-1.12.0.jar
177-
parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar
178-
parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar
179-
parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar
180-
parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar
175+
parquet-column/1.12.1//parquet-column-1.12.1.jar
176+
parquet-common/1.12.1//parquet-common-1.12.1.jar
177+
parquet-encoding/1.12.1//parquet-encoding-1.12.1.jar
178+
parquet-format-structures/1.12.1//parquet-format-structures-1.12.1.jar
179+
parquet-hadoop/1.12.1//parquet-hadoop-1.12.1.jar
180+
parquet-jackson/1.12.1//parquet-jackson-1.12.1.jar
181181
protobuf-java/2.5.0//protobuf-java-2.5.0.jar
182182
py4j/0.10.9.2//py4j-0.10.9.2.jar
183183
pyrolite/4.30//pyrolite-4.30.jar
Binary file not shown.

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,6 +855,12 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
855855
}
856856
}
857857

858+
test("SPARK-36726: test incorrect Parquet row group file offset") {
859+
readParquetFile("test-data/malformed-file-offset.parquet") { df =>
860+
assert(df.count() == 3650)
861+
}
862+
}
863+
858864
test("VectorizedParquetRecordReader - direct path read") {
859865
val data = (0 to 10).map(i => (i, (i + 'a').toChar.toString))
860866
withTempPath { dir =>

0 commit comments

Comments
 (0)