Skip to content

Commit 7a6df2c

Browse files
committed
SPARK-31364 Benchmark Parquet Predicate Pushdown
Add jdk8 benchmark result
1 parent 002a5e5 commit 7a6df2c

2 files changed

Lines changed: 22 additions & 2 deletions

File tree

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
2+
Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
3+
LoadNoRowGroupsWhenPredicatePushedDown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
4+
------------------------------------------------------------------------------------------------------------------------
5+
NestedFieldsPredicatePushDownDisabled 33189 34705 443 3.2 316.5 1.0X
6+
NestedFieldsPredicatePushDownEnabled 81 93 8 1291.5 0.8 408.8X
7+
8+
OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
9+
Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
10+
LoadSomeRowGroupsWhenPredicatePushedDown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
11+
------------------------------------------------------------------------------------------------------------------------
12+
NestedFieldsPredicatePushDownDisabled 33122 34397 901 3.2 315.9 1.0X
13+
NestedFieldsPredicatePushDownEnabled 3393 3449 54 30.9 32.4 9.8X
14+
15+
OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
16+
Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
17+
LoadAllRowGroupsWhenPredicatePushedDown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
18+
------------------------------------------------------------------------------------------------------------------------
19+
NestedFieldsPredicatePushDownDisabled 35266 35849 572 3.0 336.3 1.0X
20+
NestedFieldsPredicatePushDownEnabled 34682 36049 NaN 3.0 330.8 1.0X
21+

sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
7878
benchmark.run()
7979
}
8080
}
81+
8182
/**
8283
* Benchmark for sorted data with a filter which allows to filter out all the row groups
8384
* when nested fields predicate push down enabled
@@ -104,15 +105,13 @@ object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
104105
* overhead or not if enable nested predicate push down.
105106
*/
106107
def runLoadAllRowGroupsWhenPredicatePushedDown(): Unit = {
107-
108108
// all row groups will be loaded with a whole range filter
109109
val filterFn: DataFrame => DataFrame = { df =>
110110
df.filter("nested.x >= 0").filter(s"nested.x <= $N")
111111
}
112112
createAndRunBenchmark("LoadAllRowGroupsWhenPredicatePushedDown", filterFn)
113113
}
114114

115-
116115
override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
117116
runLoadNoRowGroupWhenPredicatePushedDown()
118117
runLoadSomeRowGroupWhenPredicatePushedDown()

0 commit comments

Comments
 (0)