File tree Expand file tree Collapse file tree
src/test/scala/org/apache/spark/sql/execution/benchmark Expand file tree Collapse file tree Original file line number Diff line number Diff line change 1+ OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
2+ Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
3+ LoadNoRowGroupsWhenPredicatePushedDown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
4+ ------------------------------------------------------------------------------------------------------------------------
5+ NestedFieldsPredicatePushDownDisabled 33189 34705 443 3.2 316.5 1.0X
6+ NestedFieldsPredicatePushDownEnabled 81 93 8 1291.5 0.8 408.8X
7+
8+ OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
9+ Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
10+ LoadSomeRowGroupsWhenPredicatePushedDown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
11+ ------------------------------------------------------------------------------------------------------------------------
12+ NestedFieldsPredicatePushDownDisabled 33122 34397 901 3.2 315.9 1.0X
13+ NestedFieldsPredicatePushDownEnabled 3393 3449 54 30.9 32.4 9.8X
14+
15+ OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
16+ Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
17+ LoadAllRowGroupsWhenPredicatePushedDown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
18+ ------------------------------------------------------------------------------------------------------------------------
19+ NestedFieldsPredicatePushDownDisabled 35266 35849 572 3.0 336.3 1.0X
20+ NestedFieldsPredicatePushDownEnabled 34682 36049 NaN 3.0 330.8 1.0X
21+
Original file line number Diff line number Diff line change @@ -78,6 +78,7 @@ object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
7878 benchmark.run()
7979 }
8080 }
81+
8182 /**
8283 * Benchmark for sorted data with a filter which allows to filter out all the row groups
8384 * when nested fields predicate push down enabled
@@ -104,15 +105,13 @@ object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
104105 * overhead or not if enable nested predicate push down.
105106 */
106107 def runLoadAllRowGroupsWhenPredicatePushedDown (): Unit = {
107-
108108 // all row groups will be loaded with a whole range filter
109109 val filterFn : DataFrame => DataFrame = { df =>
110110 df.filter(" nested.x >= 0" ).filter(s " nested.x <= $N" )
111111 }
112112 createAndRunBenchmark(" LoadAllRowGroupsWhenPredicatePushedDown" , filterFn)
113113 }
114114
115-
116115 override def runBenchmarkSuite (mainArgs : Array [String ]): Unit = {
117116 runLoadNoRowGroupWhenPredicatePushedDown()
118117 runLoadSomeRowGroupWhenPredicatePushedDown()
You can’t perform that action at this time.
0 commit comments