From 28fe9339de2e82285b88b0a5a3eac1e8525f84d8 Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Fri, 22 Nov 2024 10:29:39 -0800 Subject: [PATCH 1/2] test: Enable shuffle by default in Spark tests --- dev/diffs/3.4.3.diff | 54 +++++++++++++++++----------------- dev/diffs/3.5.1.diff | 54 +++++++++++++++++----------------- dev/diffs/4.0.0-preview1.diff | 55 +++++++++++++++++------------------ 3 files changed, 78 insertions(+), 85 deletions(-) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index b78a24a4c8..4da8146f3c 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -1329,7 +1329,7 @@ index 47679ed7865..9ffbaecb98e 100644 assert(collectWithSubqueries(plan) { case s: SortAggregateExec => s }.length == sortAggCount) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala -index b14f4a405f6..88815fd078f 100644 +index b14f4a405f6..ab7baf434a5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.QueryTest @@ -1340,12 +1340,15 @@ index b14f4a405f6..88815fd078f 100644 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession -@@ -131,7 +132,7 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { +@@ -131,7 +132,10 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { spark.range(1).write.parquet(path.getAbsolutePath) val df = spark.read.parquet(path.getAbsolutePath) val columnarToRowExec = - df.queryExecution.executedPlan.collectFirst { case p: ColumnarToRowExec => p }.get -+ df.queryExecution.executedPlan.collectFirst { case p: CometColumnarToRowExec => p }.get ++ df.queryExecution.executedPlan.collectFirst { ++ case p: ColumnarToRowExec => p ++ case p: CometColumnarToRowExec => p ++ }.get try { spark.range(1).foreach { _ => columnarToRowExec.canonicalized @@ -2790,7 +2793,7 @@ index abe606ad9c1..2d930b64cca 100644 val tblTargetName = "tbl_target" val tblSourceQualified = s"default.$tblSourceName" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala -index dd55fcfe42c..2702f87c1f1 100644 +index dd55fcfe42c..0d66bcccbdc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest @@ -2814,7 +2817,7 @@ index dd55fcfe42c..2702f87c1f1 100644 } } -@@ -242,6 +247,38 @@ private[sql] trait SQLTestUtilsBase +@@ -242,6 +247,29 @@ private[sql] trait SQLTestUtilsBase protected override def _sqlContext: SQLContext = self.spark.sqlContext } @@ -2840,20 +2843,11 @@ index dd55fcfe42c..2702f87c1f1 100644 + val v = System.getenv("ENABLE_COMET_SCAN_ONLY") + v != null && v.toBoolean + } -+ -+ /** -+ * Whether Spark should apply Comet shuffle optimization. This is only effective when -+ * [[isCometEnabled]] returns true. -+ */ -+ protected def isCometShuffleEnabled: Boolean = { -+ val v = System.getenv("ENABLE_COMET_SHUFFLE") -+ v != null && v.toBoolean -+ } + protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = { SparkSession.setActiveSession(spark) super.withSQLConf(pairs: _*)(f) -@@ -434,6 +471,8 @@ private[sql] trait SQLTestUtilsBase +@@ -434,6 +462,8 @@ private[sql] trait SQLTestUtilsBase val schema = df.schema val withoutFilters = df.queryExecution.executedPlan.transform { case FilterExec(_, child) => child @@ -2863,10 +2857,10 @@ index dd55fcfe42c..2702f87c1f1 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..59adc094970 100644 +index ed2e309fa07..c77391b2c66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,34 @@ trait SharedSparkSessionBase +@@ -74,6 +74,33 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -2880,7 +2874,15 @@ index ed2e309fa07..59adc094970 100644 + conf + .set("spark.comet.exec.enabled", "true") + .set("spark.comet.exec.all.enabled", "true") ++ .set("spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .set("spark.comet.exec.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "10g") ++ } else { ++ conf ++ .set("spark.comet.exec.enabled", "false") ++ .set("spark.comet.exec.all.enabled", "false") ++ .set("spark.comet.exec.shuffle.enabled", "false") + } + + if (enableCometAnsiMode) { @@ -2888,15 +2890,6 @@ index ed2e309fa07..59adc094970 100644 + .set("spark.sql.ansi.enabled", "true") + .set("spark.comet.ansi.enabled", "true") + } -+ -+ if (isCometShuffleEnabled) { -+ conf -+ .set("spark.shuffle.manager", -+ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") -+ } else { -+ conf.set("spark.comet.exec.shuffle.enabled", "false") -+ } + } conf.set( StaticSQLConf.WAREHOUSE_PATH, @@ -2951,10 +2944,10 @@ index 1966e1e64fd..cde97a0aafe 100644 spark.sql( """ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 07361cfdce9..e40c59a4207 100644 +index 07361cfdce9..324d75149a6 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -@@ -55,25 +55,52 @@ object TestHive +@@ -55,25 +55,57 @@ object TestHive new SparkContext( System.getProperty("spark.sql.test.master", "local[1]"), "TestSQLContext", @@ -3009,6 +3002,11 @@ index 07361cfdce9..e40c59a4207 100644 + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") ++ } else { ++ conf ++ .set("spark.comet.exec.enabled", "false") ++ .set("spark.comet.exec.all.enabled", "false") ++ .set("spark.comet.exec.shuffle.enabled", "false") + } + + val a = System.getenv("ENABLE_COMET_ANSI_MODE") diff --git a/dev/diffs/3.5.1.diff b/dev/diffs/3.5.1.diff index fff50ecd34..0f9b523aac 100644 --- a/dev/diffs/3.5.1.diff +++ b/dev/diffs/3.5.1.diff @@ -1354,7 +1354,7 @@ index 47679ed7865..9ffbaecb98e 100644 assert(collectWithSubqueries(plan) { case s: SortAggregateExec => s }.length == sortAggCount) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala -index b14f4a405f6..88815fd078f 100644 +index b14f4a405f6..ab7baf434a5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.QueryTest @@ -1365,12 +1365,15 @@ index b14f4a405f6..88815fd078f 100644 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession -@@ -131,7 +132,7 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { +@@ -131,7 +132,10 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { spark.range(1).write.parquet(path.getAbsolutePath) val df = spark.read.parquet(path.getAbsolutePath) val columnarToRowExec = - df.queryExecution.executedPlan.collectFirst { case p: ColumnarToRowExec => p }.get -+ df.queryExecution.executedPlan.collectFirst { case p: CometColumnarToRowExec => p }.get ++ df.queryExecution.executedPlan.collectFirst { ++ case p: ColumnarToRowExec => p ++ case p: CometColumnarToRowExec => p ++ }.get try { spark.range(1).foreach { _ => columnarToRowExec.canonicalized @@ -2775,7 +2778,7 @@ index abe606ad9c1..2d930b64cca 100644 val tblTargetName = "tbl_target" val tblSourceQualified = s"default.$tblSourceName" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala -index dd55fcfe42c..2702f87c1f1 100644 +index dd55fcfe42c..0d66bcccbdc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest @@ -2799,7 +2802,7 @@ index dd55fcfe42c..2702f87c1f1 100644 } } -@@ -242,6 +247,38 @@ private[sql] trait SQLTestUtilsBase +@@ -242,6 +247,29 @@ private[sql] trait SQLTestUtilsBase protected override def _sqlContext: SQLContext = self.spark.sqlContext } @@ -2825,20 +2828,11 @@ index dd55fcfe42c..2702f87c1f1 100644 + val v = System.getenv("ENABLE_COMET_SCAN_ONLY") + v != null && v.toBoolean + } -+ -+ /** -+ * Whether Spark should apply Comet shuffle optimization. This is only effective when -+ * [[isCometEnabled]] returns true. -+ */ -+ protected def isCometShuffleEnabled: Boolean = { -+ val v = System.getenv("ENABLE_COMET_SHUFFLE") -+ v != null && v.toBoolean -+ } + protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = { SparkSession.setActiveSession(spark) super.withSQLConf(pairs: _*)(f) -@@ -434,6 +471,8 @@ private[sql] trait SQLTestUtilsBase +@@ -434,6 +462,8 @@ private[sql] trait SQLTestUtilsBase val schema = df.schema val withoutFilters = df.queryExecution.executedPlan.transform { case FilterExec(_, child) => child @@ -2848,10 +2842,10 @@ index dd55fcfe42c..2702f87c1f1 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..59adc094970 100644 +index ed2e309fa07..c77391b2c66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,34 @@ trait SharedSparkSessionBase +@@ -74,6 +74,33 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -2865,7 +2859,15 @@ index ed2e309fa07..59adc094970 100644 + conf + .set("spark.comet.exec.enabled", "true") + .set("spark.comet.exec.all.enabled", "true") ++ .set("spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .set("spark.comet.exec.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "10g") ++ } else { ++ conf ++ .set("spark.comet.exec.enabled", "false") ++ .set("spark.comet.exec.all.enabled", "false") ++ .set("spark.comet.exec.shuffle.enabled", "false") + } + + if (enableCometAnsiMode) { @@ -2873,15 +2875,6 @@ index ed2e309fa07..59adc094970 100644 + .set("spark.sql.ansi.enabled", "true") + .set("spark.comet.ansi.enabled", "true") + } -+ -+ if (isCometShuffleEnabled) { -+ conf -+ .set("spark.shuffle.manager", -+ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") -+ } else { -+ conf.set("spark.comet.exec.shuffle.enabled", "false") -+ } + } conf.set( StaticSQLConf.WAREHOUSE_PATH, @@ -2936,10 +2929,10 @@ index dc8b184fcee..dd69a989d40 100644 spark.sql( """ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 9284b35fb3e..2a0269bdc16 100644 +index 9284b35fb3e..0134ba7c5b4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -@@ -53,25 +53,52 @@ object TestHive +@@ -53,25 +53,57 @@ object TestHive new SparkContext( System.getProperty("spark.sql.test.master", "local[1]"), "TestSQLContext", @@ -2994,6 +2987,11 @@ index 9284b35fb3e..2a0269bdc16 100644 + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") ++ } else { ++ conf ++ .set("spark.comet.exec.enabled", "false") ++ .set("spark.comet.exec.all.enabled", "false") ++ .set("spark.comet.exec.shuffle.enabled", "false") + } + + val a = System.getenv("ENABLE_COMET_ANSI_MODE") diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff index 2de366f335..390c474f81 100644 --- a/dev/diffs/4.0.0-preview1.diff +++ b/dev/diffs/4.0.0-preview1.diff @@ -1483,7 +1483,7 @@ index 47679ed7865..9ffbaecb98e 100644 assert(collectWithSubqueries(plan) { case s: SortAggregateExec => s }.length == sortAggCount) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala -index 966f4e74712..a715193d96d 100644 +index 966f4e74712..8017e22d7f8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala @@ -23,6 +23,7 @@ import org.apache.spark.sql.QueryTest @@ -1494,12 +1494,15 @@ index 966f4e74712..a715193d96d 100644 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession -@@ -134,7 +135,7 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { +@@ -134,7 +135,10 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession { spark.range(1).write.parquet(path.getAbsolutePath) val df = spark.read.parquet(path.getAbsolutePath) val columnarToRowExec = - df.queryExecution.executedPlan.collectFirst { case p: ColumnarToRowExec => p }.get -+ df.queryExecution.executedPlan.collectFirst { case p: CometColumnarToRowExec => p }.get ++ df.queryExecution.executedPlan.collectFirst { ++ case p: ColumnarToRowExec => p ++ case p: CometColumnarToRowExec => p ++ }.get try { spark.range(1).foreach { _ => columnarToRowExec.canonicalized @@ -2980,7 +2983,7 @@ index af07aceaed1..ed0b5e6d9be 100644 val tblTargetName = "tbl_target" val tblSourceQualified = s"default.$tblSourceName" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala -index 5fbf379644f..1f2e5d7cfa0 100644 +index 5fbf379644f..47e0f4a2c9e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest @@ -3004,7 +3007,7 @@ index 5fbf379644f..1f2e5d7cfa0 100644 } } -@@ -243,6 +248,39 @@ private[sql] trait SQLTestUtilsBase +@@ -243,6 +248,29 @@ private[sql] trait SQLTestUtilsBase protected override def _sqlContext: SQLContext = self.spark.sqlContext } @@ -3030,21 +3033,11 @@ index 5fbf379644f..1f2e5d7cfa0 100644 + val v = System.getenv("ENABLE_COMET_SCAN_ONLY") + v != null && v.toBoolean + } -+ -+ -+ /** -+ * Whether Spark should apply Comet shuffle optimization. This is only effective when -+ * [[isCometEnabled]] returns true. -+ */ -+ protected def isCometShuffleEnabled: Boolean = { -+ val v = System.getenv("ENABLE_COMET_SHUFFLE") -+ v != null && v.toBoolean -+ } + protected override def withSQLConf[T](pairs: (String, String)*)(f: => T): T = { SparkSession.setActiveSession(spark) super.withSQLConf(pairs: _*)(f) -@@ -434,6 +472,8 @@ private[sql] trait SQLTestUtilsBase +@@ -434,6 +462,8 @@ private[sql] trait SQLTestUtilsBase val schema = df.schema val withoutFilters = df.queryExecution.executedPlan.transform { case FilterExec(_, child) => child @@ -3054,10 +3047,10 @@ index 5fbf379644f..1f2e5d7cfa0 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..59adc094970 100644 +index ed2e309fa07..c77391b2c66 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,34 @@ trait SharedSparkSessionBase +@@ -74,6 +74,33 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -3071,7 +3064,15 @@ index ed2e309fa07..59adc094970 100644 + conf + .set("spark.comet.exec.enabled", "true") + .set("spark.comet.exec.all.enabled", "true") ++ .set("spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .set("spark.comet.exec.shuffle.enabled", "true") + .set("spark.comet.memoryOverhead", "10g") ++ } else { ++ conf ++ .set("spark.comet.exec.enabled", "false") ++ .set("spark.comet.exec.all.enabled", "false") ++ .set("spark.comet.exec.shuffle.enabled", "false") + } + + if (enableCometAnsiMode) { @@ -3079,15 +3080,6 @@ index ed2e309fa07..59adc094970 100644 + .set("spark.sql.ansi.enabled", "true") + .set("spark.comet.ansi.enabled", "true") + } -+ -+ if (isCometShuffleEnabled) { -+ conf -+ .set("spark.shuffle.manager", -+ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") -+ .set("spark.comet.exec.shuffle.enabled", "true") -+ } else { -+ conf.set("spark.comet.exec.shuffle.enabled", "false") -+ } + } conf.set( StaticSQLConf.WAREHOUSE_PATH, @@ -3128,10 +3120,10 @@ index 52abd248f3a..7a199931a08 100644 case d: DynamicPruningExpression => d.child } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 3f8de93b330..2912f98cc5b 100644 +index 3f8de93b330..a4cdee0f24e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -@@ -53,24 +53,52 @@ object TestHive +@@ -53,24 +53,57 @@ object TestHive new SparkContext( System.getProperty("spark.sql.test.master", "local[1]"), "TestSQLContext", @@ -3186,6 +3178,11 @@ index 3f8de93b330..2912f98cc5b 100644 + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") ++ } else { ++ conf ++ .set("spark.comet.exec.enabled", "false") ++ .set("spark.comet.exec.all.enabled", "false") ++ .set("spark.comet.exec.shuffle.enabled", "false") + } + + val a = System.getenv("ENABLE_COMET_ANSI_MODE") From bd9f20a08a8c0bba3e8091d4e03eb41f95bba9a8 Mon Sep 17 00:00:00 2001 From: Kazuyuki Tanimura Date: Wed, 8 Jan 2025 22:44:22 -0800 Subject: [PATCH 2/2] test: Enable shuffle by default in Spark tests --- dev/diffs/3.4.3.diff | 12 ++++-------- dev/diffs/3.5.1.diff | 12 ++++-------- dev/diffs/4.0.0-preview1.diff | 12 ++++-------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff index 4da8146f3c..ed1dfa21cf 100644 --- a/dev/diffs/3.4.3.diff +++ b/dev/diffs/3.4.3.diff @@ -2857,10 +2857,10 @@ index dd55fcfe42c..0d66bcccbdc 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..c77391b2c66 100644 +index ed2e309fa07..71ba6533c9d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,33 @@ trait SharedSparkSessionBase +@@ -74,6 +74,31 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -2873,7 +2873,6 @@ index ed2e309fa07..c77391b2c66 100644 + if (!isCometScanOnly) { + conf + .set("spark.comet.exec.enabled", "true") -+ .set("spark.comet.exec.all.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") @@ -2881,7 +2880,6 @@ index ed2e309fa07..c77391b2c66 100644 + } else { + conf + .set("spark.comet.exec.enabled", "false") -+ .set("spark.comet.exec.all.enabled", "false") + .set("spark.comet.exec.shuffle.enabled", "false") + } + @@ -2944,10 +2942,10 @@ index 1966e1e64fd..cde97a0aafe 100644 spark.sql( """ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 07361cfdce9..324d75149a6 100644 +index 07361cfdce9..b4d53dbe900 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -@@ -55,25 +55,57 @@ object TestHive +@@ -55,25 +55,55 @@ object TestHive new SparkContext( System.getProperty("spark.sql.test.master", "local[1]"), "TestSQLContext", @@ -2998,14 +2996,12 @@ index 07361cfdce9..324d75149a6 100644 + if (v == null || !v.toBoolean) { + conf + .set("spark.comet.exec.enabled", "true") -+ .set("spark.comet.exec.all.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") + } else { + conf + .set("spark.comet.exec.enabled", "false") -+ .set("spark.comet.exec.all.enabled", "false") + .set("spark.comet.exec.shuffle.enabled", "false") + } + diff --git a/dev/diffs/3.5.1.diff b/dev/diffs/3.5.1.diff index 0f9b523aac..43e0cd5220 100644 --- a/dev/diffs/3.5.1.diff +++ b/dev/diffs/3.5.1.diff @@ -2842,10 +2842,10 @@ index dd55fcfe42c..0d66bcccbdc 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..c77391b2c66 100644 +index ed2e309fa07..71ba6533c9d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,33 @@ trait SharedSparkSessionBase +@@ -74,6 +74,31 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -2858,7 +2858,6 @@ index ed2e309fa07..c77391b2c66 100644 + if (!isCometScanOnly) { + conf + .set("spark.comet.exec.enabled", "true") -+ .set("spark.comet.exec.all.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") @@ -2866,7 +2865,6 @@ index ed2e309fa07..c77391b2c66 100644 + } else { + conf + .set("spark.comet.exec.enabled", "false") -+ .set("spark.comet.exec.all.enabled", "false") + .set("spark.comet.exec.shuffle.enabled", "false") + } + @@ -2929,10 +2927,10 @@ index dc8b184fcee..dd69a989d40 100644 spark.sql( """ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 9284b35fb3e..0134ba7c5b4 100644 +index 9284b35fb3e..37f91610500 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -@@ -53,25 +53,57 @@ object TestHive +@@ -53,25 +53,55 @@ object TestHive new SparkContext( System.getProperty("spark.sql.test.master", "local[1]"), "TestSQLContext", @@ -2983,14 +2981,12 @@ index 9284b35fb3e..0134ba7c5b4 100644 + if (v == null || !v.toBoolean) { + conf + .set("spark.comet.exec.enabled", "true") -+ .set("spark.comet.exec.all.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") + } else { + conf + .set("spark.comet.exec.enabled", "false") -+ .set("spark.comet.exec.all.enabled", "false") + .set("spark.comet.exec.shuffle.enabled", "false") + } + diff --git a/dev/diffs/4.0.0-preview1.diff b/dev/diffs/4.0.0-preview1.diff index 390c474f81..254bff4ff7 100644 --- a/dev/diffs/4.0.0-preview1.diff +++ b/dev/diffs/4.0.0-preview1.diff @@ -3047,10 +3047,10 @@ index 5fbf379644f..47e0f4a2c9e 100644 spark.internalCreateDataFrame(withoutFilters.execute(), schema) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -index ed2e309fa07..c77391b2c66 100644 +index ed2e309fa07..71ba6533c9d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala -@@ -74,6 +74,33 @@ trait SharedSparkSessionBase +@@ -74,6 +74,31 @@ trait SharedSparkSessionBase // this rule may potentially block testing of other optimization rules such as // ConstantPropagation etc. .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName) @@ -3063,7 +3063,6 @@ index ed2e309fa07..c77391b2c66 100644 + if (!isCometScanOnly) { + conf + .set("spark.comet.exec.enabled", "true") -+ .set("spark.comet.exec.all.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") @@ -3071,7 +3070,6 @@ index ed2e309fa07..c77391b2c66 100644 + } else { + conf + .set("spark.comet.exec.enabled", "false") -+ .set("spark.comet.exec.all.enabled", "false") + .set("spark.comet.exec.shuffle.enabled", "false") + } + @@ -3120,10 +3118,10 @@ index 52abd248f3a..7a199931a08 100644 case d: DynamicPruningExpression => d.child } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -index 3f8de93b330..a4cdee0f24e 100644 +index 3f8de93b330..53417076481 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala -@@ -53,24 +53,57 @@ object TestHive +@@ -53,24 +53,55 @@ object TestHive new SparkContext( System.getProperty("spark.sql.test.master", "local[1]"), "TestSQLContext", @@ -3174,14 +3172,12 @@ index 3f8de93b330..a4cdee0f24e 100644 + if (v == null || !v.toBoolean) { + conf + .set("spark.comet.exec.enabled", "true") -+ .set("spark.comet.exec.all.enabled", "true") + .set("spark.shuffle.manager", + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .set("spark.comet.exec.shuffle.enabled", "true") + } else { + conf + .set("spark.comet.exec.enabled", "false") -+ .set("spark.comet.exec.all.enabled", "false") + .set("spark.comet.exec.shuffle.enabled", "false") + } +