Skip to content

Commit 73e01eb

Browse files
LuciferYangLuciferYang
committed
[SPARK-52280][TEST] Upgrade MiMa to use 4.0.0 as previousSparkVersion
### What changes were proposed in this pull request? The main change of this pr as follows: 1. Bump MiMa's `previousSparkVersion` to 4.0.0 and explicitly specify the `_2.13` suffix for the artifactId; otherwise, the `_2.12` suffix will be added by default, but Spark 4.0 does not support Scala 2.12, and there is no `spark-*_2.12.jar` available. 2. Clean up expired rules and case match 2.1. Remove `v40excludes` and `case v if v.startsWith("4.0")` 2.2 An exclude rule related to `SPARK-51261` has been added for `v41excludes`. This was likely an omission from before. 3. In `GenerateMIMAIgnore.scala`, replacing `Opcodes.ASM7` with `Opcodes.ASM8` has fixed the following error: ``` Exception in thread "main" java.lang.UnsupportedOperationException: Record requires ASM8 at org.objectweb.asm.ClassVisitor.visitRecordComponent(ClassVisitor.java:323) at org.objectweb.asm.ClassReader.readRecordComponent(ClassReader.java:954) at org.objectweb.asm.ClassReader.accept(ClassReader.java:732) at org.objectweb.asm.ClassReader.accept(ClassReader.java:425) at org.clapper.classutil.asm.ClassFile$.load(ClassFinderImpl.scala:224) at org.clapper.classutil.ClassFinder.classData(ClassFinder.scala:404) at org.clapper.classutil.ClassFinder.$anonfun$processOpenZip$2(ClassFinder.scala:359) at scala.collection.Iterator$$anon$9.next(Iterator.scala:584) at scala.collection.Iterator$$anon$10.nextCur(Iterator.scala:594) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:608) at scala.collection.immutable.LazyList$.scala$collection$immutable$LazyList$$stateFromIterator(LazyList.scala:1195) at scala.collection.immutable.LazyList$.$anonfun$stateFromIterator$1(LazyList.scala:1195) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state$lzycompute(LazyList.scala:282) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state(LazyList.scala:273) at scala.collection.immutable.LazyList.isEmpty(LazyList.scala:292) at scala.collection.immutable.LazyList.$anonfun$lazyAppendedAll$1(LazyList.scala:401) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state$lzycompute(LazyList.scala:282) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state(LazyList.scala:273) at scala.collection.immutable.LazyList.isEmpty(LazyList.scala:292) at scala.collection.immutable.LazyList.$anonfun$lazyAppendedAll$1(LazyList.scala:401) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state$lzycompute(LazyList.scala:282) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state(LazyList.scala:273) at scala.collection.immutable.LazyList.isEmpty(LazyList.scala:292) at scala.collection.immutable.LazyList.$anonfun$mapImpl$1(LazyList.scala:539) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state$lzycompute(LazyList.scala:282) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state(LazyList.scala:273) at scala.collection.immutable.LazyList.isEmpty(LazyList.scala:292) at scala.collection.immutable.LazyList$.$anonfun$filterImpl$1(LazyList.scala:1036) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state$lzycompute(LazyList.scala:282) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state(LazyList.scala:273) at scala.collection.immutable.LazyList.isEmpty(LazyList.scala:292) at scala.collection.immutable.LazyList$.$anonfun$filterImpl$1(LazyList.scala:1036) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state$lzycompute(LazyList.scala:282) at scala.collection.immutable.LazyList.scala$collection$immutable$LazyList$$state(LazyList.scala:273) at scala.collection.immutable.LazyList.isEmpty(LazyList.scala:292) at scala.collection.immutable.LazyList$LazyIterator.hasNext(LazyList.scala:1274) at scala.collection.mutable.Growable.addAll(Growable.scala:61) at scala.collection.mutable.Growable.addAll$(Growable.scala:57) at scala.collection.immutable.SetBuilderImpl.addAll(Set.scala:405) at scala.collection.immutable.Set$.from(Set.scala:362) at scala.collection.IterableOnceOps.toSet(IterableOnce.scala:1469) at scala.collection.IterableOnceOps.toSet$(IterableOnce.scala:1469) at scala.collection.AbstractIterable.toSet(Iterable.scala:935) at org.apache.spark.tools.GenerateMIMAIgnore$.getClasses(GenerateMIMAIgnore.scala:165) at org.apache.spark.tools.GenerateMIMAIgnore$.privateWithin(GenerateMIMAIgnore.scala:64) at org.apache.spark.tools.GenerateMIMAIgnore$.main(GenerateMIMAIgnore.scala:131) at org.apache.spark.tools.GenerateMIMAIgnore.main(GenerateMIMAIgnore.scala) ``` ### Why are the changes needed? To ensure that MiMa checks cover new APIs added in Spark 4.0.0. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass GitHub Actions - Local verification of `dev/mima` can be executed successfully. ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#50999 from LuciferYang/SPARK-52280. Authored-by: yangjie01 <[email protected]> Signed-off-by: yangjie01 <[email protected]>
1 parent 711b813 commit 73e01eb

3 files changed

Lines changed: 9 additions & 226 deletions

File tree

‎project/MimaBuild.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,13 @@ object MimaBuild {
8686

8787
def mimaSettings(sparkHome: File, projectRef: ProjectRef): Seq[Setting[_]] = {
8888
val organization = "org.apache.spark"
89-
val previousSparkVersion = "3.5.0"
89+
val previousSparkVersion = "4.0.0"
9090
val project = projectRef.project
9191
val id = "spark-" + project
9292

9393
Seq(
9494
mimaFailOnNoPrevious := true,
95-
mimaPreviousArtifacts := Set(organization %% id % previousSparkVersion),
95+
mimaPreviousArtifacts := Set(organization % s"${id}_2.13" % previousSparkVersion),
9696
mimaBinaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value)
9797
)
9898
}

‎project/MimaExcludes.scala‎

Lines changed: 6 additions & 223 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
* limitations under the License.
1616
*/
1717

18-
import com.typesafe.tools.mima.core
1918
import com.typesafe.tools.mima.core.*
2019

2120
/**
@@ -34,209 +33,14 @@ import com.typesafe.tools.mima.core.*
3433
*/
3534
object MimaExcludes {
3635

37-
lazy val v41excludes = v40excludes ++ Seq(
36+
// Exclude rules for 4.1.x from 4.0.0
37+
lazy val v41excludes = defaultExcludes ++ Seq(
3838
// [SPARK-51261][ML][CONNECT] Introduce model size estimation to control ml cache
39-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.linalg.Vector.getSizeInBytes")
40-
)
41-
42-
// Exclude rules for 4.0.x from 3.5.0
43-
lazy val v40excludes = defaultExcludes ++ Seq(
44-
// [SPARK-44863][UI] Add a button to download thread dump as a txt in Spark UI
45-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ThreadStackTrace.*"),
46-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.status.api.v1.ThreadStackTrace$"),
47-
//[SPARK-46399][Core] Add exit status to the Application End event for the use of Spark Listener
48-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerApplicationEnd.*"),
49-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerApplicationEnd$"),
50-
// [SPARK-45427][CORE] Add RPC SSL settings to SSLOptions and SparkTransportConf
51-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.network.netty.SparkTransportConf.fromSparkConf"),
52-
// [SPARK-45022][SQL] Provide context for dataset API errors
53-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.QueryContext.contextType"),
54-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.QueryContext.code"),
55-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.QueryContext.callSite"),
56-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.QueryContext.summary"),
57-
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI$default$3"),
58-
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.Decimal.fromStringANSI"),
59-
// [SPARK-45762][CORE] Support shuffle managers defined in user jars by changing startup order
60-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.SparkEnv.this"),
61-
// [SPARK-46480][CORE][SQL] Fix NPE when table cache task attempt
62-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.isFailed"),
63-
64-
// SPARK-43299: Convert StreamingQueryException in Scala Client
65-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryException"),
66-
67-
// SPARK-45856: Move ArtifactManager from Spark Connect into SparkSession (sql/core)
68-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.apply"),
69-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.userId"),
70-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.sessionId"),
71-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.copy"),
72-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.copy$default$3"),
73-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.this"),
74-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.storage.CacheId$"),
75-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.storage.CacheId.apply"),
76-
77-
// SPARK-46410: Assign error classes/subclasses to JdbcUtils.classifyException
78-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.JdbcDialect.classifyException"),
79-
// TODO(SPARK-46878): Invalid Mima report for StringType extension
80-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.types.StringType.this"),
81-
// SPARK-47011: Remove deprecated BinaryClassificationMetrics.scoreLabelsWeight
82-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.BinaryClassificationMetrics.scoreLabelsWeight"),
83-
// SPARK-46938: Javax -> Jakarta namespace change.
84-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ui.ProxyRedirectHandler$ResponseWrapper"),
85-
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ui.ProxyRedirectHandler#ResponseWrapper.this"),
86-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.DB2Dialect#DB2SQLBuilder.this"),
87-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.DB2Dialect#DB2SQLQueryBuilder.this"),
88-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect#MsSqlServerSQLBuilder.this"),
89-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.MsSqlServerDialect#MsSqlServerSQLQueryBuilder.this"),
90-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.MySQLDialect#MySQLSQLBuilder.this"),
91-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.MySQLDialect#MySQLSQLQueryBuilder.this"),
92-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.OracleDialect#OracleSQLBuilder.this"),
93-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.jdbc.OracleDialect#OracleSQLQueryBuilder.this"),
94-
// SPARK-47706: Bump json4s from 3.7.0-M11 to 4.0.7
95-
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.expressions.MutableAggregationBuffer.jsonValue"),
96-
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.types.DataType#JSortedObject.unapplySeq"),
97-
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.mllib.tree.model.TreeEnsembleModel#SaveLoadV1_0.readMetadata"),
98-
// SPARK-47814: Move `KinesisTestUtils` & `WriteInputFormatTestDataGenerator` from `main` to `test`
99-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.python.TestWritable"),
100-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.python.TestWritable$"),
101-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.python.WriteInputFormatTestDataGenerator"),
102-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.api.python.WriteInputFormatTestDataGenerator$"),
103-
// SPARK-47764: Cleanup shuffle dependencies based on ShuffleCleanupMode
104-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.shuffle.MigratableResolver.addShuffleToSkip"),
105-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext#implicits._sqlContext"),
106-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLImplicits._sqlContext"),
107-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.SQLImplicits.session"),
108-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SparkSession#implicits._sqlContext"),
109-
// SPARK-48761: Add clusterBy() to CreateTableWriter.
110-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.CreateTableWriter.clusterBy"),
111-
// SPARK-48900: Add `reason` string to all job / stage / job group cancellation calls
112-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.scheduler.JobWaiter.cancel"),
113-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.FutureAction.cancel"),
114-
// SPARK-48901: Add clusterBy() to DataStreamWriter.
115-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.DataStreamWriter.clusterBy"),
116-
// SPARK-49027: A shared Column API
117-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.ColumnName"),
118-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.TypedColumn"),
119-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.functions"),
120-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.functions$"),
121-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.Aggregator"),
122-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.MutableAggregationBuffer"),
123-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.UserDefinedAggregateFunction"),
124-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.UserDefinedFunction"),
125-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.Window"),
126-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.Window$"),
127-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.WindowSpec"),
128-
129-
// SPARK-49423: Consolidate Observation in sql/api
130-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.Observation"),
131-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.Observation$"),
132-
133-
// SPARK-49425: Create a shared DataFrameWriter interface.
134-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DataFrameWriter"),
135-
136-
// SPARK-49284: Shared Catalog interface.
137-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.catalog.CatalogMetadata"),
138-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.catalog.Column"),
139-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.catalog.Database"),
140-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.catalog.Function"),
141-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.catalog.Table"),
142-
143-
// SPARK-49426: Shared DataFrameWriterV2
144-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.CreateTableWriter"),
145-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DataFrameWriterV2"),
146-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.WriteConfigMethods"),
147-
148-
// SPARK-49424: Shared Encoders
149-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.Encoders"),
150-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.Encoders$"),
151-
152-
// SPARK-49413: Create a shared RuntimeConfig interface.
153-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.RuntimeConfig"),
154-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.RuntimeConfig$"),
155-
156-
// SPARK-49287: Shared Streaming interfaces
157-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.scheduler.SparkListenerEvent"),
158-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.ForeachWriter"),
159-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.SourceProgress"),
160-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.SourceProgress$"),
161-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StateOperatorProgress"),
162-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StateOperatorProgress$"),
163-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener"),
164-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener$"),
165-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener$Event"),
166-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener$QueryIdleEvent"),
167-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener$QueryProgressEvent"),
168-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener$QueryStartedEvent"),
169-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryListener$QueryTerminatedEvent"),
170-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryStatus"),
171-
172-
// SPARK-49415: Shared SQLImplicits.
173-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DatasetHolder"),
174-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DatasetHolder$"),
175-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.LowPrioritySQLImplicits"),
176-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.sql.SQLContext$implicits$"),
177-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.sql.SQLImplicits"),
178-
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.SQLImplicits.StringToColumn"),
179-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLImplicits.this"),
180-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLImplicits$StringToColumn"),
181-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.sql.SparkSession$implicits$"),
182-
ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.SQLImplicits.session"),
39+
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.linalg.Vector.getSizeInBytes"),
18340

184-
// SPARK-49282: Shared SparkSessionBuilder
185-
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.sql.SparkSession$Builder"),
186-
187-
// SPARK-49286: Avro/Protobuf functions in sql/api
188-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.avro.functions"),
189-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.avro.functions$"),
190-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.protobuf.functions"),
191-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.protobuf.functions$"),
192-
193-
// SPARK-49434: Move aggregators to sql/api
194-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.javalang.typed"),
195-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.scalalang.typed"),
196-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.expressions.scalalang.typed$"),
197-
198-
// SPARK-49418: Consolidate thread local handling in sql/api
199-
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.SparkSession.setActiveSession"),
200-
ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.SparkSession.setDefaultSession"),
201-
ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.api.SparkSessionCompanion.clearActiveSession"),
202-
ProblemFilters.exclude[DirectAbstractMethodProblem]("org.apache.spark.sql.api.SparkSessionCompanion.clearDefaultSession"),
203-
204-
// SPARK-49748: Add getCondition and deprecate getErrorClass in SparkThrowable
205-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.SparkThrowable.getCondition"),
206-
207-
// SPARK-50112: Moving avro files from connector to sql/core
208-
ProblemFilters.exclude[Problem]("org.apache.spark.sql.avro.*"),
209-
210-
// SPARK-49700: Unified Scala SQL Interface.
211-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DataFrameNaFunctions"),
212-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DataFrameReader"),
213-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DataFrameStatFunctions"),
214-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.KeyValueGroupedDataset"),
215-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLImplicits"),
216-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SparkSession"),
217-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SparkSession$"),
218-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SparkSession$Builder"),
219-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SparkSession$implicits$"),
220-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.package"),
221-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.package$"),
222-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.catalog.Catalog"),
223-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.DataStreamReader"),
224-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.DataStreamWriter"),
225-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.DataStreamWriter$"),
226-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQueryManager"),
227-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.StreamingQuery"),
228-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext"),
229-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$"),
230-
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$implicits$"),
231-
232-
// SPARK-50768: Introduce TaskContext.createResourceUninterruptibly to avoid stream leak by task interruption
233-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.interruptible"),
234-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.pendingInterrupt"),
235-
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.createResourceUninterruptibly"),
236-
237-
) ++ loggingExcludes("org.apache.spark.sql.DataFrameReader") ++
238-
loggingExcludes("org.apache.spark.sql.streaming.DataStreamReader") ++
239-
loggingExcludes("org.apache.spark.sql.SparkSession#Builder")
41+
// [SPARK-52221][SQL] Refactor SqlScriptingLocalVariableManager into more generic context manager
42+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.scripting.SqlScriptingExecution.withLocalVariableManager")
43+
)
24044

24145
// Default exclude rules
24246
lazy val defaultExcludes = Seq(
@@ -280,29 +84,8 @@ object MimaExcludes {
28084
}
28185
)
28286

283-
private def loggingExcludes(fqn: String) = {
284-
Seq(
285-
ProblemFilters.exclude[MissingTypesProblem](fqn),
286-
missingMethod(fqn, "logName"),
287-
missingMethod(fqn, "log"),
288-
missingMethod(fqn, "logInfo"),
289-
missingMethod(fqn, "logDebug"),
290-
missingMethod(fqn, "logTrace"),
291-
missingMethod(fqn, "logWarning"),
292-
missingMethod(fqn, "logError"),
293-
missingMethod(fqn, "isTraceEnabled"),
294-
missingMethod(fqn, "initializeLogIfNecessary"),
295-
missingMethod(fqn, "initializeLogIfNecessary$default$2"),
296-
missingMethod(fqn, "initializeForcefully"))
297-
}
298-
299-
private def missingMethod(names: String*) = {
300-
ProblemFilters.exclude[DirectMissingMethodProblem](names.mkString("."))
301-
}
302-
30387
def excludes(version: String): Seq[Problem => Boolean] = version match {
30488
case v if v.startsWith("4.1") => v41excludes
305-
case v if v.startsWith("4.0") => v40excludes
30689
case _ => Seq()
30790
}
30891
}

‎tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ object GenerateMIMAIgnore {
157157
* and subpackages both from directories and jars present on the classpath.
158158
*/
159159
private def getClasses(packageName: String): Set[String] = {
160-
val finder = ClassFinder(maybeOverrideAsmVersion = Some(Opcodes.ASM7))
160+
val finder = ClassFinder(maybeOverrideAsmVersion = Some(Opcodes.ASM8))
161161
finder
162162
.getClasses()
163163
.map(_.name)

0 commit comments

Comments
 (0)