[HADP-58922] Supplement default GC options in SparkContext initialization (apache#821)

Author: wangyum

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -442,7 +442,7 @@ class SparkContext(config: SparkConf) extends Logging {
 
     SparkContext.supplementJavaModuleOptions(_conf)
     SparkContext.supplementJavaIPv6Options(_conf)
-    SparkContext.removeMuiltipleGCFlags(_conf)
+    SparkContext.supplementJavaGCOptions(_conf)
 
     _driverLogger = DriverLogger(_conf)
 
@@ -3534,19 +3534,19 @@ object SparkContext extends Logging {
     supplement(EXECUTOR_JAVA_OPTIONS)
   }
 
-  private def removeMuiltipleGCFlags(conf: SparkConf): Unit = {
+  private def supplementJavaGCOptions(conf: SparkConf): Unit = {
     val gcFlags = Set("-XX:+UseG1GC", "-XX:+UseConcMarkSweepGC", "-XX:+UseSerialGC", "-XX:+UseZGC",
       "-XX:+UseShenandoahGC", "-XX:+UseEpsilonGC")
-    def remove(key: OptionalConfigEntry[String]): Unit = {
+    def supplement(key: OptionalConfigEntry[String]): Unit = {
       conf.get(key).foreach { opts =>
-        val splitOpts = opts.split("\\s+")
-        if (splitOpts.exists(gcFlags)) {
-          conf.set(key.key, splitOpts.filterNot(_ == "-XX:+UseParallelGC").mkString(" "))
+        if (!opts.split("\\s+").exists(gcFlags.contains)) {
+          conf.set(key.key, s"-XX:+UseParallelGC $opts")
         }
       }
     }
-    remove(DRIVER_JAVA_OPTIONS)
-    remove(EXECUTOR_JAVA_OPTIONS)
+
+    supplement(DRIVER_JAVA_OPTIONS)
+    supplement(EXECUTOR_JAVA_OPTIONS)
   }
 }
 

core/src/test/scala/org/apache/spark/SparkContextSuite.scala

@@ -1462,19 +1462,26 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     assert(sc.listJars().head.contains(tmpJar.getName))
   }
 
-  test("Remove UseParallelGC when another GC is specified") {
+  test("SPARK-53084: Supplement UseParallelGC when GC algorithm is not specified") {
     val conf = new SparkConf().setAppName("test").setMaster("local")
-      .set(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS, " -XX:+UseG1GC")
-      .set(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-XX:+UseParallelGC")
-      .set(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS, " -XX:+UseZGC")
-      .set(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "-XX:+UseParallelGC")
+    sc = new SparkContext(conf)
+    val driverJavaOptions = sc.conf.get(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
+    assert(driverJavaOptions.contains("-XX:+UseParallelGC"))
+    val executorJavaOptions = sc.getConf.get(EXECUTOR_JAVA_OPTIONS).get
+    assert(executorJavaOptions.contains("-XX:+UseParallelGC"))
+  }
+
+  test("SPARK-53084: Do not supplement UseParallelGC when GC algorithm is specified") {
+    val conf = new SparkConf().setAppName("test").setMaster("local")
+      .set(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS,
+        "-XX:+UseG1GC  -XX:PerMethodRecompilationCutoff=10000")
+      .set(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS, "-XX:+UseC4GC")
     sc = new SparkContext(conf)
     val driverJavaOptions = sc.conf.get(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
     assert(driverJavaOptions.contains("-XX:+UseG1GC"))
     assert(!driverJavaOptions.contains("-XX:+UseParallelGC"))
     val executorJavaOptions = sc.getConf.get(EXECUTOR_JAVA_OPTIONS).get
-    assert(executorJavaOptions.contains("-XX:+UseZGC"))
-    assert(!executorJavaOptions.contains("-XX:+UseParallelGC"))
+    assert(executorJavaOptions.contains("-XX:+UseParallelGC"))
   }
 }