[SPARK-37727][SQL] Show ignored confs & hide warnings for conf already set in SparkSession.builder.getOrCreate

sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala

@@ -1065,18 +1065,35 @@ object SparkSession extends Logging {
   private[sql] def applyModifiableSettings(
       session: SparkSession,
       options: java.util.HashMap[String, String]): Unit = {
+    // Lazy val to avoid an unnecessary session state initialization
+    lazy val conf = session.sessionState.conf
+
+    val dedupOptions = if (options.isEmpty) Map.empty[String, String] else (
+      options.asScala.toSet -- conf.getAllConfs.toSet).toMap
     val (staticConfs, otherConfs) =
-      options.asScala.partition(kv => SQLConf.isStaticConfigKey(kv._1))
+      dedupOptions.partition(kv => SQLConf.isStaticConfigKey(kv._1))
+
+    otherConfs.foreach { case (k, v) => conf.setConfString(k, v) }
 
-    otherConfs.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
+    // Note that other runtime SQL options, for example, for other third-party datasource
+    // can be marked as an ignored configuration here.
+    val maybeIgnoredConfs = otherConfs.filterNot { case (k, _) => conf.isModifiable(k) }
 
+    if (staticConfs.nonEmpty || maybeIgnoredConfs.nonEmpty) {
+      logWarning(
+        "Using an existing Spark session; only runtime SQL configurations will take effect.")
+    }
     if (staticConfs.nonEmpty) {
-      logWarning("Using an existing SparkSession; the static sql configurations will not take" +
-        " effect.")
+      logDebug("Ignored static SQL configurations:\n  " +
+        conf.redactOptions(staticConfs).toSeq.map { case (k, v) => s"$k=$v" }.mkString("\n  "))
     }
-    if (otherConfs.nonEmpty) {
-      logWarning("Using an existing SparkSession; some spark core configurations may not take" +
-        " effect.")
+    if (maybeIgnoredConfs.nonEmpty) {
+      // Only print out non-static and non-runtime SQL configurations.
+      // Note that this might show core configurations or source specific
+      // options defined in the third-party datasource.
+      logDebug("Configurations that might not take effect:\n  " +
+        conf.redactOptions(
+          maybeIgnoredConfs).toSeq.map { case (k, v) => s"$k=$v" }.mkString("\n  "))
     }
   }