Address

Author: zsxwing

core/src/main/scala/org/apache/spark/internal/config/Status.scala

@@ -29,13 +29,9 @@ private[spark] object Status {
     .timeConf(TimeUnit.NANOSECONDS)
     .createWithDefaultString("100ms")
 
-  val LIVE_ENTITY_UPDATE_STALENESS_LIMIT = ConfigBuilder("spark.ui.liveUpdate.stalenessLimit")
-    .internal()
-    .doc(
-      """A time limit before we force to flush all live entities. When the last flush doesn't past
-        |this limit, UI will not trigger a heavy flush to sync the states since it may slow down
-        |Spark events processing significantly. Otherwise, UI will try to flush as soon as possible.
-      """.stripMargin)
+  val LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD = ConfigBuilder("spark.ui.liveUpdate.minFlushPeriod")
+    .doc("Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when " +
+      "incoming task events are not fired frequently.")
     .timeConf(TimeUnit.NANOSECONDS)
     .createWithDefaultString("1s")
 

core/src/main/scala/org/apache/spark/status/AppStatusListener.scala

@@ -59,12 +59,10 @@ private[spark] class AppStatusListener(
   private val liveUpdatePeriodNs = if (live) conf.get(LIVE_ENTITY_UPDATE_PERIOD) else -1L
 
   /**
-   * A time limit before we force to flush all live entities. When the last flush doesn't past
-   * this limit, UI will not trigger a heavy flush to sync the states since it may slow down Spark
-   * events processing significantly. Otherwise, UI will try to flush when receiving the next
-   * executor heartbeat.
+   * Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when incoming
+   * task events are not fired frequently.
    */
-  private val liveUpdateStalenessLimit = conf.get(LIVE_ENTITY_UPDATE_STALENESS_LIMIT)
+  private val liveUpdateMinFlushPeriod = conf.get(LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD)
 
   private val maxTasksPerStage = conf.get(MAX_RETAINED_TASKS_PER_STAGE)
   private val maxGraphRootNodes = conf.get(MAX_RETAINED_ROOT_NODES)
@@ -845,8 +843,8 @@ private[spark] class AppStatusListener(
     }
     // Flush updates if necessary. Executor heartbeat is an event that happens periodically. Flush
     // here to ensure the staleness of Spark UI doesn't last more than
-    // `max(heartbeat interval, liveUpdateStalenessLimit)`.
-    if (now - lastFlushTimeNs > liveUpdateStalenessLimit) {
+    // `max(heartbeat interval, liveUpdateMinFlushPeriod)`.
+    if (now - lastFlushTimeNs > liveUpdateMinFlushPeriod) {
       flush(maybeUpdate(_, now))
       // Re-get the current system time because `flush` may be slow and `now` is stale.
       lastFlushTimeNs = System.nanoTime()

core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala

@@ -735,7 +735,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
       // Set a small heart beat interval to make the test fast
       additionalConfs = Map(
         EXECUTOR_HEARTBEAT_INTERVAL.key -> "10ms",
-        LIVE_ENTITY_UPDATE_STALENESS_LIMIT.key -> "10ms"))) { sc =>
+        LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD.key -> "10ms"))) { sc =>
       sc.setLocalProperty(SparkContext.SPARK_JOB_INTERRUPT_ON_CANCEL, "true")
       val f = sc.parallelize(1 to 1000, 1000).foreachAsync { _ =>
         // Make the task never finish so there won't be any task start/end events after the first 2

docs/configuration.md

@@ -874,6 +874,14 @@ Apart from these, the following properties are also available, and may be useful
     operations that we can live without when rapidly processing incoming task events.
   </td>
 </tr>
+<tr>
+  <td><code>spark.ui.liveUpdate.minFlushPeriod</code></td>
+  <td>1s</td>
+  <td>
+    Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when incoming
+    task events are not fired frequently.
+  </td>
+</tr>
 <tr>
   <td><code>spark.ui.port</code></td>
   <td>4040</td>