[SPARK-45209][CORE][UI] Flame Graph Support For Executor Thread Dump Page

LICENSE

@@ -216,7 +216,8 @@ core/src/main/resources/org/apache/spark/ui/static/bootstrap*
 core/src/main/resources/org/apache/spark/ui/static/vis*
 docs/js/vendor/bootstrap.js
 connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
-
+core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.min.js
+core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.css
 
 Python Software Foundation License
 ----------------------------------

LICENSE-binary

@@ -413,7 +413,8 @@ core/src/main/java/org/apache/spark/util/collection/TimSort.java
 core/src/main/resources/org/apache/spark/ui/static/bootstrap*
 core/src/main/resources/org/apache/spark/ui/static/vis*
 docs/js/vendor/bootstrap.js
-
+core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.min.js
+core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.css
 
 ------------------------------------------------------------------------------------
 This product bundles various third-party components under other open source licenses.

core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.css

@@ -0,0 +1,47 @@
+/** https://cdn.jsdelivr.net/npm/[email protected]/dist/d3-flamegraph.css */
+.d3-flame-graph rect {
+  stroke: #EEEEEE;
+  fill-opacity: .8;
+}
+
+.d3-flame-graph rect:hover {
+  stroke: #474747;
+  stroke-width: 0.5;
+  cursor: pointer;
+}
+
+.d3-flame-graph-label {
+  pointer-events: none;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  overflow: hidden;
+  font-size: 12px;
+  font-family: Verdana;
+  margin-left: 4px;
+  margin-right: 4px;
+  line-height: 1.5;
+  padding: 0 0 0;
+  font-weight: 400;
+  color: black;
+  text-align: left;
+}
+
+.d3-flame-graph .fade {
+  opacity: 0.6 !important;
+}
+
+.d3-flame-graph .title {
+  font-size: 20px;
+  font-family: Verdana;
+}
+
+.d3-flame-graph-tip {
+    background-color: black;
+    border: none;
+    border-radius: 3px;
+    padding: 5px 10px 5px 10px;
+    min-width: 250px;
+    text-align: left;
+    color: white;
+    z-index: 10;
+}

core/src/main/resources/org/apache/spark/ui/static/flamegraph.js

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* global d3, flamegraph */
+
+/* eslint-disable no-unused-vars */
+function drawFlamegraph() {
+  const width = (window.innerWidth * 95) / 100;
+  const chart = flamegraph()
+    .width(width)
+    .cellHeight(18)
+    .transitionEase(d3.easeCubic)
+    .sort(true)
+    .title("");
+  const jsonStr = d3.select("#executor-flamegraph-data").text().trim()
+  const jsonData = JSON.parse(jsonStr);
+  d3.select("#executor-flamegraph-chart")
+		.datum(jsonData)
+    .call(chart);
+  window.onresize = () => chart.width(width);
+}
+/* eslint-enable no-unused-vars */

core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala

@@ -22,7 +22,10 @@ import javax.servlet.http.HttpServletRequest
 import scala.xml.{Node, Text}
 
 import org.apache.spark.SparkContext
+import org.apache.spark.status.api.v1.ThreadStackTrace
 import org.apache.spark.ui.{SparkUITab, UIUtils, WebUIPage}
+import org.apache.spark.ui.UIUtils.prependBaseUri
+import org.apache.spark.ui.flamegraph.FlamegraphNode
 
 private[ui] class ExecutorThreadDumpPage(
     parent: SparkUITab,
@@ -67,8 +70,10 @@ private[ui] class ExecutorThreadDumpPage(
     <div class="row">
       <div class="col-12">
         <p>Updated at {UIUtils.formatDate(time)}</p>
+        {drawExecutorFlamegraph(request, threadDump)}
         {
           // scalastyle:off
+          <p></p>
           <div style="display: flex; align-items: center;">
             <a class="expandbutton" onClick="expandAllThreadStackTrace(true)">Expand All</a>
             <a class="expandbutton d-none" onClick="collapseAllThreadStackTrace(true)">Collapse All</a>
@@ -106,4 +111,19 @@ private[ui] class ExecutorThreadDumpPage(
     }.getOrElse(Text("Error fetching thread dump"))
     UIUtils.headerSparkPage(request, s"Thread dump for executor $executorId", content, parent)
   }
+
+  // scalastyle:off
+  private def drawExecutorFlamegraph(request: HttpServletRequest, thread: Array[ThreadStackTrace]): Seq[Node] = {
+    <div>
+      <div id="executor-flamegraph-data" class="d-none">{FlamegraphNode(thread).toJsonString}</div>
+      <div id="executor-flamegraph-chart">
+        <link rel="stylesheet" type="text/css" href={prependBaseUri(request, "/static/d3-flamegraph.css")}></link>
+        <script src={UIUtils.prependBaseUri(request, "/static/d3-flamegraph.min.js")}></script>
+        <script src={UIUtils.prependBaseUri(request, "/static/d3.min.js")}></script>
+        <script src={UIUtils.prependBaseUri(request, "/static/flamegraph.js")}></script>
+        <script>drawFlamegraph()</script>
+      </div>
+    </div>
+  }
+  // scalastyle:off
 }

core/src/main/scala/org/apache/spark/ui/flamegraph/FlamegraphNode.scala

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ui.flamegraph
+
+import scala.collection.mutable.HashMap
+
+import org.apache.commons.text.StringEscapeUtils
+
+import org.apache.spark.status.api.v1.ThreadStackTrace
+
+case class FlamegraphNode(name: String) {
+  private val children = new HashMap[String, FlamegraphNode]()
+  private var value: Int = 0
+  def toJsonString: String = {
+    // scalastyle:off line.size.limit
+    s"""{"name":"$name","value":$value,"children":[${children.map(_._2.toJsonString).mkString(",")}]}"""
+    // scalastyle:on line.size.limit
+  }
+}
+
+object FlamegraphNode {
+  def apply(stacks: Array[ThreadStackTrace]): FlamegraphNode = {
+    val root = FlamegraphNode("root")
+    stacks.foreach { stack =>
+      root.value += 1
+      var cur = root
+      stack.stackTrace.elems.reverse.foreach { e =>
+        val head = e.split("\n").head
+        val name = StringEscapeUtils.escapeJson(head)
+        cur = cur.children.getOrElseUpdate(name, FlamegraphNode(name))
+        cur.value += 1
+      }
+    }
+    root
+  }
+}

dev/.rat-excludes

@@ -26,6 +26,8 @@ bootstrap.bundle.min.js
 bootstrap.min.css
 jquery-3.5.1.min.js
 d3.min.js
+d3-flamegraph.css
+d3-flamegraph.min.js
 dagre-d3.min.js
 graphlib-dot.min.js
 sorttable.js