Skip to content

Commit fd66a3b

Browse files
attilapirosMarcelo Vanzin
authored andcommitted
[SPARK-23394][UI] In RDD storage page show the executor addresses instead of the IDs
## What changes were proposed in this pull request? Extending RDD storage page to show executor addresses in the block table. ## How was this patch tested? Manually: ![screen shot 2018-02-13 at 10 30 59](https://user-images.githubusercontent.com/2017933/36142668-0b3578f8-10a9-11e8-95ea-2f57703ee4af.png) Author: “attilapiros” <[email protected]> Closes #20589 from attilapiros/SPARK-23394. (cherry picked from commit 140f875) Signed-off-by: Marcelo Vanzin <[email protected]>
1 parent bb26bdb commit fd66a3b

1 file changed

Lines changed: 13 additions & 6 deletions

File tree

core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import javax.servlet.http.HttpServletRequest
2323
import scala.xml.{Node, Unparsed}
2424

2525
import org.apache.spark.status.AppStatusStore
26-
import org.apache.spark.status.api.v1.{RDDDataDistribution, RDDPartitionInfo}
26+
import org.apache.spark.status.api.v1.{ExecutorSummary, RDDDataDistribution, RDDPartitionInfo}
2727
import org.apache.spark.ui._
2828
import org.apache.spark.util.Utils
2929

@@ -76,7 +76,8 @@ private[ui] class RDDPage(parent: SparkUITab, store: AppStatusStore) extends Web
7676
rddStorageInfo.partitions.get,
7777
blockPageSize,
7878
blockSortColumn,
79-
blockSortDesc)
79+
blockSortDesc,
80+
store.executorList(true))
8081
_blockTable.table(page)
8182
} catch {
8283
case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
@@ -182,7 +183,8 @@ private[ui] class BlockDataSource(
182183
rddPartitions: Seq[RDDPartitionInfo],
183184
pageSize: Int,
184185
sortColumn: String,
185-
desc: Boolean) extends PagedDataSource[BlockTableRowData](pageSize) {
186+
desc: Boolean,
187+
executorIdToAddress: Map[String, String]) extends PagedDataSource[BlockTableRowData](pageSize) {
186188

187189
private val data = rddPartitions.map(blockRow).sorted(ordering(sortColumn, desc))
188190

@@ -198,7 +200,10 @@ private[ui] class BlockDataSource(
198200
rddPartition.storageLevel,
199201
rddPartition.memoryUsed,
200202
rddPartition.diskUsed,
201-
rddPartition.executors.mkString(" "))
203+
rddPartition.executors
204+
.map { id => executorIdToAddress.get(id).getOrElse(id) }
205+
.sorted
206+
.mkString(" "))
202207
}
203208

204209
/**
@@ -226,7 +231,8 @@ private[ui] class BlockPagedTable(
226231
rddPartitions: Seq[RDDPartitionInfo],
227232
pageSize: Int,
228233
sortColumn: String,
229-
desc: Boolean) extends PagedTable[BlockTableRowData] {
234+
desc: Boolean,
235+
executorSummaries: Seq[ExecutorSummary]) extends PagedTable[BlockTableRowData] {
230236

231237
override def tableId: String = "rdd-storage-by-block-table"
232238

@@ -243,7 +249,8 @@ private[ui] class BlockPagedTable(
243249
rddPartitions,
244250
pageSize,
245251
sortColumn,
246-
desc)
252+
desc,
253+
executorSummaries.map { ex => (ex.id, ex.hostPort) }.toMap)
247254

248255
override def pageLink(page: Int): String = {
249256
val encodedSortColumn = URLEncoder.encode(sortColumn, "UTF-8")

0 commit comments

Comments
 (0)