Skip to content

Commit e1d618b

Browse files
jerryshaojerryshao
committed
Address the comment to use MemoryMetrics instead
Change-Id: Ic1edf428c8b15c15434ce510b63dd4f60a0a1326
1 parent 135e233 commit e1d618b

5 files changed

Lines changed: 105 additions & 75 deletions

File tree

‎core/src/main/resources/org/apache/spark/ui/static/executorspage.js‎

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -253,21 +253,25 @@ $(document).ready(function () {
253253
var deadTotalBlacklisted = 0;
254254

255255
response.forEach(function (exec) {
256-
exec.usedOnHeapStorageMemory = exec.hasOwnProperty('usedOnHeapStorageMemory') ? exec.usedOnHeapStorageMemory : 0;
257-
exec.totalOnHeapStorageMemory = exec.hasOwnProperty('totalOnHeapStorageMemory') ? exec.totalOnHeapStorageMemory : 0;
258-
exec.usedOffHeapStorageMemory = exec.hasOwnProperty('usedOffHeapStorageMemory') ? exec.usedOffHeapStorageMemory : 0;
259-
exec.totalOffHeapStorageMemory = exec.hasOwnProperty('totalOffHeapStorageMemory') ? exec.totalOffHeapStorageMemory : 0;
256+
var memoryMetrics = {
257+
usedOnHeapStorageMemory: 0,
258+
usedOffHeapStorageMemory: 0,
259+
totalOnHeapStorageMemory: 0,
260+
totalOffHeapStorageMemory: 0
261+
};
262+
263+
exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
260264
});
261265

262266
response.forEach(function (exec) {
263267
allExecCnt += 1;
264268
allRDDBlocks += exec.rddBlocks;
265269
allMemoryUsed += exec.memoryUsed;
266270
allMaxMemory += exec.maxMemory;
267-
allOnHeapMemoryUsed += exec.usedOnHeapStorageMemory;
268-
allOnHeapMaxMemory += exec.totalOnHeapStorageMemory;
269-
allOffHeapMemoryUsed += exec.usedOffHeapStorageMemory;
270-
allOffHeapMaxMemory += exec.totalOffHeapStorageMemory;
271+
allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
272+
allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
273+
allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
274+
allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
271275
allDiskUsed += exec.diskUsed;
272276
allTotalCores += exec.totalCores;
273277
allMaxTasks += exec.maxTasks;
@@ -286,10 +290,10 @@ $(document).ready(function () {
286290
activeRDDBlocks += exec.rddBlocks;
287291
activeMemoryUsed += exec.memoryUsed;
288292
activeMaxMemory += exec.maxMemory;
289-
activeOnHeapMemoryUsed += exec.usedOnHeapStorageMemory;
290-
activeOnHeapMaxMemory += exec.totalOnHeapStorageMemory;
291-
activeOffHeapMemoryUsed += exec.usedOffHeapStorageMemory;
292-
activeOffHeapMaxMemory += exec.totalOffHeapStorageMemory;
293+
activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
294+
activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
295+
activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
296+
activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
293297
activeDiskUsed += exec.diskUsed;
294298
activeTotalCores += exec.totalCores;
295299
activeMaxTasks += exec.maxTasks;
@@ -308,10 +312,10 @@ $(document).ready(function () {
308312
deadRDDBlocks += exec.rddBlocks;
309313
deadMemoryUsed += exec.memoryUsed;
310314
deadMaxMemory += exec.maxMemory;
311-
deadOnHeapMemoryUsed += exec.usedOnHeapStorageMemory;
312-
deadOnHeapMaxMemory += exec.totalOnHeapStorageMemory;
313-
deadOffHeapMemoryUsed += exec.usedOffHeapStorageMemory;
314-
deadOffHeapMaxMemory += exec.totalOffHeapStorageMemory;
315+
deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
316+
deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
317+
deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
318+
deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
315319
deadDiskUsed += exec.diskUsed;
316320
deadTotalCores += exec.totalCores;
317321
deadMaxTasks += exec.maxTasks;
@@ -431,10 +435,10 @@ $(document).ready(function () {
431435
{
432436
data: function (row, type) {
433437
if (type !== 'display')
434-
return row.usedOnHeapStorageMemory;
438+
return row.memoryMetrics.usedOnHeapStorageMemory;
435439
else
436-
return (formatBytes(row.usedOnHeapStorageMemory, type) + ' / ' +
437-
formatBytes(row.totalOnHeapStorageMemory, type));
440+
return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
441+
formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
438442
},
439443
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
440444
$(nTd).addClass('on_heap_memory')
@@ -443,10 +447,10 @@ $(document).ready(function () {
443447
{
444448
data: function (row, type) {
445449
if (type !== 'display')
446-
return row.usedOffHeapStorageMemory;
450+
return row.memoryMetrics.usedOffHeapStorageMemory;
447451
else
448-
return (formatBytes(row.usedOffHeapStorageMemory, type) + ' / ' +
449-
formatBytes(row.totalOffHeapStorageMemory, type));
452+
return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
453+
formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
450454
},
451455
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
452456
$(nTd).addClass('off_heap_memory')

‎core/src/main/scala/org/apache/spark/status/api/v1/api.scala‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,13 @@ class ExecutorSummary private[spark](
7676
val isBlacklisted: Boolean,
7777
val maxMemory: Long,
7878
val executorLogs: Map[String, String],
79-
val usedOnHeapStorageMemory: Option[Long],
80-
val usedOffHeapStorageMemory: Option[Long],
81-
val totalOnHeapStorageMemory: Option[Long],
82-
val totalOffHeapStorageMemory: Option[Long])
79+
val memoryMetrics: Option[MemoryMetrics])
80+
81+
class MemoryMetrics private[spark](
82+
val usedOnHeapStorageMemory: Long,
83+
val usedOffHeapStorageMemory: Long,
84+
val totalOnHeapStorageMemory: Long,
85+
val totalOffHeapStorageMemory: Long)
8386

8487
class JobData private[spark](
8588
val jobId: Int,

‎core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest
2121

2222
import scala.xml.Node
2323

24-
import org.apache.spark.status.api.v1.ExecutorSummary
24+
import org.apache.spark.status.api.v1.{ExecutorSummary, MemoryMetrics}
2525
import org.apache.spark.ui.{UIUtils, WebUIPage}
2626

2727
// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive
@@ -114,10 +114,16 @@ private[spark] object ExecutorsPage {
114114
val rddBlocks = status.numBlocks
115115
val memUsed = status.memUsed
116116
val maxMem = status.maxMem
117-
val onHeapMemUsed = status.onHeapMemUsed
118-
val offHeapMemUsed = status.offHeapMemUsed
119-
val maxOnHeapMem = status.maxOnHeapMem
120-
val maxOffHeapMem = status.maxOffHeapMem
117+
val memoryMetrics = for {
118+
onHeapUsed <- status.onHeapMemUsed
119+
offHeapUsed <- status.offHeapMemUsed
120+
maxOnHeap <- status.maxOnHeapMem
121+
maxOffHeap <- status.maxOffHeapMem
122+
} yield {
123+
new MemoryMetrics(onHeapUsed, offHeapUsed, maxOnHeap, maxOffHeap)
124+
}
125+
126+
121127
val diskUsed = status.diskUsed
122128
val taskSummary = listener.executorToTaskSummary.getOrElse(execId, ExecutorTaskSummary(execId))
123129

@@ -142,10 +148,7 @@ private[spark] object ExecutorsPage {
142148
taskSummary.isBlacklisted,
143149
maxMem,
144150
taskSummary.executorLogs,
145-
onHeapMemUsed,
146-
offHeapMemUsed,
147-
maxOnHeapMem,
148-
maxOffHeapMem
151+
memoryMetrics
149152
)
150153
}
151154
}

‎core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
2323
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
2424
},
25-
"usedOnHeapStorageMemory" : 0,
26-
"usedOffHeapStorageMemory" : 0,
27-
"totalOnHeapStorageMemory" : 384093388,
28-
"totalOffHeapStorageMemory" : 524288000
25+
"memoryMetrics": {
26+
"usedOnHeapStorageMemory": 0,
27+
"usedOffHeapStorageMemory": 0,
28+
"totalOnHeapStorageMemory": 384093388,
29+
"totalOffHeapStorageMemory": 524288000
30+
}
2931
}, {
3032
"id" : "driver",
3133
"hostPort" : "172.22.0.167:51475",
@@ -47,10 +49,12 @@
4749
"isBlacklisted" : true,
4850
"maxMemory" : 908381388,
4951
"executorLogs" : { },
50-
"usedOnHeapStorageMemory" : 0,
51-
"usedOffHeapStorageMemory" : 0,
52-
"totalOnHeapStorageMemory" : 384093388,
53-
"totalOffHeapStorageMemory" : 524288000
52+
"memoryMetrics": {
53+
"usedOnHeapStorageMemory": 0,
54+
"usedOffHeapStorageMemory": 0,
55+
"totalOnHeapStorageMemory": 384093388,
56+
"totalOffHeapStorageMemory": 524288000
57+
}
5458
}, {
5559
"id" : "1",
5660
"hostPort" : "172.22.0.167:51490",
@@ -75,10 +79,12 @@
7579
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
7680
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
7781
},
78-
"usedOnHeapStorageMemory" : 0,
79-
"usedOffHeapStorageMemory" : 0,
80-
"totalOnHeapStorageMemory" : 384093388,
81-
"totalOffHeapStorageMemory" : 524288000
82+
"memoryMetrics": {
83+
"usedOnHeapStorageMemory": 0,
84+
"usedOffHeapStorageMemory": 0,
85+
"totalOnHeapStorageMemory": 384093388,
86+
"totalOffHeapStorageMemory": 524288000
87+
}
8288
}, {
8389
"id" : "0",
8490
"hostPort" : "172.22.0.167:51491",
@@ -103,10 +109,12 @@
103109
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
104110
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
105111
},
106-
"usedOnHeapStorageMemory" : 0,
107-
"usedOffHeapStorageMemory" : 0,
108-
"totalOnHeapStorageMemory" : 384093388,
109-
"totalOffHeapStorageMemory" : 524288000
112+
"memoryMetrics": {
113+
"usedOnHeapStorageMemory": 0,
114+
"usedOffHeapStorageMemory": 0,
115+
"totalOnHeapStorageMemory": 384093388,
116+
"totalOffHeapStorageMemory": 524288000
117+
}
110118
}, {
111119
"id" : "3",
112120
"hostPort" : "172.22.0.167:51485",
@@ -131,8 +139,10 @@
131139
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
132140
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
133141
},
134-
"usedOnHeapStorageMemory" : 0,
135-
"usedOffHeapStorageMemory" : 0,
136-
"totalOnHeapStorageMemory" : 384093388,
137-
"totalOffHeapStorageMemory" : 524288000
142+
"memoryMetrics": {
143+
"usedOnHeapStorageMemory": 0,
144+
"usedOffHeapStorageMemory": 0,
145+
"totalOnHeapStorageMemory": 384093388,
146+
"totalOffHeapStorageMemory": 524288000
147+
}
138148
} ]

‎core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
2323
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
2424
},
25-
"usedOnHeapStorageMemory" : 0,
26-
"usedOffHeapStorageMemory" : 0,
27-
"totalOnHeapStorageMemory" : 384093388,
28-
"totalOffHeapStorageMemory" : 524288000
25+
"memoryMetrics": {
26+
"usedOnHeapStorageMemory": 0,
27+
"usedOffHeapStorageMemory": 0,
28+
"totalOnHeapStorageMemory": 384093388,
29+
"totalOffHeapStorageMemory": 524288000
30+
}
2931
}, {
3032
"id" : "driver",
3133
"hostPort" : "172.22.0.167:51475",
@@ -47,10 +49,12 @@
4749
"isBlacklisted" : true,
4850
"maxMemory" : 908381388,
4951
"executorLogs" : { },
50-
"usedOnHeapStorageMemory" : 0,
51-
"usedOffHeapStorageMemory" : 0,
52-
"totalOnHeapStorageMemory" : 384093388,
53-
"totalOffHeapStorageMemory" : 524288000
52+
"memoryMetrics": {
53+
"usedOnHeapStorageMemory": 0,
54+
"usedOffHeapStorageMemory": 0,
55+
"totalOnHeapStorageMemory": 384093388,
56+
"totalOffHeapStorageMemory": 524288000
57+
}
5458
}, {
5559
"id" : "1",
5660
"hostPort" : "172.22.0.167:51490",
@@ -75,10 +79,12 @@
7579
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
7680
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
7781
},
78-
"usedOnHeapStorageMemory" : 0,
79-
"usedOffHeapStorageMemory" : 0,
80-
"totalOnHeapStorageMemory" : 384093388,
81-
"totalOffHeapStorageMemory" : 524288000
82+
"memoryMetrics": {
83+
"usedOnHeapStorageMemory": 0,
84+
"usedOffHeapStorageMemory": 0,
85+
"totalOnHeapStorageMemory": 384093388,
86+
"totalOffHeapStorageMemory": 524288000
87+
}
8288
}, {
8389
"id" : "0",
8490
"hostPort" : "172.22.0.167:51491",
@@ -103,10 +109,12 @@
103109
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
104110
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
105111
},
106-
"usedOnHeapStorageMemory" : 0,
107-
"usedOffHeapStorageMemory" : 0,
108-
"totalOnHeapStorageMemory" : 384093388,
109-
"totalOffHeapStorageMemory" : 524288000
112+
"memoryMetrics": {
113+
"usedOnHeapStorageMemory": 0,
114+
"usedOffHeapStorageMemory": 0,
115+
"totalOnHeapStorageMemory": 384093388,
116+
"totalOffHeapStorageMemory": 524288000
117+
}
110118
}, {
111119
"id" : "3",
112120
"hostPort" : "172.22.0.167:51485",
@@ -131,8 +139,10 @@
131139
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
132140
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
133141
},
134-
"usedOnHeapStorageMemory" : 0,
135-
"usedOffHeapStorageMemory" : 0,
136-
"totalOnHeapStorageMemory" : 384093388,
137-
"totalOffHeapStorageMemory" : 524288000
142+
"memoryMetrics": {
143+
"usedOnHeapStorageMemory": 0,
144+
"usedOffHeapStorageMemory": 0,
145+
"totalOnHeapStorageMemory": 384093388,
146+
"totalOffHeapStorageMemory": 524288000
147+
}
138148
} ]

0 commit comments

Comments
 (0)