Skip to content

[SPARK-19824][Core] Update JsonProtocol to keep consistent with the UI #18303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jiangxb1987 wants to merge 3 commits into from
Closed

[SPARK-19824][Core] Update JsonProtocol to keep consistent with the UI #18303

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
244bbae
update JsonProtocol
jiangxb1987 Jun 14, 2017
8dc5218
add comments
jiangxb1987 Jun 18, 2017
8c39912
update comments.
jiangxb1987 Jun 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ private[deploy] object JsonProtocol {
("id" -> obj.id) ~
("host" -> obj.host) ~
("port" -> obj.port) ~
("address" -> obj.hostPort) ~
("webuiaddress" -> obj.webUiAddress) ~
("cores" -> obj.cores) ~
("coresused" -> obj.coresUsed) ~
Expand All @@ -44,7 +45,7 @@ private[deploy] object JsonProtocol {
("starttime" -> obj.startTime) ~
("id" -> obj.id) ~
("name" -> obj.desc.name) ~
("cores" -> obj.desc.maxCores) ~
("cores" -> obj.coresGranted) ~
("user" -> obj.desc.user) ~
("memoryperslave" -> obj.desc.memoryPerExecutorMB) ~
("submitdate" -> obj.submitDate.toString) ~
Expand All @@ -54,7 +55,7 @@ private[deploy] object JsonProtocol {

def writeApplicationDescription(obj: ApplicationDescription): JObject = {
Copy link
Member

@gatorsmile gatorsmile Jun 16, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO, we need to add the function descriptions for these public functions and provide a clear definition for each field. How about creating a few JIRAs and asking the open source community to do it?

Copy link
Contributor Author

@jiangxb1987 jiangxb1987 Jun 16, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me do this at the weekend in this PR.

("name" -> obj.name) ~
("cores" -> obj.maxCores) ~
("cores" -> obj.maxCores.getOrElse(0)) ~
("memoryperslave" -> obj.memoryPerExecutorMB) ~
("user" -> obj.user) ~
("command" -> obj.command.toString)
Expand All @@ -72,20 +73,25 @@ private[deploy] object JsonProtocol {
("starttime" -> obj.startTime.toString) ~
("state" -> obj.state.toString) ~
("cores" -> obj.desc.cores) ~
("memory" -> obj.desc.mem)
("memory" -> obj.desc.mem) ~
("submitdate" -> obj.submitDate.toString) ~
("worker" -> obj.worker.map(_.id).getOrElse("None")) ~
("mainclass" -> obj.desc.command.arguments(2))
}

def writeMasterState(obj: MasterStateResponse): JObject = {
val aliveWorkers = obj.workers.filter(_.isAlive())
("url" -> obj.uri) ~
("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
("aliveworkers" -> aliveWorkers.length) ~
("cores" -> aliveWorkers.map(_.cores).sum) ~
("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
("memory" -> aliveWorkers.map(_.memory).sum) ~
("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
("completeddrivers" -> obj.completedDrivers.toList.map(writeDriverInfo)) ~
("status" -> obj.status.toString)
}

Expand Down
4 changes: 2 additions & 2 deletions core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,15 @@ private[deploy] object DeployTestUtils {
}

def createDriverCommand(): Command = new Command(
"org.apache.spark.FakeClass", Seq("some arg --and-some options -g foo"),
"org.apache.spark.FakeClass", Seq("WORKER_URL", "USER_JAR", "mainClass"),
Map(("K1", "V1"), ("K2", "V2")), Seq("cp1", "cp2"), Seq("lp1", "lp2"), Seq("-Dfoo")
)

def createDriverDesc(): DriverDescription =
new DriverDescription("hdfs://some-dir/some.jar", 100, 3, false, createDriverCommand())

def createDriverInfo(): DriverInfo = new DriverInfo(3, "driver-3",
createDriverDesc(), new Date())
createDriverDesc(), JsonConstants.submitDate)

def createWorkerInfo(): WorkerInfo = {
val workerInfo = new WorkerInfo("id", "host", 8080, 4, 1234, null, "http://publicAddress:80")
Expand Down
14 changes: 10 additions & 4 deletions core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ object JsonConstants {
val appInfoJsonStr =
"""
|{"starttime":3,"id":"id","name":"name",
|"cores":4,"user":"%s",
|"cores":0,"user":"%s",
|"memoryperslave":1234,"submitdate":"%s",
|"state":"WAITING","duration":%d}
""".format(System.getProperty("user.name", "<unknown>"),
Expand All @@ -114,6 +114,7 @@ object JsonConstants {
val workerInfoJsonStr =
"""
|{"id":"id","host":"host","port":8080,
|"address":"host:8080",
|"webuiaddress":"http://publicAddress:80",
|"cores":4,"coresused":0,"coresfree":4,
|"memory":1234,"memoryused":0,"memoryfree":1234,
Expand All @@ -134,19 +135,24 @@ object JsonConstants {

val driverInfoJsonStr =
"""
|{"id":"driver-3","starttime":"3","state":"SUBMITTED","cores":3,"memory":100}
""".stripMargin
|{"id":"driver-3","starttime":"3",
|"state":"SUBMITTED","cores":3,"memory":100,
|"submitdate":"%s","worker":"None",
|"mainclass":"mainClass"}
""".format(submitDate.toString).stripMargin

val masterStateJsonStr =
"""
|{"url":"spark://host:8080",
|"workers":[%s,%s],
|"aliveworkers":2,
|"cores":8,"coresused":0,"memory":2468,"memoryused":0,
|"activeapps":[%s],"completedapps":[],
|"activedrivers":[%s],
|"completeddrivers":[%s],
|"status":"ALIVE"}
""".format(workerInfoJsonStr, workerInfoJsonStr,
appInfoJsonStr, driverInfoJsonStr).stripMargin
appInfoJsonStr, driverInfoJsonStr, driverInfoJsonStr).stripMargin

val workerStateJsonStr =
"""
Expand Down