Skip to content

Commit 3c6ba79

Browse files
committed
[EXT][SPARK-18750][yarn] Avoid using "mapValues" when allocating containers. apache#16667
without adding LocalityPlacementStrategySuite.scala
1 parent 49b0a18 commit 3c6ba79

1 file changed

Lines changed: 6 additions & 5 deletions

File tree

yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -131,9 +131,9 @@ private[yarn] class LocalityPreferredContainerPlacementStrategy(
131131
val largestRatio = updatedHostToContainerCount.values.max
132132
// Round the ratio of preferred locality to the number of locality required container
133133
// number, which is used for locality preferred host calculating.
134-
var preferredLocalityRatio = updatedHostToContainerCount.mapValues { ratio =>
134+
var preferredLocalityRatio = updatedHostToContainerCount.map { case(k, ratio) =>
135135
val adjustedRatio = ratio.toDouble * requiredLocalityAwareContainerNum / largestRatio
136-
adjustedRatio.ceil.toInt
136+
(k, adjustedRatio.ceil.toInt)
137137
}
138138

139139
for (i <- 0 until requiredLocalityAwareContainerNum) {
@@ -147,7 +147,7 @@ private[yarn] class LocalityPreferredContainerPlacementStrategy(
147147

148148
// Minus 1 each time when the host is used. When the current ratio is 0,
149149
// which means all the required ratio is satisfied, this host will not be allocated again.
150-
preferredLocalityRatio = preferredLocalityRatio.mapValues(_ - 1)
150+
preferredLocalityRatio = preferredLocalityRatio.map { case (k, v) => (k, v - 1) }
151151
}
152152
}
153153

@@ -220,7 +220,8 @@ private[yarn] class LocalityPreferredContainerPlacementStrategy(
220220

221221
val possibleTotalContainerNum = pendingHostToContainerCount.values.sum
222222
val localityMatchedPendingNum = localityMatchedPendingAllocations.size.toDouble
223-
pendingHostToContainerCount.mapValues(_ * localityMatchedPendingNum / possibleTotalContainerNum)
224-
.toMap
223+
pendingHostToContainerCount.map { case (k, v) =>
224+
(k, v * localityMatchedPendingNum / possibleTotalContainerNum)
225+
}.toMap
225226
}
226227
}

0 commit comments

Comments
 (0)