[CARMEL-3499] Change vCores to 1 in ContainerRequest for hadoop limit (#17)

* [CARMEL-3499] Change vCores to 1 in ContainerRequest for hadoop limit

* change the default value of DEFAULT_RESOURCE_CALCULATOR to false

Author: Luan, Xuedong

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala

@@ -406,6 +406,9 @@ private[yarn] class YarnAllocator(
       resource: Resource,
       nodes: Array[String],
       racks: Array[String]): ContainerRequest = {
+    if (sparkConf.get(DEFAULT_RESOURCE_CALCULATOR)) {
+      resource.setVirtualCores(1)
+    }
     new ContainerRequest(resource, nodes, racks, RM_REQUEST_PRIORITY, true, labelExpression.orNull)
   }
 

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala

@@ -178,6 +178,13 @@ package object config {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1s")
 
+  private[spark] val DEFAULT_RESOURCE_CALCULATOR =
+    ConfigBuilder("spark.yarn.scheduler.default.resource.calculator")
+      .doc("After enable default resource calculator, the value of vCores " +
+        "in ContainerRequest is always 1.")
+      .booleanConf
+      .createWithDefault(false)
+
   /* Shared Client-mode AM / Driver configuration. */
 
   private[spark] val AM_MAX_WAIT_TIME = ConfigBuilder("spark.yarn.am.waitTime")