diff --git a/assembly/pom.xml b/assembly/pom.xml index d731003c0af29..09e8020bda4af 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/bagel/pom.xml b/bagel/pom.xml index 83746120a999c..fd26ddd55234c 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index ceeabd6cbd299..4d4bdc228c15d 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index aeb15adb9a34d..450c1d596d10a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -640,7 +640,7 @@ private[spark] class Master( def registerApplication(app: ApplicationInfo): Unit = { val appAddress = app.driver.path.address - if (addressToWorker.contains(appAddress)) { + if (addressToApp.contains(appAddress)) { logInfo("Attempted to re-register application at same address: " + appAddress) return } diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index 9f54312074856..747ecf075a397 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -42,9 +42,6 @@ private[spark] trait Spillable[C] extends Logging { // Memory manager that can be used to acquire/release memory private[this] val shuffleMemoryManager = SparkEnv.get.shuffleMemoryManager - // Threshold for `elementsRead` before we start tracking this collection's memory usage - private[this] val trackMemoryThreshold = 1000 - // Initial threshold for the size of a collection before we start tracking its memory usage // Exposed for testing private[this] val initialMemoryThreshold: Long = @@ -72,8 +69,7 @@ private[spark] trait Spillable[C] extends Logging { * @return true if `collection` was spilled to disk; false otherwise */ protected def maybeSpill(collection: C, currentMemory: Long): Boolean = { - if (elementsRead > trackMemoryThreshold && elementsRead % 32 == 0 && - currentMemory >= myMemoryThreshold) { + if (elementsRead % 32 == 0 && currentMemory >= myMemoryThreshold) { // Claim up to double our current memory from the shuffle memory pool val amountToRequest = 2 * currentMemory - myMemoryThreshold val granted = shuffleMemoryManager.tryToAcquire(amountToRequest) diff --git a/examples/pom.xml b/examples/pom.xml index c7ad4dc2b3e26..6005c1341fc06 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index e0b3eaea41d20..253eff5a34e1b 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index f9559c1fe5663..f9bac309c8c5e 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index e2bb69d307f2c..ba9ee1e165887 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index 3a04c23487204..8c3b53b83a322 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index b7003cfd8a9c1..fd6ff1eb5dd90 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index d3ea28ce19c15..da8c215c39aae 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index bd4f6f558c2a6..bef1ceead4538 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index 5fc5bc2899b23..d0061cd9dccfc 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml index e51ff8123ab97..d90a8f5e22d1f 100644 --- a/extras/spark-ganglia-lgpl/pom.xml +++ b/extras/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index a4a9ef815e197..411710a4c0aff 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index eaaafdc3d9cbb..5ce21df761771 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/network/common/pom.xml b/network/common/pom.xml index c8ee0a6a02f55..852f60784a4be 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index 7d624a4bc95b1..18c0e7554e5ca 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 5f93edb2d0329..592c8ef877a5c 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/pom.xml b/pom.xml index f406b2d636bdb..4328e4a75a131 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 3831dfa649aa8..760788ba6cf74 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 347fe378552d8..b2ab1ec2f1474 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 8ab77087359e6..f3cceaec1e858 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 15a5aa1bbc7de..4231f83381132 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 177d6eb8c927a..506136593c65d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 554a903e0d529..025bc2ed4947b 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 07610409e4d01..203cc03fa7125 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml index 531c9a37a6365..32213a95a3dbc 100644 --- a/yarn/alpha/pom.xml +++ b/yarn/alpha/pom.xml @@ -20,7 +20,7 @@ org.apache.spark yarn-parent_2.10 - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index a36fe590a6978..3b4781212d83e 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent - 1.2.1 + 1.2.1-palantir1 ../pom.xml diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml index db181d294127d..68bfff2664acc 100644 --- a/yarn/stable/pom.xml +++ b/yarn/stable/pom.xml @@ -20,7 +20,7 @@ org.apache.spark yarn-parent_2.10 - 1.2.1 + 1.2.1-palantir1 ../pom.xml