From 811f28b73df4a6ead1ea15872da3138c0996ec15 Mon Sep 17 00:00:00 2001 From: Li Yichao Date: Sat, 27 May 2017 15:49:48 +0800 Subject: [PATCH 1/5] Remove LocalSchem when add path to ClassPath. --- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 9956071fd6e3..70cd7fe6947a 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -1275,7 +1275,8 @@ private object Client extends Logging { if (sparkConf.get(SPARK_ARCHIVE).isEmpty) { sparkConf.get(SPARK_JARS).foreach { jars => jars.filter(isLocalUri).foreach { jar => - addClasspathEntry(getClusterPath(sparkConf, jar), env) + val noLocal = jar.substring(LOCAL_SCHEME.length) + addClasspathEntry(getClusterPath(sparkConf, noLocal), env) } } } From 4ed085b98d0a393c4a98324c987fe259e667e891 Mon Sep 17 00:00:00 2001 From: Li Yichao Date: Sat, 27 May 2017 16:35:41 +0800 Subject: [PATCH 2/5] Add test for checking local is not in ClassPath. --- .../scala/org/apache/spark/deploy/yarn/Client.scala | 4 ++-- .../org/apache/spark/deploy/yarn/ClientSuite.scala | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 70cd7fe6947a..e2ab1819da21 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -1275,8 +1275,8 @@ private object Client extends Logging { if (sparkConf.get(SPARK_ARCHIVE).isEmpty) { sparkConf.get(SPARK_JARS).foreach { jars => jars.filter(isLocalUri).foreach { jar => - val noLocal = jar.substring(LOCAL_SCHEME.length) - addClasspathEntry(getClusterPath(sparkConf, noLocal), env) + val uri = new URI(jar) + addClasspathEntry(getClusterPath(sparkConf, uri.getPath), env) } } } diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index 3a11787aa57d..b043e091e073 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -116,15 +116,16 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll val cp = env("CLASSPATH").split(":|;|") s"$SPARK,$USER,$ADDED".split(",").foreach({ entry => val uri = new URI(entry) - if (LOCAL_SCHEME.equals(uri.getScheme())) { - cp should contain (uri.getPath()) + if (LOCAL_SCHEME.equals(uri.getScheme)) { + cp should contain (uri.getPath) } else { - cp should not contain (uri.getPath()) + cp should not contain uri.getPath } }) + cp should not contain "local" cp should contain(PWD) - cp should contain (s"$PWD${Path.SEPARATOR}${LOCALIZED_CONF_DIR}") - cp should not contain (APP_JAR) + cp should contain (s"$PWD${Path.SEPARATOR}$LOCALIZED_CONF_DIR") + cp should not contain APP_JAR } test("Jar path propagation through SparkConf") { From 79105f881cf0766e1e5a3712639b97af27d4b775 Mon Sep 17 00:00:00 2001 From: Li Yichao Date: Wed, 31 May 2017 17:23:51 +0800 Subject: [PATCH 3/5] Revert style changes. --- .../scala/org/apache/spark/deploy/yarn/ClientSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index b043e091e073..ffdc589e7d45 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -116,15 +116,15 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll val cp = env("CLASSPATH").split(":|;|") s"$SPARK,$USER,$ADDED".split(",").foreach({ entry => val uri = new URI(entry) - if (LOCAL_SCHEME.equals(uri.getScheme)) { - cp should contain (uri.getPath) + if (LOCAL_SCHEME.equals(uri.getScheme())) { + cp should contain (uri.getPath()) } else { - cp should not contain uri.getPath + cp should not contain uri.getPath() } }) cp should not contain "local" cp should contain(PWD) - cp should contain (s"$PWD${Path.SEPARATOR}$LOCALIZED_CONF_DIR") + cp should contain (s"$PWD${Path.SEPARATOR}${LOCALIZED_CONF_DIR}") cp should not contain APP_JAR } From 683f0d9ca96f67c0cc45299bb5695c685b6939a3 Mon Sep 17 00:00:00 2001 From: Li Yichao Date: Thu, 1 Jun 2017 11:28:46 +0800 Subject: [PATCH 4/5] Avoid noisy changes and keep the existing style. --- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index e2ab1819da21..1fb7edf2a6e3 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -1276,7 +1276,7 @@ private object Client extends Logging { sparkConf.get(SPARK_JARS).foreach { jars => jars.filter(isLocalUri).foreach { jar => val uri = new URI(jar) - addClasspathEntry(getClusterPath(sparkConf, uri.getPath), env) + addClasspathEntry(getClusterPath(sparkConf, uri.getPath()), env) } } } diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index ffdc589e7d45..1bd43096ca8d 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -119,13 +119,13 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll if (LOCAL_SCHEME.equals(uri.getScheme())) { cp should contain (uri.getPath()) } else { - cp should not contain uri.getPath() + cp should not contain (uri.getPath()) } }) cp should not contain "local" cp should contain(PWD) cp should contain (s"$PWD${Path.SEPARATOR}${LOCALIZED_CONF_DIR}") - cp should not contain APP_JAR + cp should not contain (APP_JAR) } test("Jar path propagation through SparkConf") { From 35e585974d8b49d1a092b53854f1b7fd5f30a319 Mon Sep 17 00:00:00 2001 From: Li Yichao Date: Thu, 1 Jun 2017 23:33:51 +0800 Subject: [PATCH 5/5] Fix style. --- .../test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index 1bd43096ca8d..6cf68427921f 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -122,7 +122,7 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll cp should not contain (uri.getPath()) } }) - cp should not contain "local" + cp should not contain ("local") cp should contain(PWD) cp should contain (s"$PWD${Path.SEPARATOR}${LOCALIZED_CONF_DIR}") cp should not contain (APP_JAR)