Skip to content

Commit 3ee5c17

Browse files
gvramanaJoshRosen
authored andcommitted
[SPARK-4504][Examples] fix run-example failure if multiple assembly jars exist
Fix run-example script to fail fast with useful error message if multiple example assembly JARs are present. Author: Venkata Ramana Gollamudi <ramana.gollamudi@huawei.com> Closes #3377 from gvramana/run-example_fails and squashes the following commits: fa7f481 [Venkata Ramana Gollamudi] Fixed review comments, avoiding ls output scanning. 6aa1ab7 [Venkata Ramana Gollamudi] Fix run-examples script error during multiple jars (cherry picked from commit 74de94e) Signed-off-by: Josh Rosen <joshrosen@databricks.com> Conflicts: bin/compute-classpath.sh bin/run-example
1 parent 68df734 commit 3ee5c17

2 files changed

Lines changed: 36 additions & 18 deletions

File tree

bin/compute-classpath.sh

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,22 +69,25 @@ else
6969
assembly_folder="$ASSEMBLY_DIR"
7070
fi
7171

72-
num_jars=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar" | wc -l)
73-
if [ "$num_jars" -eq "0" ]; then
74-
echo "Failed to find Spark assembly in $assembly_folder"
75-
echo "You need to build Spark before running this program."
76-
exit 1
77-
fi
72+
num_jars=0
73+
74+
for f in ${assembly_folder}/spark-assembly*hadoop*.jar; do
75+
if [[ ! -e "$f" ]]; then
76+
echo "Failed to find Spark assembly in $assembly_folder" 1>&2
77+
echo "You need to build Spark before running this program." 1>&2
78+
exit 1
79+
fi
80+
ASSEMBLY_JAR="$f"
81+
num_jars=$((num_jars+1))
82+
done
83+
7884
if [ "$num_jars" -gt "1" ]; then
79-
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar")
80-
echo "Found multiple Spark assembly jars in $assembly_folder:"
81-
echo "$jars_list"
82-
echo "Please remove all but one jar."
85+
echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2
86+
ls ${assembly_folder}/spark-assembly*hadoop*.jar 1>&2
87+
echo "Please remove all but one jar." 1>&2
8388
exit 1
8489
fi
8590

86-
ASSEMBLY_JAR=$(ls "$assembly_folder"/spark-assembly*hadoop*.jar 2>/dev/null)
87-
8891
# Verify that versions of java used to build the jars and run Spark are compatible
8992
jar_error_check=$("$JAR_CMD" -tf "$ASSEMBLY_JAR" nonexistent/class/path 2>&1)
9093
if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then

bin/run-example

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,32 @@ else
3535
fi
3636

3737
if [ -f "$FWDIR/RELEASE" ]; then
38-
export SPARK_EXAMPLES_JAR=`ls "$FWDIR"/lib/spark-examples-*hadoop*.jar`
39-
elif [ -e "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar ]; then
40-
export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR"/target/scala-$SCALA_VERSION/spark-examples-*hadoop*.jar`
38+
JAR_PATH="${FWDIR}/lib"
39+
else
40+
JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}"
4141
fi
4242

43-
if [[ -z $SPARK_EXAMPLES_JAR ]]; then
44-
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
45-
echo "You need to build Spark before running this program" 1>&2
43+
JAR_COUNT=0
44+
45+
for f in ${JAR_PATH}/spark-examples-*hadoop*.jar; do
46+
if [[ ! -e "$f" ]]; then
47+
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
48+
echo "You need to build Spark before running this program" 1>&2
49+
exit 1
50+
fi
51+
SPARK_EXAMPLES_JAR="$f"
52+
JAR_COUNT=$((JAR_COUNT+1))
53+
done
54+
55+
if [ "$JAR_COUNT" -gt "1" ]; then
56+
echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2
57+
ls ${JAR_PATH}/spark-examples-*hadoop*.jar 1>&2
58+
echo "Please remove all but one jar." 1>&2
4659
exit 1
4760
fi
4861

62+
export SPARK_EXAMPLES_JAR
63+
4964
EXAMPLE_MASTER=${MASTER:-"local[*]"}
5065

5166
if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then

0 commit comments

Comments
 (0)