File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change 7272 assembly_folder=" $ASSEMBLY_DIR "
7373fi
7474
75- num_jars=" $( ls " $assembly_folder " | grep " spark-assembly.*hadoop.*\.jar$" | wc -l) "
76- if [ " $num_jars " -eq " 0" ]; then
77- echo " Failed to find Spark assembly in $assembly_folder "
78- echo " You need to build Spark before running this program."
79- exit 1
80- fi
75+ num_jars=0
76+
77+ for f in ${assembly_folder} /spark-assembly* hadoop* .jar; do
78+ if [[ ! -e " $f " ]]; then
79+ echo " Failed to find Spark assembly in $assembly_folder " 1>&2
80+ echo " You need to build Spark before running this program." 1>&2
81+ exit 1
82+ fi
83+ ASSEMBLY_JAR=" $f "
84+ num_jars=$(( num_jars+ 1 ))
85+ done
86+
8187if [ " $num_jars " -gt " 1" ]; then
82- jars_list=$( ls " $assembly_folder " | grep " spark-assembly.*hadoop.*.jar$" )
83- echo " Found multiple Spark assembly jars in $assembly_folder :"
84- echo " $jars_list "
85- echo " Please remove all but one jar."
88+ echo " Found multiple Spark assembly jars in $assembly_folder :" 1>&2
89+ ls ${assembly_folder} /spark-assembly* hadoop* .jar 1>&2
90+ echo " Please remove all but one jar." 1>&2
8691 exit 1
8792fi
8893
89- ASSEMBLY_JAR=" $( ls " $assembly_folder " /spark-assembly* hadoop* .jar 2> /dev/null) "
90-
9194# Verify that versions of java used to build the jars and run Spark are compatible
9295jar_error_check=$( " $JAR_CMD " -tf " $ASSEMBLY_JAR " nonexistent/class/path 2>&1 )
9396if [[ " $jar_error_check " =~ " invalid CEN header" ]]; then
Original file line number Diff line number Diff line change 4040 JAR_PATH=" ${EXAMPLES_DIR} /target/scala-${SPARK_SCALA_VERSION} "
4141fi
4242
43- JAR_COUNT=" ` ls ${JAR_PATH} /spark-examples- * hadoop * .jar 2>> /dev/null | wc -l ` "
43+ JAR_COUNT=0
4444
45- if [ " $JAR_COUNT " -eq " 0" ]; then
46- echo " Failed to find Spark examples assembly in $FWDIR /lib or $FWDIR /examples/target" 1>&2
47- echo " You need to build Spark before running this program" 1>&2
48- exit 1
49- fi
45+ for f in ${JAR_PATH} /spark-examples-* hadoop* .jar; do
46+ if [[ ! -e " $f " ]]; then
47+ echo " Failed to find Spark examples assembly in $FWDIR /lib or $FWDIR /examples/target" 1>&2
48+ echo " You need to build Spark before running this program" 1>&2
49+ exit 1
50+ fi
51+ SPARK_EXAMPLES_JAR=" $f "
52+ JAR_COUNT=$(( JAR_COUNT+ 1 ))
53+ done
5054
5155if [ " $JAR_COUNT " -gt " 1" ]; then
52- JARS_LIST=" ` ls -t ${JAR_PATH} /spark-examples-* hadoop* .jar` "
5356 echo " Found multiple Spark examples assembly jars in ${JAR_PATH} " 1>&2
54- echo " $JARS_LIST " 1>&2
57+ ls ${JAR_PATH} /spark-examples- * hadoop * .jar 1>&2
5558 echo " Please remove all but one jar." 1>&2
5659 exit 1
5760fi
5861
59- export SPARK_EXAMPLES_JAR= " ` ls ${JAR_PATH} /spark-examples- * hadoop * .jar 2>> /dev/null ` "
62+ export SPARK_EXAMPLES_JAR
6063
6164EXAMPLE_MASTER=${MASTER:- " local[*]" }
6265
You can’t perform that action at this time.
0 commit comments