Skip to content

Commit 7514aee

Browse files
Baohe ZhangDhruve Ashar
authored andcommitted
Update hive example to avoid conflict (apache#38)
1 parent b62ce89 commit 7514aee

2 files changed

Lines changed: 7 additions & 5 deletions

File tree

src/main/resources/oozie/spark_hive/workflow.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
<spark-opts>--num-executors 3 --executor-memory 2g --executor-cores 1 --queue default --conf spark.yarn.security.tokens.hive.enabled=false</spark-opts>
3737
<arg>hdfs:///user/${wf:conf('user.name')}/${wfRoot}/output/hive_location</arg>
3838
<arg>hdfs:///user/${wf:conf('user.name')}/${wfRoot}/output/kv1.txt</arg>
39+
<arg>${sparkTag}_hive_test</arg>
3940
</spark>
4041
<ok to="end" />
4142
<error to="fail" />

src/main/scala/com/yahoo/spark/starter/SparkHiveExample.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,14 @@ case class Record(key: Int, value: String)
99
object SparkHiveExample {
1010
def main(args: Array[String]) {
1111

12-
if (args.length < 2) {
13-
System.err.println("Usage: SparkHiveExample <databaseDir> <inputFile>")
12+
if (args.length < 3) {
13+
System.err.println("Usage: SparkHiveExample <databaseDir> <inputFile> <database>")
1414
System.exit(1)
1515
}
1616

1717
val databaseDir = args(0)
1818
val inputFile = args(1)
19+
val database = args(2)
1920

2021
val spark = SparkSession
2122
.builder()
@@ -27,9 +28,9 @@ object SparkHiveExample {
2728
import spark.sql
2829

2930
// Create a database under hdfs home directory for testing purpose
30-
sql(s"CREATE DATABASE IF NOT EXISTS spark_hive_test LOCATION '$databaseDir'")
31+
sql(s"CREATE DATABASE IF NOT EXISTS $database LOCATION '$databaseDir'")
3132

32-
sql("USE spark_hive_test")
33+
sql(s"USE $database")
3334

3435
// Create the table and load data from a text file on hdfs
3536
sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING) USING hive")
@@ -113,7 +114,7 @@ object SparkHiveExample {
113114
// ...
114115

115116
// Drop the database
116-
sql("DROP DATABASE IF EXISTS spark_hive_test CASCADE")
117+
sql(s"DROP DATABASE IF EXISTS $database CASCADE")
117118

118119
spark.stop()
119120
}

0 commit comments

Comments
 (0)