File tree Expand file tree Collapse file tree
resources/oozie/spark_hive
scala/com/yahoo/spark/starter Expand file tree Collapse file tree Original file line number Diff line number Diff line change 3636 <spark-opts >--num-executors 3 --executor-memory 2g --executor-cores 1 --queue default --conf spark.yarn.security.tokens.hive.enabled=false</spark-opts >
3737 <arg >hdfs:///user/${wf:conf('user.name')}/${wfRoot}/output/hive_location</arg >
3838 <arg >hdfs:///user/${wf:conf('user.name')}/${wfRoot}/output/kv1.txt</arg >
39+ <arg >${sparkTag}_hive_test</arg >
3940 </spark >
4041 <ok to =" end" />
4142 <error to =" fail" />
Original file line number Diff line number Diff line change @@ -9,13 +9,14 @@ case class Record(key: Int, value: String)
99object SparkHiveExample {
1010 def main (args : Array [String ]) {
1111
12- if (args.length < 2 ) {
13- System .err.println(" Usage: SparkHiveExample <databaseDir> <inputFile>" )
12+ if (args.length < 3 ) {
13+ System .err.println(" Usage: SparkHiveExample <databaseDir> <inputFile> <database> " )
1414 System .exit(1 )
1515 }
1616
1717 val databaseDir = args(0 )
1818 val inputFile = args(1 )
19+ val database = args(2 )
1920
2021 val spark = SparkSession
2122 .builder()
@@ -27,9 +28,9 @@ object SparkHiveExample {
2728 import spark .sql
2829
2930 // Create a database under hdfs home directory for testing purpose
30- sql(s " CREATE DATABASE IF NOT EXISTS spark_hive_test LOCATION ' $databaseDir' " )
31+ sql(s " CREATE DATABASE IF NOT EXISTS $database LOCATION ' $databaseDir' " )
3132
32- sql(" USE spark_hive_test " )
33+ sql(s " USE $database " )
3334
3435 // Create the table and load data from a text file on hdfs
3536 sql(" CREATE TABLE IF NOT EXISTS src (key INT, value STRING) USING hive" )
@@ -113,7 +114,7 @@ object SparkHiveExample {
113114 // ...
114115
115116 // Drop the database
116- sql(" DROP DATABASE IF EXISTS spark_hive_test CASCADE" )
117+ sql(s " DROP DATABASE IF EXISTS $database CASCADE " )
117118
118119 spark.stop()
119120 }
You can’t perform that action at this time.
0 commit comments