Commit fccb3d72 authored by 高雅喆's avatar 高雅喆

add hive support

parent d6967541
...@@ -48,6 +48,7 @@ object GmeiConfig extends Serializable { ...@@ -48,6 +48,7 @@ object GmeiConfig extends Serializable {
.builder() .builder()
.config(sparkConf) .config(sparkConf)
.appName("feededa") .appName("feededa")
.enableHiveSupport()
.getOrCreate() .getOrCreate()
val context = SparkContext.getOrCreate(sparkConf) val context = SparkContext.getOrCreate(sparkConf)
......
...@@ -81,7 +81,9 @@ object WeafareStat { ...@@ -81,7 +81,9 @@ object WeafareStat {
) )
video_count.show() video_count.show()
val output1 = "./test_vd_cids.csv" val output1 = "./test_vd_cids.csv"
video_count.repartition(1).write.format("com.databricks.spark.csv").option("header","true").save(output1) video_count.repartition(1)
.write.format("com.databricks.spark.csv")
.option("header","true").save(output1)
val txt_count = sc.sql( val txt_count = sc.sql(
s""" s"""
...@@ -95,7 +97,9 @@ object WeafareStat { ...@@ -95,7 +97,9 @@ object WeafareStat {
) )
txt_count.show() txt_count.show()
val output2 = "./test_txt_cids.csv" val output2 = "./test_txt_cids.csv"
txt_count.repartition(1).write.format("com.databricks.spark.csv").option("header","true").save(output2) txt_count.repartition(1)
.write.format("com.databricks.spark.csv")
.option("header","true").save(output2)
sc.stop() sc.stop()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment