Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

生成spark时增加架包

eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala

@@ -51,11 +51,16 @@ object data_feed_exposure_precise {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-      val stat_date = GmeiConfig.getMinusNDate(1)
-//      val stat_date = param.date
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
+      sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/brickhouse-0.7.1-SNAPSHOT.jar")
+      sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar")
+      sc.sql("CREATE TEMPORARY FUNCTION json_map AS 'brickhouse.udf.json.JsonMapUDF'")
+      sc.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJsonFormatCheck'")
+
       val result01=sc.sql(
         s"""
            |select

eda/feededa/src/main/scala/com/gmei/testt.scala

@@ -61,6 +61,12 @@ object testt {
       )
       blacklist.createOrReplaceTempView("blacklist")
 
+      sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/brickhouse-0.7.1-SNAPSHOT.jar")
+      sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar")
+      sc.sql("CREATE TEMPORARY FUNCTION json_map AS 'brickhouse.udf.json.JsonMapUDF'")
+      sc.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJsonFormatCheck'")
+
+
       val agency_id = sc.sql(
         s"""
            |SELECT DISTINCT(cl_id) as device_id
@@ -454,10 +460,16 @@ object diary_clk_card {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure_precise")
 
-      val stat_date = GmeiConfig.getMinusNDate(1)
-//      val stat_date=param.date
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date=param.date
       val partition_date = stat_date.replace("-","")
       //机构id
+
+      sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/brickhouse-0.7.1-SNAPSHOT.jar")
+      sc.sql("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar")
+      sc.sql("CREATE TEMPORARY FUNCTION json_map AS 'brickhouse.udf.json.JsonMapUDF'")
+      sc.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJsonFormatCheck'")
+
       val blacklist = sc.sql(
         s"""
            |select device_id from blacklist