Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

change test file

eda/feededa/src/main/resources/application.properties

@@ -10,7 +10,6 @@ pre.tidb.jdbcuri=jdbc:mysql://192.168.16.11:4000/eagle?user=root&password=&rewri
 pre.tispark.pd.addresses=192.168.16.11:2379
 pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas_prod?user=mimas&password=workwork&rewriteBatchedStatements=true
 
-<<<<<<< HEAD
 #prod.tidb.jdbcuri=jdbc:mysql://10.66.157.22:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
 #prod.gold.jdbcuri=jdbc:mysql://rm-m5e842126ng59jrv6.mysql.rds.aliyuncs.com/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
 #prod.mimas.jdbcuri=jdbc:mysql://rm-m5emg41za2w7l6au3.mysql.rds.aliyuncs.com/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
@@ -19,7 +18,6 @@ pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas
 #prod.redis.host=10.30.50.58
 #prod.redis.port=6379
 
-=======
 
 
 #阿里云线上配置
@@ -34,7 +32,6 @@ pre.mimas.jdbcuri=jdbc:mysql://rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com:3308/mimas
 #prod.jerry.jdbcuri_new=jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true
 
 #腾讯云线上配置
->>>>>>> 6bb8533b68efef7c647251ef08479560d5e1216a
 prod.gold.jdbcuri=jdbc:mysql://172.16.30.136/doris_prod?user=doris&password=o5gbA27hXHHm&rewriteBatchedStatements=true
 prod.mimas.jdbcuri=jdbc:mysql://172.16.30.138/mimas_prod?user=mimas&password=GJL3UJe1Ck9ggL6aKnZCq4cRvM&rewriteBatchedStatements=true
 prod.gaia.jdbcuri=jdbc:mysql://172.16.30.143/zhengxing?user=work&password=BJQaT9VzDcuPBqkd&rewriteBatchedStatements=true

eda/feededa/src/main/scala/com/gmei/data_feed_exposure_precise.scala

@@ -3,7 +3,7 @@ package com.gmei
 import java.io.Serializable
 
 import com.gmei.WeafareStat.{defaultParams, parser}
-import org.apache.spark.sql.{SaveMode}
+import org.apache.spark.sql.SaveMode
 import org.apache.log4j.{Level, Logger}
 import scopt.OptionParser
 import com.gmei.lib.AbstractParams
@@ -82,6 +82,7 @@ object data_feed_exposure_precise {
            |    and partition_date='${partition_date}'
        """.stripMargin
       )
+      result01.show()
 
       result01.createOrReplaceTempView("temp_result")
 
@@ -411,10 +412,12 @@ object data_feed_exposure_precise {
       )
 
 //      GmeiConfig.writeToJDBCTable(result, table = "data_feed_exposure_precise", SaveMode.Append)
+      print("写入开始")
       GmeiConfig.writeToJDBCTable("jdbc:mysql://152.136.44.138:4000/jerry_prod?user=root&password=3SYz54LS9#^9sBvC&rewriteBatchedStatements=true",result, table="data_feed_exposure_precise",SaveMode.Append)
+      print("写入结束")
 
 //      println("开始写入")
-//      GmeiConfig.writeToJDBCTable("jerry.jdbcuri",result, table="data_feed_exposure_precise",SaveMode.Append)
+//      GmeiConfig.writeToJDBCTable(config.getString("jerry.jdbcuri"),result, table="data_feed_exposure_precise",SaveMode.Append)
 //      println("写入完成")
 
     }

eda/feededa/src/main/scala/com/gmei/find_bug.scala

@@ -55,7 +55,7 @@ object find_bug {
 
       val stat_date = GmeiConfig.getMinusNDate(1)
 //      val stat_date = param.date
-      //println(param.date)
+      println(stat_date)
       val partition_date = stat_date.replace("-","")
       val decive_id_oldUser = sc.sql(
         s"""
@@ -72,13 +72,14 @@ object find_bug {
            |and partition_date ='${partition_date}'
          """.stripMargin
       )
+      decive_id_oldUser.show()
       decive_id_oldUser.createOrReplaceTempView("device_id_old")
 
 
       val clk_count_oldUser_Contrast = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_Contrast
-           |from data_feed_click jd inner join device_id_old
+           |from jerry_prod.data_feed_click jd inner join device_id_old
            |on jd.device_id = device_id_old.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
            |and jd.device_id regexp'1$$'
@@ -86,11 +87,13 @@ object find_bug {
            |and jd.stat_date ='${stat_date}'
          """.stripMargin
       )
+      clk_count_oldUser_Contrast.show()
+
 
       val imp_count_oldUser_Contrast = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_Contrast
-           |from data_feed_exposure je inner join device_id_old
+           |from jerry_prod.data_feed_exposure je inner join device_id_old
            |on je.device_id = device_id_old.device_id
            |where je.cid_type = 'diary'
            |and je.device_id regexp'1$$'
@@ -98,28 +101,32 @@ object find_bug {
            |and je.stat_date ='${stat_date}'
          """.stripMargin
       )
+      imp_count_oldUser_Contrast.show()
+
 
       val clk_count_oldUser_all = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as clk_count_oldUser_all
-           |from data_feed_click jd inner join device_id_old
+           |from jerry_prod.data_feed_click jd inner join device_id_old
            |on jd.device_id = device_id_old.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
            |and jd.device_id not in (select device_id from blacklist)
            |and jd.stat_date ='${stat_date}'
          """.stripMargin
       )
+      clk_count_oldUser_all.show()
 
       val imp_count_oldUser_all = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as imp_count_oldUser_all
-           |from data_feed_exposure je inner join device_id_old
+           |from jerry_prod.data_feed_exposure je inner join device_id_old
            |on je.device_id = device_id_old.device_id
            |where je.cid_type = 'diary'
            |and je.device_id not in (select device_id from blacklist)
            |and je.stat_date ='${stat_date}'
          """.stripMargin
       )
+      imp_count_oldUser_all.show()
 
 
       val result1 = clk_count_oldUser_Contrast.join(imp_count_oldUser_Contrast,"stat_date")
@@ -141,7 +148,7 @@ object find_bug {
       val clk_active_1 = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(jd.cid_id) as clk_active_1
-           |from data_feed_click jd inner join device_id_old
+           |from jerry_prod.data_feed_click jd inner join device_id_old
            |on jd.device_id = device_id_old.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
            |and jd.device_id regexp'1$$'
@@ -149,12 +156,13 @@ object find_bug {
            |and jd.stat_date ='${stat_date}'
          """.stripMargin
       )
+      clk_active_1.show()
 
       //device_id尾号1有点击用户日记本曝光数
       val imp_active_1 = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(je.cid_id) as imp_active_1
-           |from data_feed_exposure je inner join device_id_old
+           |from jerry_prod.data_feed_exposure je inner join device_id_old
            |on je.device_id = device_id_old.device_id
            |where je.cid_type = 'diary'
            |and je.device_id in (select distinct(device_id) from data_feed_click where device_id regexp '1$$' and stat_date = '${stat_date}')
@@ -162,12 +170,13 @@ object find_bug {
            |and je.stat_date ='${stat_date}'
          """.stripMargin
       )
+      imp_active_1.show()
 
       //device_id尾号1点击日记本用户数
       val clk_diary_device = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(distinct(jd.device_id)) as clk_diary_device
-           |from data_feed_click jd inner join device_id_old
+           |from jerry_prod.data_feed_click jd inner join device_id_old
            |on jd.device_id = device_id_old.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
            |and jd.device_id regexp'1$$'
@@ -175,24 +184,26 @@ object find_bug {
            |and jd.stat_date ='${stat_date}'
             """.stripMargin
       )
+      clk_diary_device.show()
 
       //所有有点击用户日记本点击数
       val clk_active_all = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(jd.cid_id) as clk_active_all
-           |from data_feed_click jd inner join device_id_old
+           |from jerry_prod.data_feed_click jd inner join device_id_old
            |on jd.device_id = device_id_old.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
            |and jd.device_id not in (select device_id from blacklist)
            |and jd.stat_date ='${stat_date}'
           """.stripMargin
       )
+      clk_active_all.show()
 
       //所有有点击用户日记本曝光数
       val imp_active_all = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(je.cid_id) as imp_active_all
-           |from data_feed_exposure je inner join device_id_old
+           |from jerry_prod.data_feed_exposure je inner join device_id_old
            |on je.device_id = device_id_old.device_id
            |where je.cid_type = 'diary'
            |and je.device_id in (select distinct(device_id) from data_feed_click where stat_date = '${stat_date}')
@@ -200,23 +211,27 @@ object find_bug {
            |and je.stat_date ='${stat_date}'
           """.stripMargin
       )
+      imp_active_all.show()
 
       //策略命中用户点击日记本用户数
       val clk_diary_device_cover = sc.sql(
         s"""
            |select '${stat_date}' as stat_date,count(distinct(device_id)) as clk_diary_device_cover
-           |from merge_queue_table
+           |from jerry_prod.merge_queue_table
            |where device_id in (select distinct(device_id) from data_feed_click where stat_date = '${stat_date}')
           """.stripMargin
       )
+      clk_diary_device_cover.show()
 
       //策略命中用户总数
       val device_all_cover = sc.sql(
         s"""
            |select '${stat_date}' as stat_date,count(distinct(device_id)) as device_all_cover
-           |from merge_queue_table
+           |from jerry_prod.merge_queue_table
           """.stripMargin
       )
+      device_all_cover.show()
+
 
       val result2 = clk_active_1.join(imp_active_1,"stat_date")
         .join(clk_active_all,"stat_date")
@@ -254,19 +269,20 @@ object find_bug {
       val clk_count_newUser_Contrast = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as clk_count_newUser_Contrast
-           |from data_feed_click jd inner join device_id_new
+           |from jerry_prod.data_feed_click jd inner join device_id_new
            |on jd.device_id = device_id_new.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
            |and jd.device_id regexp'1$$'
-           |and jd.device_id not in (select device_id from blacklist)
+           |and jd.device_id not in (select device_id from jerry_prod.blacklist)
            |and jd.stat_date ='${stat_date}'
          """.stripMargin
       )
+      clk_count_newUser_Contrast.show()
 
       val imp_count_newUser_Contrast = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as imp_count_newUser_Contrast
-           |from data_feed_exposure je inner join device_id_new
+           |from jerry_prod.data_feed_exposure je inner join device_id_new
            |on je.device_id = device_id_new.device_id
            |where je.cid_type = 'diary'
            |and je.device_id regexp'1$$'
@@ -274,28 +290,31 @@ object find_bug {
            |and je.stat_date ='${stat_date}'
          """.stripMargin
       )
+      imp_count_newUser_Contrast.show()
 
       val clk_count_newUser_all = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as clk_count_newUser_all
-           |from data_feed_click jd inner join device_id_new
+           |from jerry_prod.data_feed_click jd inner join device_id_new
            |on jd.device_id = device_id_new.device_id
            |where  (jd.cid_type = 'diary' or jd.cid_type = 'diary_video')
-           |and jd.device_id not in (select device_id from blacklist)
+           |and jd.device_id not in (select device_id from jerry_prod.blacklist)
            |and jd.stat_date ='${stat_date}'
          """.stripMargin
       )
+      clk_count_newUser_all.show()
 
       val imp_count_newUser_all = sc.sql(
         s"""
            |select '${stat_date}' as stat_date, count(cid_id) as imp_count_newUser_all
-           |from data_feed_exposure je inner join device_id_new
+           |from jerry_prod.data_feed_exposure je inner join device_id_new
            |on je.device_id = device_id_new.device_id
            |where je.cid_type = 'diary'
-           |and je.device_id not in (select device_id from blacklist)
+           |and je.device_id not in (select device_id from jerry_prod.blacklist)
            |and je.stat_date ='${stat_date}'
          """.stripMargin
       )
+      imp_count_newUser_all.show()
 
       val result3 = clk_count_newUser_Contrast.join(imp_count_newUser_Contrast,"stat_date")
         .join(clk_count_newUser_all,"stat_date")