Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

change test file

eda/feededa/src/main/scala/com/gmei/find_bug.scala

@@ -51,8 +51,8 @@ object find_bug {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-//      val stat_date = GmeiConfig.getMinusNDate(1)
-      val stat_date = param.date
+      val stat_date = GmeiConfig.getMinusNDate(1)
+//      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
       val decive_id_oldUser = sc.sql(
@@ -346,8 +346,8 @@ object CTR_precise {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-//      val stat_date = GmeiConfig.getMinusNDate(1)
-      val stat_date = param.date
+      val stat_date = GmeiConfig.getMinusNDate(1)
+//      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
       val decive_id_oldUser = sc.sql(

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -8,6 +8,8 @@ import org.apache.log4j.{Level, Logger}
 import scopt.OptionParser
 import com.gmei.lib.AbstractParams
 import java.io._
+import scala.util.parsing.json._
+
 
 object temp_analysis {
 
@@ -160,22 +162,6 @@ object temp_analysis {
       }
 
 
-
-      //5.登录人数
-      val log_device_temp = sc.sql(
-        s"""
-           |select oe.stat_date,count(distinct(oe.device_id)) as log_num
-           |from data_feed_exposure oe left join final_id
-           |on oe.device_id = final_id.device_id
-           |and oe.stat_date >='2018-11-01'
-           |and final_id.device_id is null
-           |group by oe.stat_date
-           |order by oe.stat_date
-         """.stripMargin
-      )
-      println("登录人数统计：")
-      log_device_temp.show(80)
-
     }
 
 
@@ -522,7 +508,7 @@ object meigou_xiaofei_renshu {
 
 
 
-object smart_rank_count {
+object alpha_ctr {
 
   Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
   Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
@@ -571,155 +557,54 @@ object smart_rank_count {
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
-      val agency_id = sc.sql(
-        s"""
-           |SELECT DISTINCT(cl_id) as device_id
-           |FROM online.ml_hospital_spam_pv_day
-           |WHERE partition_date >= '20180402'
-           |AND partition_date <= '${partition_date}'
-           |AND pv_ratio >= 0.95
-           |UNION ALL
-           |SELECT DISTINCT(cl_id) as device_id
-           |FROM online.ml_hospital_spam_pv_month
-           |WHERE partition_date >= '20171101'
-           |AND partition_date <= '${partition_date}'
-           |AND pv_ratio >= 0.95
-         """.stripMargin
-      )
-      agency_id.createOrReplaceTempView("agency_id")
-
-      val blacklist_id = sc.sql(
-        s"""
-           |SELECT device_id
-           |from blacklist
-         """.stripMargin
-      )
-      blacklist_id.createOrReplaceTempView("blacklist_id")
-
-      val final_id = sc.sql(
+      val click_count_recommend = sc.sql(
         s"""
-           |select device_id
-           |from agency_id
-           |UNION ALL
-           |select device_id
-           |from blacklist_id
-         """.stripMargin
-      )
-      final_id.createOrReplaceTempView("final_id")
-
-
-      val user_city_meigou_view = sc.sql(
-        s"""
-           |select ov.cl_id as device_id,ov.city_id as device_city,ov.params['business_id'] as meigou_id
-           |from online.tl_hdfs_maidian_view ov left join final_id
-           |on ov.cl_id = final_id.device_id
-           |where ov.action = "page_view"
-           |and ov.params['page_name']="welfare_detail"
-           |and ov.partition_date >='20181101'
-           |and ov.partition_date <'20181201'
-           |and ov.city_id is not null
-           |and final_id.device_id  is null
+           |select '${stat_date}' as stat_date,count(*) as count_recommend
+           |from bl.bl_alpha_et_mg_maidianlog_inc_d
+           |where params['tab_name']='recommend'
+           |and params['page_name']='home'
+           |and type='on_click_feed_topic_card'
+           |and partition_day='${partition_date}'
        """.stripMargin
       )
-      user_city_meigou_view.createOrReplaceTempView("user_city_meigou_view")
+      click_count_recommend.show()
 
-      val meigou_city = sc.sql(
+      val click_count_focus = sc.sql(
         s"""
-           |select b.id as meigou_id,d.city_id as meigou_city
-           |from online.tl_meigou_service_view b
-           |left join online.tl_hdfs_doctor_view c on b.doctor_id=c.id
-           |left join online.tl_hdfs_hospital_view d on c.hospital_id=d.id
-           |where b.partition_date='20181228'
-           |and c.partition_date='20181228'
-           |and d.partition_date='20181228'
+           |select '${stat_date}' as stat_date,count(*) as count_focus
+           |from bl.bl_alpha_et_mg_maidianlog_inc_d
+           |where params['tab_name']='focus'
+           |and params['page_name']='home'
+           |and type='on_click_feed_topic_card'
+           |and partition_day='${partition_date}'
        """.stripMargin
       )
-      meigou_city.createOrReplaceTempView("meigou_city")
-
+      click_count_focus.show()
 
-      val meigou_pv_tongcheng = sc.sql(
+      val expoure_count_recommend = sc.sql(
         s"""
-           |select a.device_id,a.device_city,a.meigou_id,b.meigou_city
-           |from user_city_meigou_view a
-           |left join meigou_city b
-           |on a.meigou_id = b.meigou_id
+           |select params['exposure_cards'] as exposure_cards
+           |from bl.bl_alpha_et_mg_maidianlog_inc_d
+           |where params['tab_name'] = 'recommend'
+           |and params['page_name'] = 'home'
+           |and type = 'page_precise_exposure'
+           |and partition_day='${partition_date}'
        """.stripMargin
       )
-      meigou_pv_tongcheng.createOrReplaceTempView("meigou_pv_tongcheng")
+      expoure_count_recommend.rdd.map(row =>{
+        val s = row.getAs[List[String]]("exposure_cards")
+        (stat_date,s)
+      })
+        //.reduceByKey((x,y) => x + y)
+        .toDF("stat_date","count").show()
 
-      val meigou_pv_count = sc.sql(
-        s"""
-           |select '2018-11' as stat_date,meigou_city,count(device_id) as meigou_pv,count(distinct(device_id)) as meigou_device_num
-           |from meigou_pv_tongcheng
-           |where device_city = meigou_city
-           |group by meigou_city
-       """.stripMargin
-      )
-      meigou_pv_count.createOrReplaceTempView("meigou_pv_count")
 
 
-//开始计算咨询
-      val zixun_meigou_view = sc.sql(
-        s"""
-           |select ov.cl_id as device_id,ov.city_id as device_city,ov.params['service_id'] as meigou_id
-           |from online.tl_hdfs_maidian_view ov left join final_id
-           |on ov.cl_id = final_id.device_id
-           |where ov.partition_date >= '20181101'
-           |and ov.partition_date < '20181201'
-           |and ov.action = 'welfare_detail_click_message'
-           |and final_id.device_id is null
-       """.stripMargin
-      )
-      zixun_meigou_view.createOrReplaceTempView("zixun_meigou_view")
-
-      val zixun_meigou_tongcheng = sc.sql(
-        s"""
-           |select a.device_id,a.device_city,a.meigou_id,b.meigou_city
-           |from zixun_meigou_view a
-           |left join meigou_city b
-           |on a.meigou_id=b.meigou_id
-       """.stripMargin
-      )
-      zixun_meigou_tongcheng.createOrReplaceTempView("zixun_meigou_tongcheng")
-
-      val zixun_pv_count = sc.sql(
-        s"""
-           |select '2018-11' as stat_date,meigou_city,count(device_id) as meigou_zixun,count(distinct(device_id)) as meigou_zixun_device_num
-           |from zixun_meigou_tongcheng
-           |where device_city=meigou_city
-           |group by meigou_city
-       """.stripMargin
-      )
-      zixun_pv_count.createOrReplaceTempView("zixun_pv_count")
 
 
-      //开始计算每个地区每月新增设备
 
-      val device_new_count = sc.sql(
-        s"""
-           |select first_city,count(distinct(device_id)) as new_device_month
-           |from online.ml_device_day_active_status
-           |where active_type != '4'
-           |and partition_date >='20181101'
-           |and partition_date <'20181201'
-           |group by first_city
-       """.stripMargin
-      )
-      device_new_count.createOrReplaceTempView("device_new_count")
-
-
-//将所有的数据综合一起
-      val all_count = sc.sql(
-        s"""
-           |select mc.stat_date,mc.meigou_city,mc.meigou_pv,mc.meigou_device_num,zc.meigou_zixun,zc.meigou_zixun_device_num,dc.new_device_month
-           |from meigou_pv_count mc
-           |left join zixun_pv_count zc on mc.meigou_city = zc.meigou_city
-           |left join device_new_count dc on dc.first_city=mc.meigou_city
-       """.stripMargin
-      )
-      all_count.show()
 
-      GmeiConfig.writeToJDBCTable(all_count, "smart_rank_count", SaveMode.Append)
+//      GmeiConfig.writeToJDBCTable(click_count_recommend, "smart_rank_count", SaveMode.Append)
 
 
     }

eda/feededa/src/main/scala/com/gmei/temp_count.scala

@@ -515,8 +515,8 @@ object GetHiveSearchData {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-      val stat_date = GmeiConfig.getMinusNDate(1)
-//      val stat_date = param.date
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date = param.date
       val partition_date = stat_date.replace("-","")