补充数据

eda/feededa/src/main/scala/com/gmei/find_bug.scala

@@ -51,8 +51,8 @@ object find_bug {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-//      val stat_date = GmeiConfig.getMinusNDate(1)
-      val stat_date = param.date
+      val stat_date = GmeiConfig.getMinusNDate(1)
+//      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
       val decive_id_oldUser = sc.sql(
@@ -346,8 +346,8 @@ object CTR_precise {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "merge_queue_table")
 
 
-//      val stat_date = GmeiConfig.getMinusNDate(1)
-      val stat_date = param.date
+      val stat_date = GmeiConfig.getMinusNDate(1)
+//      val stat_date = param.date
       //println(param.date)
       val partition_date = stat_date.replace("-","")
       val decive_id_oldUser = sc.sql(

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -160,22 +160,6 @@ object temp_analysis {
       }
 
 
-
-      //5.登录人数
-      val log_device_temp = sc.sql(
-        s"""
-           |select oe.stat_date,count(distinct(oe.device_id)) as log_num
-           |from data_feed_exposure oe left join final_id
-           |on oe.device_id = final_id.device_id
-           |and oe.stat_date >='2018-11-01'
-           |and final_id.device_id is null
-           |group by oe.stat_date
-           |order by oe.stat_date
-         """.stripMargin
-      )
-      println("登录人数统计：")
-      log_device_temp.show(80)
-
     }
 
 
@@ -522,7 +506,7 @@ object meigou_xiaofei_renshu {
 
 
 
-object smart_rank_count {
+object alpha_ctr {
 
   Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
   Logger.getLogger("org.apache.eclipse.jetty.server").setLevel(Level.OFF)
@@ -571,145 +555,7 @@ object smart_rank_count {
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
-      val agency_id = sc.sql(
-        s"""
-           |SELECT DISTINCT(cl_id) as device_id
-           |FROM online.ml_hospital_spam_pv_day
-           |WHERE partition_date >= '20180402'
-           |AND partition_date <= '${partition_date}'
-           |AND pv_ratio >= 0.95
-           |UNION ALL
-           |SELECT DISTINCT(cl_id) as device_id
-           |FROM online.ml_hospital_spam_pv_month
-           |WHERE partition_date >= '20171101'
-           |AND partition_date <= '${partition_date}'
-           |AND pv_ratio >= 0.95
-         """.stripMargin
-      )
-      agency_id.createOrReplaceTempView("agency_id")
-
-      val blacklist_id = sc.sql(
-        s"""
-           |SELECT device_id
-           |from blacklist
-         """.stripMargin
-      )
-      blacklist_id.createOrReplaceTempView("blacklist_id")
-
-      val final_id = sc.sql(
-        s"""
-           |select device_id
-           |from agency_id
-           |UNION ALL
-           |select device_id
-           |from blacklist_id
-         """.stripMargin
-      )
-      final_id.createOrReplaceTempView("final_id")
-
-
-      val user_city_meigou_view = sc.sql(
-        s"""
-           |select ov.cl_id as device_id,ov.city_id as device_city,ov.params['business_id'] as meigou_id
-           |from online.tl_hdfs_maidian_view ov left join final_id
-           |on ov.cl_id = final_id.device_id
-           |where ov.action = "page_view"
-           |and ov.params['page_name']="welfare_detail"
-           |and ov.partition_date >='20181101'
-           |and ov.partition_date <'20181201'
-           |and ov.city_id is not null
-           |and final_id.device_id  is null
-       """.stripMargin
-      )
-      user_city_meigou_view.createOrReplaceTempView("user_city_meigou_view")
-
-      val meigou_city = sc.sql(
-        s"""
-           |select b.id as meigou_id,d.city_id as meigou_city
-           |from online.tl_meigou_service_view b
-           |left join online.tl_hdfs_doctor_view c on b.doctor_id=c.id
-           |left join online.tl_hdfs_hospital_view d on c.hospital_id=d.id
-           |where b.partition_date='20181228'
-           |and c.partition_date='20181228'
-           |and d.partition_date='20181228'
-       """.stripMargin
-      )
-      meigou_city.createOrReplaceTempView("meigou_city")
-
-
-      val meigou_pv_tongcheng = sc.sql(
-        s"""
-           |select a.device_id,a.device_city,a.meigou_id,b.meigou_city
-           |from user_city_meigou_view a
-           |left join meigou_city b
-           |on a.meigou_id = b.meigou_id
-       """.stripMargin
-      )
-      meigou_pv_tongcheng.createOrReplaceTempView("meigou_pv_tongcheng")
-
-      val meigou_pv_count = sc.sql(
-        s"""
-           |select '2018-11' as stat_date,meigou_city,count(device_id) as meigou_pv,count(distinct(device_id)) as meigou_device_num
-           |from meigou_pv_tongcheng
-           |where device_city = meigou_city
-           |group by meigou_city
-       """.stripMargin
-      )
-      meigou_pv_count.createOrReplaceTempView("meigou_pv_count")
-
-
-//开始计算咨询
-      val zixun_meigou_view = sc.sql(
-        s"""
-           |select ov.cl_id as device_id,ov.city_id as device_city,ov.params['service_id'] as meigou_id
-           |from online.tl_hdfs_maidian_view ov left join final_id
-           |on ov.cl_id = final_id.device_id
-           |where ov.partition_date >= '20181101'
-           |and ov.partition_date < '20181201'
-           |and ov.action = 'welfare_detail_click_message'
-           |and final_id.device_id is null
-       """.stripMargin
-      )
-      zixun_meigou_view.createOrReplaceTempView("zixun_meigou_view")
-
-      val zixun_meigou_tongcheng = sc.sql(
-        s"""
-           |select a.device_id,a.device_city,a.meigou_id,b.meigou_city
-           |from zixun_meigou_view a
-           |left join meigou_city b
-           |on a.meigou_id=b.meigou_id
-       """.stripMargin
-      )
-      zixun_meigou_tongcheng.createOrReplaceTempView("zixun_meigou_tongcheng")
-
-      val zixun_pv_count = sc.sql(
-        s"""
-           |select '2018-11' as stat_date,meigou_city,count(device_id) as meigou_zixun,count(distinct(device_id)) as meigou_zixun_device_num
-           |from zixun_meigou_tongcheng
-           |where device_city=meigou_city
-           |group by meigou_city
-       """.stripMargin
-      )
-      zixun_pv_count.createOrReplaceTempView("zixun_pv_count")
-
-
-      //开始计算每个地区每月新增设备
-
-      val device_new_count = sc.sql(
-        s"""
-           |select first_city,count(distinct(device_id)) as new_device_month
-           |from online.ml_device_day_active_status
-           |where active_type != '4'
-           |and partition_date >='20181101'
-           |and partition_date <'20181201'
-           |group by first_city
-       """.stripMargin
-      )
-      device_new_count.createOrReplaceTempView("device_new_count")
-
-
-//将所有的数据综合一起
-      val all_count = sc.sql(
+      val click_count_recommend = sc.sql(
         s"""
            |select mc.stat_date,mc.meigou_city,mc.meigou_pv,mc.meigou_device_num,zc.meigou_zixun,zc.meigou_zixun_device_num,dc.new_device_month
            |from meigou_pv_count mc

eda/feededa/src/main/scala/com/gmei/testt.scala

@@ -454,8 +454,8 @@ object diary_clk_card {
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure")
       ti.tidbMapTable(dbName = "jerry_prod", tableName = "data_feed_exposure_precise")
 
-      val stat_date = GmeiConfig.getMinusNDate(1)
-//      val stat_date=param.date
+//      val stat_date = GmeiConfig.getMinusNDate(1)
+      val stat_date=param.date
       val partition_date = stat_date.replace("-","")
       //机构id
       val blacklist = sc.sql(