Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

git commit -m "filter imp data where count > 1"

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -76,23 +76,23 @@ object temp_analysis {
       agency_id.createOrReplaceTempView("agency_id")
 
 
-      //每日新用户
-      val device_id_newUser = sc.sql(
-        s"""
-           |select distinct(device_id) as device_id
-           |from online.ml_device_day_active_status
-           |where active_type != '4'
-           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
-           |        ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
-           |        ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
-           |        ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
-           |        ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
-           |        ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
-           |        ,'promotion_shike','promotion_julang_jl03')
-           |and partition_date ='${partition_date}'
-         """.stripMargin
-      )
-      device_id_newUser.createOrReplaceTempView("device_id_new")
+//      //每日新用户
+//      val device_id_newUser = sc.sql(
+//        s"""
+//           |select distinct(device_id) as device_id
+//           |from online.ml_device_day_active_status
+//           |where active_type != '4'
+//           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+//           |        ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+//           |        ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+//           |        ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+//           |        ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+//           |        ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+//           |        ,'promotion_shike','promotion_julang_jl03')
+//           |and partition_date ='${partition_date}'
+//         """.stripMargin
+//      )
+//      device_id_newUser.createOrReplaceTempView("device_id_new")
 
       val blacklist_id = sc.sql(
         s"""
@@ -108,16 +108,34 @@ object temp_analysis {
            |from agency_id
            |UNION ALL
            |select device_id
-           |from device_id_new
-           |UNION ALL
-           |select device_id
            |from blacklist_id
          """.stripMargin
       )
       final_id.createOrReplaceTempView("final_id")
 
+
+      val diary_clk_all = sc.sql(
+        s"""
+           |select ov.partition_date,count(ov.cl_id) as clk_num,count(distinct(ov.cl_id)),count(ov.cl_id)/count(distinct(ov.cl_id))
+           |from online.tl_hdfs_maidian_view ov left join final_id
+           |on ov.cl_id = final_id.device_id
+           |where ov.action = "page_view"
+           |and params['page_name']="diary_detail"
+           |and ov.cl_id != "NULL"
+           |and ov.partition_date >='20181201'
+           |and final_id.device_id is  null
+           |group by ov.partition_date
+           |order by ov.partition_date
+       """.stripMargin
+      )
+      diary_clk_all.show(80)
+
+
 //日记本点击
-      val referrer=List("all_case_service_comment","all_cases","diary_detail","diary_list","diary_listof_related_service",
+      val referrer=List("about_me_message_list","all_case_service_comment","all_cases","diary_detail","diary_list"
+        ,"diary_listof_related_service","answer_detail","community_home","conversation_detail","create_diary_title","diary_listof_related_service",
+        "doctor_all_cases","hospital_all_cases","my_favor","my_order","order_detail","personal_store_diary_list","received_votes",
+        "topic_detail","welfare_detail","welfare_list","welfare_special","wiki_detail","zone_detail",
       "expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail",
       "search_result_diary","search_result_more","welfare_detail","zone_v3")
       for( a <- referrer ){
@@ -130,7 +148,7 @@ object temp_analysis {
              |and params['page_name']="diary_detail"
              |and params['referrer']='${a}'
              |and ov.cl_id != "NULL"
-             |and ov.partition_date >='20181101'
+             |and ov.partition_date >='20181201'
              |and final_id.device_id is  null
              |group by ov.partition_date
              |order by ov.partition_date
@@ -141,6 +159,8 @@ object temp_analysis {
 
       }
 
+
+
       //5.登录人数
       val log_device_temp = sc.sql(
         s"""

eda/feededa/src/main/scala/com/gmei/testt.scala

@@ -399,3 +399,4 @@ object testt {
 
 
 }
+