统计日记本点击的各个来源数量

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -58,46 +58,107 @@ object temp_analysis {
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
-      //获取策略命中用户device_id
-      val device_id_cover = sc.sql(
+      val agency_id = sc.sql(
         s"""
-           |select distinct(device_id) as device_id
-           |from merge_queue_table
+           |SELECT DISTINCT(cl_id) as device_id
+           |FROM online.ml_hospital_spam_pv_day
+           |WHERE partition_date >= '20180402'
+           |AND partition_date <= '20181203'
+           |AND pv_ratio >= 0.95
+           |UNION ALL
+           |SELECT DISTINCT(cl_id) as device_id
+           |FROM online.ml_hospital_spam_pv_month
+           |WHERE partition_date >= '20171101'
+           |AND partition_date <= '20181203'
+           |AND pv_ratio >= 0.95
          """.stripMargin
       )
-      device_id_cover.createOrReplaceTempView("device_id_cover_older")
+      agency_id.show()
+      agency_id.createOrReplaceTempView("agency_id")
 
 
-      val diary_pv = sc.sql(
+      //每日新用户
+      val device_id_newUser = sc.sql(
         s"""
-           |select '${stat_date}' as stat_date,count(params["business_id"]) as diary_pv,count(distinct(cl_id)) as device_num_diary
-           |from online.tl_hdfs_maidian_view
-           |where action="page_view"
-           |and params["page_name"]="diary_detail"
-           |and (params["out"]-params["in"])<7200
+           |select distinct(device_id) as device_id
+           |from online.ml_device_day_active_status
+           |where active_type != '4'
+           |and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+           |        ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+           |        ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+           |        ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+           |        ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+           |        ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+           |        ,'promotion_shike','promotion_julang_jl03')
            |and partition_date ='${partition_date}'
          """.stripMargin
       )
+      device_id_newUser.show()
+      device_id_newUser.createOrReplaceTempView("device_id_new")
 
-      val meigou_pv = sc.sql(
+      val blacklist_id = sc.sql(
         s"""
-           |select '${stat_date}' as stat_date,count(params["business_id"]) as meigou_pv,count(distinct(cl_id)) as device_num_meigou
-           |from online.tl_hdfs_maidian_view
-           |where action="page_view"
-           |and params["page_name"]="welfare_detail"
-           |and (params["out"]-params["in"])<7200
-           |and partition_date ='${partition_date}'
+           |SELECT device_id
+           |from blacklist
          """.stripMargin
       )
+      blacklist_id.show()
+      blacklist_id.createOrReplaceTempView("blacklist_id")
 
-
-      val result = diary_pv.join(meigou_pv,"stat_date")
-      result.show()
-      GmeiConfig.writeToJDBCTable(result, "diary_pv", SaveMode.Append)
-
-//      result.select("stat_date","diary_pv","device_num_diary","meigou_pv","device_num_meigou").write.mode(SaveMode.Append).save("/data2/test.txt")
-
-
+      val final_id = sc.sql(
+        s"""
+           |select device_id
+           |from agency_id
+           |UNION ALL
+           |select device_id
+           |from device_id_new
+           |UNION ALL
+           |select device_id
+           |from blacklist_id
+         """.stripMargin
+      )
+      final_id.show()
+      final_id.createOrReplaceTempView("final_id")
+
+//日记本点击
+      val referrer=List("all_case_service_comment","all_cases","diary_detail","diary_list","diary_listof_related_service",
+      "expert_detail","free_activity_detail","home","message_home","my_diary","organization_detail","other_homepage","question_detail",
+      "search_result_diary","search_result_more","welfare_detail","zone_v3")
+      for( a <- referrer ){
+        val diary_clk_temp = sc.sql(
+          s"""
+             |select ov.partition_date,count(ov.cl_id) as clk_num
+             |from online.tl_hdfs_maidian_view ov left join final_id
+             |on ov.cl_id = final_id.device_id
+             |where ov.action = "page_view"
+             |and params['page_name']="diary_detail"
+             |and params['referrer']='${a}'
+             |and ov.cl_id != "NULL"
+             |and ov.partition_date >='20181101'
+             |and final_id.device_id is  null
+             |group by ov.partition_date
+             |order by ov.partition_date
+       """.stripMargin
+        )
+        println("来源：",a)
+        diary_clk_temp.show(80)
+
+      }
+
+      //5.登录人数
+      val log_device_temp = sc.sql(
+        s"""
+           |select oe.stat_date,count(distinct(oe.device_id)) as log_num
+           |from data_feed_exposure oe left join final_id
+           |on oe.device_id = final_id.device_id
+           |and oe.stat_date >='2018-11-01'
+           |and final_id.device_id is null
+           |group by oe.stat_date
+           |order by oe.stat_date
+         """.stripMargin
+      )
+      println("登录人数统计：")
+      log_device_temp.show(80)
 
     }