统计发现页面的点击率

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -782,25 +782,39 @@ object question_count {
       //println(param.date)
       val partition_date = stat_date.replace("-","")
 
-      val question_id=List(212264,212266,212272,212281,212287,212436,212439,212437,212505,212506,212507,212522,212523,212526,212532,212783,212787,212789,212793,212796,213202,213199,213216,213219,213297,213224,213226,213239,213300,213302,213316,213307,213308,213370,213377,213349,213358,213368,213392,213393,213435,213453,213445,213448,213458,213466,213471,213478,213485,213638,213642,213644,213727,213729,213775,213776,213810,213817,213805,213821,213884,213885,213892,213834,213879,214043,214050,214062,214055,214056,214058,214064,214159,214182,214149,214184,214190,214206,214227,214243,214242,214288,214289,214293,214295,214541,214544,214546,214614,214618,214619,214620,214682,214683,214684,214848,214850,214854,214856,214857,214903,214908,214913,214918,214919,214980,214981,214988,214985,215031,215034,215036,215039,215094,215098,215104,215107,215112,215222,215225,215233,215237,215265,215366,215347,215346,215343)
-
-      for( a <- question_id ){
-        val agency_id = sc.sql(
-          s"""
-             |SELECT partition_date,count(cl_id)
-             |FROM online.tl_hdfs_maidian_view
-             |WHERE partition_date >= '20190101'
-             |and action='community_home_click_feed_card'
-             |and params["card_type"]="问题"
-             |and params['business_id']='212264'
-             |group by partition_date
-             |order by partition_date
-         """.stripMargin
+      val agency_id = sc.sql(
+        s"""
+           |SELECT DISTINCT(cl_id) as device_id
+           |FROM online.ml_hospital_spam_pv_day
+           |WHERE partition_date >= '20180402'
+           |AND partition_date <= '20190117'
+           |AND pv_ratio >= 0.95
+           |UNION ALL
+           |SELECT DISTINCT(cl_id) as device_id
+           |FROM online.ml_hospital_spam_pv_month
+           |WHERE partition_date >= '20171101'
+           |AND partition_date <= '20190117'
+           |AND pv_ratio >= 0.95
+       """.stripMargin
         )
-        println("question_id:",a)
-        agency_id.show(20)
 
-      }
+      val question_count = sc.sql(
+        s"""
+           |SELECT partition_date,count(cl_id)
+           |FROM online.tl_hdfs_maidian_view ov inner join agency_id
+           |on ov.cl_id = agency_id.device_id
+           |WHERE ov.partition_date >= '20190101'
+           |and ov.action='community_home_click_feed_card'
+           |and ov.params["card_type"]="问题"
+           |and ov.cl_id not in (select device_id from blacklist)
+           |GROUP BY ov.partition_date
+           |order by ov.partition_date
+       """.stripMargin
+      )
+
+      question_count.show(30)
+
+
 
     }