搜索统计去除用户黑名单

eda/feededa/src/main/scala/com/gmei/temp_count.scala

@@ -509,6 +509,24 @@ object GetHiveSearchData {
       val partition_date = stat_date.replace("-","")
 
       val exposureAccum = spark.sparkContext.longAccumulator("search exposure data")
+
+      val jigou_id = spark.sql(
+        s"""
+           |SELECT cl_id
+           |FROM online.ml_hospital_spam_pv_day
+           |WHERE partition_date>='20180402' AND partition_date<'${partition_date}'
+           |AND pv_ratio>=0.95
+           |UNION ALL
+           |SELECT cl_id
+           |FROM online.ml_hospital_spam_pv_month
+           |WHERE partition_date>='20171101' AND partition_date<'${partition_date}'
+           |AND pv_ratio>=0.95
+           |UNION ALL
+           |select device_id as cl_id from blacklist
+       """.stripMargin
+      )
+      jigou_id.createOrReplaceTempView("jigou_id")
+
       val exposureSql = spark.sql(
         s"""
            |select action,user_id,city_id,app
@@ -579,7 +597,11 @@ object GetHiveSearchData {
         s"""
            |select
            |count(1) click_num
-           |from  online.tl_hdfs_maidian_view where partition_date='${partition_date}' and action='$strClickAction'
+           |from  online.tl_hdfs_maidian_view  ov left join jigou_id
+           |on ov.cl_id = jigou_id.cl_id
+           |where ov.partition_date='${partition_date}'
+           |and ov.action='$strClickAction'
+           |jigou_id.cl_id is null
      """.stripMargin
       )