搜索统计去除用户黑名单

eda/feededa/src/main/scala/com/gmei/temp_count.scala

@@ -434,13 +434,35 @@ object GetHiveSearchData {
       var (searchHospitalExposureVal,searchHospitalClickNum,searchHospitalExposureMapCount,searchHospitalExposureFilterCount) = GetSearchResultData(sc,strSearchHospitalExposureAction,strSearchHospitalClickAction,stat_date)
 
 
+
+      val jigou_id = sc.sql(
+        s"""
+           |SELECT cl_id
+           |FROM online.ml_hospital_spam_pv_day
+           |WHERE partition_date>='20180402' AND partition_date<'${partition_date}'
+           |AND pv_ratio>=0.95
+           |UNION ALL
+           |SELECT cl_id
+           |FROM online.ml_hospital_spam_pv_month
+           |WHERE partition_date>='20171101' AND partition_date<'${partition_date}'
+           |AND pv_ratio>=0.95
+           |UNION ALL
+           |select device_id as cl_id from blacklist
+       """.stripMargin
+      )
+      jigou_id.createOrReplaceTempView("jigou_id")
+
+
       val diary_clickSql = sc.sql(
         s"""
            |select
            |count(1) click_num
-           |from  online.tl_hdfs_maidian_view where partition_date='${partition_date}'
-           |and action='on_click_diary_card'
-           |and params['page_name']='search_result_diary'
+           |from  online.tl_hdfs_maidian_view ov left join jigou_id
+           |on ov.cl_id = jigou_id.cl_id
+           |where ov.partition_date='${partition_date}'
+           |and ov.action='on_click_diary_card'
+           |and ov.params['page_name']='search_result_diary'
+           |and jigou_id.cl_id is null
        """.stripMargin
       )
 
@@ -451,9 +473,12 @@ object GetHiveSearchData {
         s"""
            |select
            |count(1) click_num
-           |from  online.tl_hdfs_maidian_view where partition_date='${partition_date}'
-           |and action='on_click_diary_card'
-           |and  params['page_name']='search_result_more'
+           |from  online.tl_hdfs_maidian_view ov left join jigou_id
+           |on ov.cl_id = jigou_id.cl_id
+           |where ov.partition_date='${partition_date}'
+           |and ov.action='on_click_diary_card'
+           |and  ov.params['page_name']='search_result_more'
+           |and jigou_id.cl_id is null
        """.stripMargin
       )
 
@@ -472,7 +497,7 @@ object GetHiveSearchData {
 
 
       val result=List((stat_date,diaryExposureVal,(diary_click_num+diaryClickNum),meigouExposureVal,meigouClickNum,searchResultExposureVal,(searchResultClickNum+content_diary_click_num),searchDoctorExposureVal,searchDoctorClickNum,searchHospitalExposureVal,searchHospitalClickNum))
-      val df_result = sc.createDataFrame(result).withColumnRenamed("_1,_2,_3,_4,_5,_6,_7,_8,_9,_10,_11","stat_date,diaryExposureVal,diary_click_num,meigouExposureVal,meigouClickNum,searchResultExposureVal,searchResultClickNum,searchDoctorExposureVal,searchDoctorClickNum,searchHospitalExposureVal,searchHospitalClickNum")
+      val df_result = sc.createDataFrame(result)
 
       GmeiConfig.writeToJDBCTable(df_result, table = "GetHiveSearchData_CTR", SaveMode.Append)
 
@@ -486,12 +511,12 @@ object GetHiveSearchData {
       val exposureAccum = spark.sparkContext.longAccumulator("search exposure data")
       val exposureSql = spark.sql(
         s"""
-           |select
-           |action,
-           |user_id,
-           |city_id,
-           |app
-           |from online.tl_hdfs_backend_view where  action='$strExposureAction' and partition_date='${partition_date}'
+           |select action,user_id,city_id,app
+           |from online.tl_hdfs_backend_view ov left join jigou_id
+           |on ov.cl_id = jigou_id.cl_id
+           |where  ov.action='$strExposureAction'
+           |and ov.partition_date='${partition_date}'
+           |and jigou_id.cl_id is null
     """.stripMargin
       )