update

task/search_tractate_ctr.py

@@ -98,7 +98,7 @@ for t in range(1, task_days):
             FROM
             (--搜索结果页卡片精准曝光
               SELECT  partition_day,
-                      dev.device_id,
+                      device_id,
                       count(CASE WHEN card_content_type='service' THEN 1 END) as service_exp_pv,
                       count(CASE WHEN card_content_type<>'service' THEN 1 END) as neirong_exp_pv
               FROM
@@ -114,72 +114,7 @@ for t in range(1, task_days):
                         ,'search_result_wiki','search_result_question_answer')
                 AND card_content_type IN ('user_post')
               )a
-              left join 
-            (
-                SELECT DISTINCT device_id
-                FROM ml.ml_d_ct_dv_devicespam_d  --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
-                WHERE partition_day='{partition_day}'
-            
-                UNION ALL 
-                SELECT DISTINCT device_id
-                FROM dim.dim_device_user_staff   --去除内网用户
-            )spam_pv
-            on spam_pv.device_id=a.device_id
-            LEFT JOIN
-            (
-                SELECT partition_date,device_id
-                FROM 
-                (--找出user_id当天活跃的第一个设备id
-                    SELECT user_id,partition_date,
-                            if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
-                      FROM online.ml_user_updates
-                      WHERE partition_date>='{partition_day}' AND partition_date<'{end_date}'
-                )t1
-                JOIN
-                (  --医生账号
-                    SELECT distinct user_id
-                    FROM online.tl_hdfs_doctor_view
-                    WHERE partition_date = '{partition_day}'
-            
-                    --马甲账号/模特用户
-                    UNION ALL
-                    SELECT user_id
-                    FROM ml.ml_c_ct_ui_user_dimen_d
-                    WHERE partition_day = '{partition_day}'
-                    AND (is_puppet = 'true' or is_classifyuser = 'true')
-            
-                    UNION ALL
-                    --公司内网覆盖用户
-                    select distinct user_id
-                    from dim.dim_device_user_staff
-            
-                    UNION ALL
-                    --登陆过医生设备
-                    SELECT distinct t1.user_id
-                    FROM
-                    (
-                        SELECT user_id, v.device_id as device_id
-                        FROM online.ml_user_history_detail
-                            LATERAL VIEW EXPLODE(device_history_list) v AS device_id
-                        WHERE partition_date = '{partition_day}'
-                    )t1
-                    JOIN
-                    (
-                        SELECT device_id
-                        FROM online.ml_device_history_detail
-                        WHERE partition_date = '{partition_day}'
-                        AND is_login_doctor = '1'
-                    )t2
-                        ON t1.device_id = t2.device_id
-                )t2
-                on t1.user_id=t2.user_id
-                group by partition_date,device_id
-            )dev 
-            on a.device_id=dev.device_id
-            WHERE (a.device_id IS NULL or a.device_id = '')
-            and (dev.device_id is null or dev.device_id='')
-              
-              group by partition_day,card_content_type,dev.device_id
+              group by partition_day,card_content_type,device_id
             )t1
           
             LEFT JOIN