update

task/recommend_strategy_fix.py

@@ -780,69 +780,13 @@ for t in range(0, 3):
             and t13.page_name=t3.card_content_type
         LEFT JOIN
         (
-            select distinct device_id
-            from ml.ml_d_ct_dv_devicespam_d  --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
-            WHERE partition_day={partition_day}
-    
-            union all
-    
-            select distinct device_id
-            from dim.dim_device_user_staff   --去除内网用户
-        )spam_pv
-        on spam_pv.device_id=t2.cl_id
-        LEFT JOIN
-        (
-            SELECT partition_date,device_id
-            FROM
-                (--找出user_id当天活跃的第一个设备id
-              SELECT user_id,partition_date,
-                      if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
-                FROM online.ml_user_updates
-                WHERE partition_date={partition_day}
-            )t1
-                    JOIN
-                (  --医生账号
-              SELECT distinct user_id
-              FROM online.tl_hdfs_doctor_view
-              WHERE partition_date = {partition_day}
-    
-              --马甲账号/模特用户
-              UNION ALL
-              SELECT user_id
-              FROM ml.ml_c_ct_ui_user_dimen_d
-              WHERE partition_day = {partition_day}
-              AND (is_puppet = 'true' or is_classifyuser = 'true')
-    
-              UNION ALL
-              --公司内网覆盖用户
-              select distinct user_id
-              from dim.dim_device_user_staff
-    
-              UNION ALL
-              --登陆过医生设备
-              SELECT distinct t1.user_id
-                FROM
-                (
-                    SELECT user_id, v.device_id as device_id
-                    FROM online.ml_user_history_detail
-                        LATERAL VIEW EXPLODE(device_history_list) v AS device_id
-                    WHERE partition_date = {partition_day}
-                ) t1
-                JOIN
-                (
-                    SELECT device_id
-                    FROM online.ml_device_history_detail
-                    WHERE partition_date = {partition_day}
-                    AND is_login_doctor = '1'
-                ) t2
-                    ON t1.device_id = t2.device_id
-            )t2
-                on t1.user_id=t2.user_id
-            group by partition_date,device_id
-        )dev
-        on t2.partition_date=dev.partition_date and t2.cl_id=dev.device_id
-    WHERE spam_pv.device_id IS NULL
-      and dev.device_id is null
+    select distinct device_id
+    from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
+    where PARTITION_DAY = '{partition_day}'
+    AND is_abnormal_device = 'true'
+)dev
+        on  t2.cl_id=dev.device_id
+    WHERE dev.device_id is null
     GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
     order by day_id,device_os_type,active_type,card_content_type,recommend_type
     """.format(partition_day=today_str)