update

task/search_strategy_d.py

@@ -78,41 +78,7 @@ for t in range(0, task_days):
               FROM online.ml_user_updates
               WHERE partition_date>='{yesterday_str}' AND partition_date<'{today_str}'
         )t1
-        JOIN
-        (  
-            SELECT distinct user_id
-            FROM online.tl_hdfs_doctor_view
-            WHERE partition_date = '{yesterday_str}'
-    
-            UNION ALL
-            SELECT user_id
-            FROM ml.ml_c_ct_ui_user_dimen_d
-            WHERE partition_day = '{yesterday_str}'
-            AND (is_puppet = 'true' or is_classifyuser = 'true')
-    
-            UNION ALL
-            select distinct user_id
-            from dim.dim_device_user_staff
-            UNION ALL
-            
-            SELECT distinct t1.user_id
-            FROM
-            (
-                SELECT user_id, v.device_id as device_id
-                FROM online.ml_user_history_detail
-                    LATERAL VIEW EXPLODE(device_history_list) v AS device_id
-                WHERE partition_date = '{yesterday_str}'
-            )t1
-            JOIN
-            (
-                SELECT device_id
-                FROM online.ml_device_history_detail
-                WHERE partition_date = '{yesterday_str}'
-                AND is_login_doctor = '1'
-            )t2
-                ON t1.device_id = t2.device_id
-        )t3
-        on t1.user_id=t3.user_id group by partition_date,device_id
+        group by partition_date,device_id
 """.format(today_str=today_str, yesterday_str=yesterday_str)
     print(sql_distinct_device_id)
     distinct_device_id_df = spark.sql(sql_distinct_device_id)