update

task/search_strategy_d.py

@@ -81,18 +81,18 @@ for t in range(1, task_days):
     yesterday_str = (now + datetime.timedelta(days=-1)).strftime("%Y%m%d")
     one_week_age_str = (now + datetime.timedelta(days=-7)).strftime("%Y%m%d")
     sql_distinct_device_id = """
-    select spam_pv.device_id as spam_pv_device_id,dev.device_id as dev_device_id from 
-    (SELECT DISTINCT device_id
+
+    SELECT DISTINCT device_id
     FROM ml.ml_d_ct_dv_devicespam_d  --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
     WHERE partition_day={yesterday_str}
 
     UNION ALL 
     SELECT DISTINCT dev.device_id 
     FROM dim.dim_device_user_staff   --去除内网用户
-    )spam_pv
-    LEFT JOIN
+
+    UNION ALL 
     (
-        SELECT partition_date,t1.device_id
+        SELECT t1.device_id
         FROM 
         (--找出user_id当天活跃的第一个设备id
             SELECT user_id,partition_date,
@@ -138,7 +138,7 @@ for t in range(1, task_days):
                 ON t1.device_id = t2.device_id
         )t2
         on t1.user_id=t2.user_id
-        group by partition_date,device_id
+        group by device_id
 		)dev """.format(yesterday_str=yesterday_str,today_str=today_str)
     print(sql_distinct_device_id)
     distinct_device_id_df = spark.sql(sql_distinct_device_id,)
@@ -258,9 +258,8 @@ FROM
 		on t1.cl_id=t2.device_id AND t1.partition_date = t2.partition_date
 
 		LEFT JOIN distinct_device_id_view
-		on  t1.cl_id=distinct_device_id_view.device_id
+		on  t1.partition_date=distinct_device_id_view.partition_date AND t1.cl_id=distinct_device_id_view.device_id
 		WHERE (distinct_device_id_view.dev_device_id IS NULL or distinct_device_id_view.dev_device_id ='')
-		and (distinct_device_id_view.spam_pv_device_id is null or distinct_device_id_view.spam_pv_device_id ='')
 		GROUP BY t1.partition_date,t2.active_type,device_os_type,channel
 	)t
 )t3
@@ -353,7 +352,6 @@ LEFT JOIN
 	LEFT JOIN distinct_device_id_view
 	on t1.cl_id=distinct_device_id_view.device_id
 	WHERE (distinct_device_id_view.dev_device_id IS NULL or distinct_device_id_view.dev_device_id ='')
-	and (distinct_device_id_view.spam_pv_device_id is null or distinct_device_id_view.spam_pv_device_id ='')
 	GROUP BY t1.partition_date,active_type,device_os_type,channel
 )t4
 on t3.partition_date=t4.partition_date and t3.active_type=t4.active_type and t3.device_os_type = t4.device_os_type AND t3.channel = t4.channel