update

task/high_quality_diary_analysis.py

@@ -241,33 +241,81 @@ for redis_count, device_id in enumerate(device_id_dict):
 # 7000片内容的曝光
 
 exposure_sql = """
-  SELECT
-        cl_id,
-        card_id,
-        count(distinct app_session_id) as session_pv
-        FROM
-        (
-           SELECT 
-                  cl_id,
-                  card_id,
-                  app_session_id
-           from online.ml_community_precise_exposure_detail
-           WHERE partition_date>={today_str}
-           AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
-           AND is_exposure = '1'  ----精准曝光
-           AND page_name ='home'
-           AND tab_name = '精选'
-           AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','FIXEDSTRATEGY','FIXEDSTRATEGY_VIDEO')
-                or transaction_type like '%ctr' or transaction_type like '%cvr' or transaction_type like 'deeplink%')
-           AND card_content_type in ('diary')
-           AND transaction_type in ('high_quality_fmctr')
-           group by 
-                  cl_id,
-                  transaction_type,
-                  card_id,
-                  app_session_id
-        ) a  group by cl_id,card_id
-""".format(today_str="20200918")
+    select C.DEVICE_ID as devcie_id,C.CARD_ID as card_id from
+
+(SELECT T.DEVICE_ID as DEVICE_ID, --设备ID
+        T.CARD_ID as CARD_ID --卡片ID
+        --COUNT(T.CARD_ID) AS EXPOSURE --点击次数
+  FROM ML.MID_ML_C_ET_PE_PRECISEEXPOSURE_DIMEN_D T
+  WHERE T.PARTITION_DAY >= '20200918'
+    AND T.PAGE_CODE = 'search_result_diary'
+  GROUP BY T.DEVICE_ID,
+          T.CARD_ID) C
+        LEFT JOIN
+(
+    SELECT DISTINCT device_id
+    FROM ml.ml_d_ct_dv_devicespam_d  --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
+    WHERE partition_day='{partition_day}'
+
+    UNION ALL 
+    SELECT DISTINCT device_id
+    FROM dim.dim_device_user_staff   --去除内网用户
+)spam_pv
+on spam_pv.device_id=T.DEVICE_ID
+LEFT JOIN
+(
+	SELECT partition_date,device_id
+	FROM 
+	(--找出user_id当天活跃的第一个设备id
+	    SELECT user_id,partition_date,
+	            if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
+	      FROM online.ml_user_updates
+	      WHERE partition_date>='{partition_day}' AND partition_date<'{end_date}'
+	)t1
+	JOIN
+	(  --医生账号
+	    SELECT distinct user_id
+	    FROM online.tl_hdfs_doctor_view
+	    WHERE partition_date = '{partition_day}'
+
+	    --马甲账号/模特用户
+	    UNION ALL
+	    SELECT user_id
+	    FROM ml.ml_c_ct_ui_user_dimen_d
+	    WHERE partition_day = '{partition_day}'
+	    AND (is_puppet = 'true' or is_classifyuser = 'true')
+
+	    UNION ALL
+	    --公司内网覆盖用户
+	    select distinct user_id
+	    from dim.dim_device_user_staff
+
+	    UNION ALL
+	    --登陆过医生设备
+	    SELECT distinct t1.user_id
+	    FROM
+	    (
+	        SELECT user_id, v.device_id as device_id
+	        FROM online.ml_user_history_detail
+	            LATERAL VIEW EXPLODE(device_history_list) v AS device_id
+	        WHERE partition_date = '{partition_day}'
+	    )t1
+	    JOIN
+	    (
+	        SELECT device_id
+	        FROM online.ml_device_history_detail
+	        WHERE partition_date = '{partition_day}'
+	        AND is_login_doctor = '1'
+	    )t2
+	        ON t1.device_id = t2.device_id
+	)t2
+	on t1.user_id=t2.user_id
+	group by partition_date,device_id
+)dev 
+on C.DEVICE_ID=dev.device_id
+WHERE (spam_pv.device_id IS NULL or spam_pv.device_id = '')
+and (dev.device_id is null or dev.device_id='')
+    """.format(partition_day=yesterday_str, end_date=today_str)
 
 print(exposure_sql)
 exposure_df = spark.sql(exposure_sql)