push codes

7131016e · 魏艺敏 · 0b945c78 · 7131016e · 7131016e · 7131016e
Commit 7131016e authored Nov 17, 2020 by 魏艺敏
3 changed files
--- a/pm/daily_grey_recommend/etl/daily_grey_recommend.sql
+++ b/pm/daily_grey_recommend/etl/daily_grey_recommend.sql
@@ -46,12 +46,7 @@ from
                  when active_type ='4' then '老活跃设备' end  as active_type
            ,array(case when ((partition_date>='20201024' and partition_date<='20201113' and substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in (5,6,7,8))
                    or (partition_date>='20201114' and substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in (0,1,2,3,4,5,6,7,8,9)))then '灰度' else '非灰' end,'合计') as grey_type
-            ,array(CASE WHEN (a.device_id is not null or b.device_id is not null
+            ,array(CASE WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '医美' END , '合计') as channel
-                        or first_channel_source_type like '%xinyouxingkong%'
-                        or first_channel_source_type like '%jingmeng%'
-                        or first_channel_source_type like '%longyuzhixing%'
-                        or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
-                WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
        from online.ml_device_day_active_status m
        LEFT JOIN
            (SELECT code,is_spam,is_ai_channel,partition_day
@@ -59,17 +54,6 @@ from
                WHERE partition_day>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
                and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
            on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
-            LEFT JOIN
-            (SELECT DISTINCT device_id
-                FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
-                WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
-            ON m.device_id = a.device_id
-            LEFT JOIN
-            (SELECT device_id,day_id
-                FROM pm.tl_pm_channel_d
-                WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-                GROUP BY device_id,day_id)b
-            ON m.device_id = b.device_id AND m.partition_date = b.day_id
        where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
        and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
        and active_type in ('1','2','4')
@@ -200,60 +184,64 @@ left join
 )t9
 on t1.partition_date =t9.partition_date and t1.device_id=t9.cl_id
 LEFT JOIN
+(--首页浏览
+    SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+          ,cl_id
+    FROM online.bl_hdfs_maidian_updates
+    where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
+    and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+    AND action = 'page_view'
+    AND page_name='home'
+    group by concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)),cl_id
+)t11
+on t1.partition_date=t11.partition_date and t1.device_id=t11.cl_id
+LEFT JOIN
 (
-    select partition_date,cl_id
+    select t10.partition_date,t10.cl_id,
+            count(distinct t12.business_id) as business_num
    from
-    (   --首页feed卡片点击
+    (
-        SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+        select partition_date,cl_id,card_id
-              ,cl_id
+        from
-            --   ,count(distinct array(params['card_id'],app_session_id)) as click_pv
+        (   --首页feed卡片点击
-        FROM online.bl_hdfs_maidian_updates
+            SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
-        where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
+                  ,cl_id
-        and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+                  ,params['card_id'] as card_id
-        and action='on_click_card'
+            FROM online.bl_hdfs_maidian_updates
-        and page_name='home'
+            where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
-        and params['card_type']='card'
+            and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-        group by partition_date
+            and action='on_click_card'
-               ,cl_id
+            and page_name='home'
+            and params['card_type'] in ('card','video','search_word')
-        UNION ALL
+            group by partition_date
+                   ,cl_id,params['card_id']
-        --点击埋点有丢失，用浏览事件补充一下
+            UNION ALL
-        SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
-              ,cl_id
-            --   ,count(distinct array(params['card_id'],app_session_id)) as click_pv
-        FROM online.bl_hdfs_maidian_updates
-        where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
-        and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-        and action='page_view'
-        and (referrer in ('home') or
-                  (params['referrer_link'] like '%[%' and
-                   json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1] in ('home')))
-        and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
-        group by partition_date
-               ,cl_id
+            --点击埋点有丢失，用浏览事件补充一下
+            SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+                  ,cl_id
+                  ,params['business_id'] as card_id
+            FROM online.bl_hdfs_maidian_updates
+            where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
+            and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+            and action='page_view'
+            and (referrer in ('home') or
+                      (params['referrer_link'] like '%[%' and
+                       json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1] in ('home')))
+            and (page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
+                                      'video_steep','article_detail') or page_name like 'search_result_%')
+            group by partition_date
+                   ,cl_id,params['business_id']
+        )t10
+        group by partition_date,cl_id,card_id
    )t10
-    group by partition_date,cl_id
-)t10
+    LEFT JOIN
-    on t1.partition_date =t10.partition_date and t1.device_id=t10.cl_id
-LEFT JOIN
-    (--首页浏览
-        SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
-              ,cl_id
-        FROM online.bl_hdfs_maidian_updates
-        where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
-        and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-        AND action = 'page_view'
-        AND page_name='home'
-        group by concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)),cl_id
-    )t11
-    on t1.partition_date=t11.partition_date and t1.device_id=t11.cl_id
-LEFT JOIN
    (   --内容的good click代码
        select concat_ws('-',substr(a.partition_date,1,4),substr(a.partition_date,5,2),substr(a.partition_date,7,2)) as partition_date
            ,a.cl_id
-            ,count(distinct a.business_id) as business_num
+            ,a.business_id
        from
        (
            select *,case when page_name in ('diary_detail','topic_detail') THEN 'diary'
@@ -265,8 +253,9 @@ LEFT JOIN
            and (referrer='home' or
                  (params['referrer_link'] like '%[%' and
                   json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
-            and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
+            and (page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
-                                  'video_steep','article_detail')
+                                  'video_steep','article_detail') or page_name like 'search_result_%')
+--            and (params['is_push']=0 or params['is_push'] is null)
        )a
        left join
        (--评论的埋点有缺失，所以用业务库数据来补充
@@ -328,10 +317,14 @@ LEFT JOIN
                 or (action='on_click_favor' and params['motion']='do')--点击收藏
                 or action='page_click_share'--点击分享
                 or (action='on_click_vote' and params['motion']='vote')--点击点赞
-                 or b.user_id is not null )--当天有针对内容的评论
+                 or b.user_id is not null--当天有针对内容的评论
-        group by a.partition_date,a.cl_id
+                 or page_name like 'search_result_%' and action not in ('upload_page','report_status','popup_view','refresh_page','on_app_session_over'))--搜索结果页上的任意有效点击动作
+        group by a.partition_date,a.cl_id,a.business_id
    )t12
-    on t1.partition_date=t12.partition_date and t1.device_id=t12.cl_id
+    on t10.partition_date=t12.partition_date and t10.cl_id=t12.cl_id and t10.card_id=t12.business_id
+    group by t10.partition_date,t10.cl_id
+)t10
+on t11.partition_date =t10.partition_date and t11.cl_id=t10.cl_id
 left join
 ( -- 去掉黑名单设备
@@ -344,3 +337,5 @@ on t1.device_id =spam_pv.device_id
 WHERE spam_pv.device_id IS NULL
 group by t1.partition_date,t1.device_os_type,t1.active_type,t1.channel,t1.grey_type
--- a/pm/daily_grey_recommend/report/home_grey_recommend.sql
+++ b/pm/daily_grey_recommend/report/home_grey_recommend.sql
@@ -14,12 +14,12 @@ SELECT day_id as `日期`
        ,avg_content_pagestay as `单内容浏览时长（m）`
        ,avg_app_duration as `单设备使用时长（m）`
        ,pagestay_in_app_duration as `内容浏览时长在app使用时长上的占比`
-        ,wel_second_in_content_pv`来自内容页的商业二跳/内容pv`
+--        ,wel_second_in_content_pv`来自内容页的商业二跳/内容pv`
-        ,content_second_in_content_pv`来自内容页的内容二跳/内容pv`
+--        ,content_second_in_content_pv`来自内容页的内容二跳/内容pv`
        ,home_cardclick_uv_in_home_uv as `首页卡片点击uv/首页uv`
        ,home_goodclick_uv_in_homeclick_uv as `good click卡片数uv/首页点击卡片数uv`
 from pm.tl_pm_grey_recommend_d
-where ((partition_day='20201112' and day_id<='2020-11-11')
+where ((partition_day='20201116' and day_id<='2020-11-15')
-       or (partition_day>'20201112' and partition_day<regexp_replace(DATE_SUB(current_date,1) ,'-','')
+       or (partition_day>'20201116' and partition_day<regexp_replace(DATE_SUB(current_date,1) ,'-','')
           and day_id=date_sub(concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)),1))
       or partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))
--- a/pm/daily_push/etl/daily_push.sql
+++ b/pm/daily_push/etl/daily_push.sql
@@ -537,7 +537,7 @@ FROM
        from online.ml_device_day_active_status
        where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
        AND active_type in ('1','2','3')
-        and device_os_type='ios'
+--        and device_os_type='ios'
    )d
    on a.partition_day=d.partition_date and a.device_id=d.device_id
    where d.device_id is null