Merge branch 'weiyimin' into 'master'

push codes

See merge request !116

pm/daily_grey_recommend/etl/daily_grey_recommend.sql

@@ -46,12 +46,7 @@ from
                   when active_type ='4' then '老活跃设备' end  as active_type
             ,array(case when ((partition_date>='20201024' and partition_date<='20201113' and substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in (5,6,7,8))
                     or (partition_date>='20201114' and substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in (0,1,2,3,4,5,6,7,8,9)))then '灰度' else '非灰' end,'合计') as grey_type
-            ,array(CASE WHEN (a.device_id is not null or b.device_id is not null
-                        or first_channel_source_type like '%xinyouxingkong%'
-                        or first_channel_source_type like '%jingmeng%'
-                        or first_channel_source_type like '%longyuzhixing%'
-                        or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
-                WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
+            ,array(CASE WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '医美' END , '合计') as channel
         from online.ml_device_day_active_status m
         LEFT JOIN
             (SELECT code,is_spam,is_ai_channel,partition_day
@@ -59,17 +54,6 @@ from
                 WHERE partition_day>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
                 and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
             on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
-            LEFT JOIN
-            (SELECT DISTINCT device_id
-                FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
-                WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
-            ON m.device_id = a.device_id
-            LEFT JOIN
-            (SELECT device_id,day_id
-                FROM pm.tl_pm_channel_d
-                WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-                GROUP BY device_id,day_id)b
-            ON m.device_id = b.device_id AND m.partition_date = b.day_id
         where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
         and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         and active_type in ('1','2','4')
@@ -200,28 +184,44 @@ left join
 )t9
 on t1.partition_date =t9.partition_date and t1.device_id=t9.cl_id
 LEFT JOIN
+(--首页浏览
+    SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+          ,cl_id
+    FROM online.bl_hdfs_maidian_updates
+    where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
+    and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+    AND action = 'page_view'
+    AND page_name='home'
+    group by concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)),cl_id
+)t11
+on t1.partition_date=t11.partition_date and t1.device_id=t11.cl_id
+LEFT JOIN
 (
-    select partition_date,cl_id
+    select t10.partition_date,t10.cl_id,
+            count(distinct t12.business_id) as business_num
+    from
+    (
+        select partition_date,cl_id,card_id
         from
         (   --首页feed卡片点击
             SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
                   ,cl_id
-            --   ,count(distinct array(params['card_id'],app_session_id)) as click_pv
+                  ,params['card_id'] as card_id
             FROM online.bl_hdfs_maidian_updates
             where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
             and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             and action='on_click_card'
             and page_name='home'
-        and params['card_type']='card'
+            and params['card_type'] in ('card','video','search_word')
             group by partition_date
-               ,cl_id
+                   ,cl_id,params['card_id']
 
             UNION ALL
 
             --点击埋点有丢失，用浏览事件补充一下
             SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
                   ,cl_id
-            --   ,count(distinct array(params['card_id'],app_session_id)) as click_pv
+                  ,params['business_id'] as card_id
             FROM online.bl_hdfs_maidian_updates
             where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
             and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
@@ -229,31 +229,19 @@ LEFT JOIN
             and (referrer in ('home') or
                       (params['referrer_link'] like '%[%' and
                        json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1] in ('home')))
-        and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
+            and (page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
+                                      'video_steep','article_detail') or page_name like 'search_result_%')
             group by partition_date
-               ,cl_id
-
+                   ,cl_id,params['business_id']
         )t10
-    group by partition_date,cl_id
-)t10
-    on t1.partition_date =t10.partition_date and t1.device_id=t10.cl_id
-LEFT JOIN
-    (--首页浏览
-        SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
-              ,cl_id
-        FROM online.bl_hdfs_maidian_updates
-        where partition_date>=regexp_replace(DATE_SUB(current_date,2) ,'-','')
-        and partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-        AND action = 'page_view'
-        AND page_name='home'
-        group by concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)),cl_id
-    )t11
-    on t1.partition_date=t11.partition_date and t1.device_id=t11.cl_id
-LEFT JOIN
+        group by partition_date,cl_id,card_id
+    )t10
+
+    LEFT JOIN
     (   --内容的good click代码
         select concat_ws('-',substr(a.partition_date,1,4),substr(a.partition_date,5,2),substr(a.partition_date,7,2)) as partition_date
             ,a.cl_id
-            ,count(distinct a.business_id) as business_num
+            ,a.business_id
         from
         (
             select *,case when page_name in ('diary_detail','topic_detail') THEN 'diary'
@@ -265,8 +253,9 @@ LEFT JOIN
             and (referrer='home' or
                   (params['referrer_link'] like '%[%' and
                    json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
-            and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
-                                  'video_steep','article_detail')
+            and (page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
+                                  'video_steep','article_detail') or page_name like 'search_result_%')
+--            and (params['is_push']=0 or params['is_push'] is null)
         )a
         left join
         (--评论的埋点有缺失，所以用业务库数据来补充
@@ -328,10 +317,14 @@ LEFT JOIN
                  or (action='on_click_favor' and params['motion']='do')--点击收藏
                  or action='page_click_share'--点击分享
                  or (action='on_click_vote' and params['motion']='vote')--点击点赞
-                 or b.user_id is not null )--当天有针对内容的评论
-        group by a.partition_date,a.cl_id
+                 or b.user_id is not null--当天有针对内容的评论
+                 or page_name like 'search_result_%' and action not in ('upload_page','report_status','popup_view','refresh_page','on_app_session_over'))--搜索结果页上的任意有效点击动作
+        group by a.partition_date,a.cl_id,a.business_id
     )t12
-    on t1.partition_date=t12.partition_date and t1.device_id=t12.cl_id
+    on t10.partition_date=t12.partition_date and t10.cl_id=t12.cl_id and t10.card_id=t12.business_id
+    group by t10.partition_date,t10.cl_id
+)t10
+on t11.partition_date =t10.partition_date and t11.cl_id=t10.cl_id
 
 left join
 ( -- 去掉黑名单设备
@@ -344,3 +337,5 @@ on t1.device_id =spam_pv.device_id
 WHERE spam_pv.device_id IS NULL
 group by t1.partition_date,t1.device_os_type,t1.active_type,t1.channel,t1.grey_type
 
+
+

pm/daily_grey_recommend/report/home_grey_recommend.sql

@@ -14,12 +14,12 @@ SELECT day_id as `日期`
         ,avg_content_pagestay as `单内容浏览时长（m）`
         ,avg_app_duration as `单设备使用时长（m）`
         ,pagestay_in_app_duration as `内容浏览时长在app使用时长上的占比`
-        ,wel_second_in_content_pv`来自内容页的商业二跳/内容pv`
-        ,content_second_in_content_pv`来自内容页的内容二跳/内容pv`
+--        ,wel_second_in_content_pv`来自内容页的商业二跳/内容pv`
+--        ,content_second_in_content_pv`来自内容页的内容二跳/内容pv`
         ,home_cardclick_uv_in_home_uv as `首页卡片点击uv/首页uv`
         ,home_goodclick_uv_in_homeclick_uv as `good click卡片数uv/首页点击卡片数uv`
 from pm.tl_pm_grey_recommend_d
-where ((partition_day='20201112' and day_id<='2020-11-11')
-       or (partition_day>'20201112' and partition_day<regexp_replace(DATE_SUB(current_date,1) ,'-','')
+where ((partition_day='20201116' and day_id<='2020-11-15')
+       or (partition_day>'20201116' and partition_day<regexp_replace(DATE_SUB(current_date,1) ,'-','')
            and day_id=date_sub(concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)),1))
        or partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))

pm/daily_push/etl/daily_push.sql

@@ -537,7 +537,7 @@ FROM
         from online.ml_device_day_active_status
         where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         AND active_type in ('1','2','3')
-        and device_os_type='ios'
+--        and device_os_type='ios'
     )d
     on a.partition_day=d.partition_date and a.device_id=d.device_id
     where d.device_id is null