Merge branch 'weiyimin' into 'master'

push codes

See merge request !79

pm/daily_content/etl/create_daily_content.sql

@@ -27,10 +27,11 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_content_v2
  home_card_click_uv BIGINT comment '{"chs_name":"首页卡片点击uv","description":"","etl":"","value":"","remark":""}',
  home_card_click_pv BIGINT comment '{"chs_name":"首页卡片点击pv","description":"","etl":"","value":"","remark":""}',
  home_good_click BIGINT comment '{"chs_name":"来源于首页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}',
- page_stay_per_device BIGINT comment '{"chs_name":"单设备内容浏览时长(m)","description":"","etl":"","value":"","remark":""}',
+ page_stay_per_device string comment '{"chs_name":"单设备内容浏览时长(m)","description":"","etl":"","value":"","remark":""}',
+ ai_report_uv BIGINT comment '{"chs_name":"ai结果页uv","description":"","etl":"","value":"","remark":""}',
  ai_report_card_click_uv BIGINT comment '{"chs_name":"ai结果页内容卡片点击uv","description":"","etl":"","value":"","remark":""}',
  ai_report_card_click_pv BIGINT comment '{"chs_name":"ai结果页内容卡片点击pv","description":"","etl":"","value":"","remark":""}',
- ai_report_good_click BIGINT comment '{"chs_name":"来源于结果页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}',
+ ai_report_good_click BIGINT comment '{"chs_name":"来源于结果页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}'
 )comment '内容日报-简化版'
 PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
 ROW FORMAT DELIMITED

pm/daily_content/etl/daily_content.sql

@@ -23,6 +23,7 @@ SELECT
         ,NVL(t2.home_click_pv,0) AS home_card_click_pv
         ,NVL(t2.good_pv,0) AS home_good_click
         ,NVL(t2.avg_page_stay,0) AS page_stay_per_device
+        ,NVL(t3.ai_report_uv,0) AS ai_report_uv
         ,NVL(t3.ai_click_uv,0) AS ai_report_card_click_uv
         ,NVL(t3.ai_click_pv,0) AS ai_report_card_click_pv
         ,NVL(t3.good_pv,0) AS ai_report_good_click
@@ -52,9 +53,9 @@ FROM
                 LEFT JOIN
                 (SELECT code,is_spam,is_ai_channel,partition_day
                     FROM DIM.DIM_AI_CHANNEL_ZP_NEW
-                    WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
+                    WHERE partition_day >='20190701' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
                 on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
-            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+            WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             AND active_type IN ('1','2','4')
             AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
                     ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
@@ -74,7 +75,7 @@ FROM
         SELECT device_id
                 ,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS partition_date
         FROM online.ml_device_day_active_status
-        WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
     )t2
     ON t1.device_id=t2.device_id
     left join
@@ -114,9 +115,9 @@ left join
                 LEFT JOIN
                 (SELECT code,is_spam,is_ai_channel,partition_day
                     FROM DIM.DIM_AI_CHANNEL_ZP_NEW
-                    WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
+                    WHERE partition_day >='20190701' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
                 on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
-            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+            WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             AND active_type IN ('1','2','4')
             AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
                     ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
@@ -137,12 +138,12 @@ left join
                 ,cl_id
                 ,count(distinct time_str) as pv
         FROM online.bl_hdfs_maidian_updates
-        WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         and referrer='home'
         AND action = 'page_view'
         -- and params['is_first']=1
         AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
-                          'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect')
+                          'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect','welfare_detail')--9月11日新增了可以跳转商详页评论的日记卡片
         and page_stay>=20
         group by partition_date,cl_id
     )t2
@@ -152,7 +153,7 @@ left join
     (
         SELECT partition_date,cl_id,count(distinct time_str) as pv
         from online.bl_hdfs_maidian_updates
-        WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         AND action='on_click_card'
         AND params['page_name'] ='home'
         and params['card_content_type'] in ('diary','user_post','answer','qa')--首页内容卡片点击，未限制tab
@@ -165,7 +166,7 @@ left join
                 ,cl_id
                 ,sum(page_stay) as page_stay
         FROM online.bl_hdfs_maidian_updates
-        WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         AND action = 'page_view'
         AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
                           'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect')
@@ -194,6 +195,7 @@ left join
             ,device_os_type
             ,active_type
             ,channel
+            ,count(distinct t4.cl_id) as ai_report_uv
             ,count(distinct t3.cl_id) as ai_click_uv
             ,sum(t3.pv) as ai_click_pv
             ,count(distinct t2.cl_id) as good_uv
@@ -213,9 +215,9 @@ left join
                 LEFT JOIN
                 (SELECT code,is_spam,is_ai_channel,partition_day
                     FROM DIM.DIM_AI_CHANNEL_ZP_NEW
-                    WHERE partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
+                    WHERE partition_day >='20190701' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-',''))tmp
                 on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
-            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+            WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             AND active_type IN ('1','2','4')
             AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
                     ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
@@ -236,7 +238,7 @@ left join
                 ,cl_id
                 ,count(distinct time_str) as pv
         FROM online.bl_hdfs_maidian_updates
-        WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        WHERE partition_date>='20190701' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         and referrer in ('report_result','face_detect_result','float_tag_detail')
         AND action = 'page_view'
         -- and params['is_first']=1
@@ -251,7 +253,7 @@ left join
     (
         SELECT cl_id,partition_date,count(distinct time_str) as pv
         FROM  online.bl_hdfs_maidian_updates
-        WHERE partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        WHERE partition_date >= '20190701'  and partition_date <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
         AND action = 'on_click_card'
         and page_name in ('report_result','face_detect_result','float_tag_detail')   --ai测颜值、ai测肤质、模拟整形结果页
         AND params['card_content_type'] in ('diary','user_post','answer','qa')
@@ -260,6 +262,18 @@ left join
     )t3
     ON t1.partition_date=t3.partition_date AND t1.device_id=t3.cl_id
     left join
+    (
+        SELECT cl_id,partition_date,count(distinct time_str) as pv
+        FROM  online.bl_hdfs_maidian_updates
+        WHERE partition_date >= '20190701'  and partition_date <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
+        AND action = 'page_view'
+        and page_name in ('report_result','face_detect_result','float_tag_detail')   --ai测颜值、ai测肤质、模拟整形结果页
+        -- AND params['card_content_type'] in ('diary','user_post','answer','qa')
+        -- AND params['tab_name'] in ('猜你喜欢','为你定制','reference_case')--不限制tab name，只要是从ai结果页到内容页的就算
+        group by cl_id,partition_date
+    )t4
+    ON t1.partition_date=t4.partition_date AND t1.device_id=t4.cl_id
+    left join
     ( -- 去掉黑名单设备
         select distinct device_id
         from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D

pm/daily_reply_content/report/daily_reply_content.sql

@@ -72,12 +72,7 @@ left join
     select distinct user_id
     from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
     where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
-    and (is_doctor = 'true'
-        or is_puppet = 'true'
-        or is_classify_user = 'true'
-        or is_reply_fake = 'true'
-        or is_staff = 'true'
-        or is_associated_issuedevice = 'true')
+    and is_abnormal_user = 'true'
 )c
 on a.user_id=c.user_id
 where c.user_id is null