Merge branch 'weiyimin' into 'master'

push codes

See merge request !60

pm/daily_userpost/etl/create_userpost_v2.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 线索日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_userpost_d_v2
+(
+ post_id string comment '{"chs_name":"帖子id","description":"","etl":"","value":"","remark":""}',
+ title string comment '{"chs_name":"帖子标题","description":"","etl":"","value":"","remark":""}',
+ audit_date string comment '{"chs_name":"最新审核时间","description":"","etl":"","value":"","remark":""}',
+ tag_list string comment '{"chs_name":"关联标签","description":"","etl":"","value":"","remark":""}',
+ click_pv_1 bigint comment '{"chs_name":"前1日点击","description":"","etl":"","value":"","remark":""}',
+ exp_pv_1 bigint comment '{"chs_name":"前1日曝光","description":"","etl":"","value":"","remark":""}',
+ page_pv_1 bigint comment '{"chs_name":"前1日浏览pv","description":"","etl":"","value":"","remark":""}',
+ page_pv_20s_1 bigint comment '{"chs_name":"前1日浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
+ reply_num_1 bigint comment '{"chs_name":"前1日真实评论","description":"","etl":"","value":"","remark":""}',
+ vote_num_1 bigint comment '{"chs_name":"前1日真实点赞","description":"","etl":"","value":"","remark":""}',
+ favor_num_1 bigint comment '{"chs_name":"前1日收藏","description":"","etl":"","value":"","remark":""}',
+ share_num_1 bigint comment '{"chs_name":"前1日转发","description":"","etl":"","value":"","remark":""}',
+ avg_page_stay_1 double comment '{"chs_name":"前1日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
+
+ click_pv_3 bigint comment '{"chs_name":"前3日点击","description":"","etl":"","value":"","remark":""}',
+ exp_pv_3 bigint comment '{"chs_name":"前3日曝光","description":"","etl":"","value":"","remark":""}',
+ page_pv_3 bigint comment '{"chs_name":"前3日浏览pv","description":"","etl":"","value":"","remark":""}',
+ page_pv_20s_3 bigint comment '{"chs_name":"前3日浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
+ reply_num_3 bigint comment '{"chs_name":"前3日真实评论","description":"","etl":"","value":"","remark":""}',
+ vote_num_3 bigint comment '{"chs_name":"前3日真实点赞","description":"","etl":"","value":"","remark":""}',
+ favor_num_3 bigint comment '{"chs_name":"前3日收藏","description":"","etl":"","value":"","remark":""}',
+ share_num_3 bigint comment '{"chs_name":"前3日转发","description":"","etl":"","value":"","remark":""}',
+ avg_page_stay_3 double comment '{"chs_name":"前3日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
+
+ click_pv_10 bigint comment '{"chs_name":"前10日点击","description":"","etl":"","value":"","remark":""}',
+ exp_pv_10 bigint comment '{"chs_name":"前10日曝光","description":"","etl":"","value":"","remark":""}',
+ page_pv_10 bigint comment '{"chs_name":"前10日浏览pv","description":"","etl":"","value":"","remark":""}',
+ page_pv_20s_10 bigint comment '{"chs_name":"前10日浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
+ reply_num_10 bigint comment '{"chs_name":"前10日真实评论","description":"","etl":"","value":"","remark":""}',
+ vote_num_10 bigint comment '{"chs_name":"前10日真实点赞","description":"","etl":"","value":"","remark":""}',
+ favor_num_10 bigint comment '{"chs_name":"前10日收藏","description":"","etl":"","value":"","remark":""}',
+ share_num_10 bigint comment '{"chs_name":"前10日转发","description":"","etl":"","value":"","remark":""}',
+ avg_page_stay_10 double comment '{"chs_name":"前10日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
+
+ click_pv bigint comment '{"chs_name":"历史点击","description":"","etl":"","value":"","remark":""}',
+ exp_pv bigint comment '{"chs_name":"历史曝光","description":"","etl":"","value":"","remark":""}',
+ page_pv bigint comment '{"chs_name":"历史浏览pv","description":"","etl":"","value":"","remark":""}',
+ page_pv_20s bigint comment '{"chs_name":"历史浏览20s以上pv","description":"","etl":"","value":"","remark":""}',
+ reply_num bigint comment '{"chs_name":"历史真实评论","description":"","etl":"","value":"","remark":""}',
+ vote_num bigint comment '{"chs_name":"历史真实点赞","description":"","etl":"","value":"","remark":""}',
+ favor_num bigint comment '{"chs_name":"历史收藏","description":"","etl":"","value":"","remark":""}',
+ share_num bigint comment '{"chs_name":"历史转发","description":"","etl":"","value":"","remark":""}',
+ avg_page_stay double comment '{"chs_name":"历史平均阅读时长(s)","description":"","etl":"","value":"","remark":""}'
+)comment '新手精选帖日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_userpost/etl/daily_userpost.sql

@@ -9,7 +9,7 @@ SET mapred.reduce.tasks=20;
 SET role admin;
 
 
-INSERT OVERWRITE TABLE pm.tl_pm_userpost_d PARTITION (PARTITION_DAY = ${partition_day})
+INSERT OVERWRITE TABLE pm.tl_pm_userpost_d_v2 PARTITION (PARTITION_DAY = ${partition_day})
 SELECT t1.id as post_id
         ,title
         ,audit_date
@@ -17,6 +17,7 @@ SELECT t1.id as post_id
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then exp_pv end),0) as exp_pv_1
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then click_pv end),0) as click_pv_1
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then page_pv end),0) as page_pv_1
+        ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then page_pv_20s end),0) as page_pv_20s_1
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then reply_num end),0) as reply_num_1
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then vote_num end),0) as vote_num_1
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,1) then favor_num end),0) as favor_num_1
@@ -26,6 +27,7 @@ SELECT t1.id as post_id
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then exp_pv end),0) as exp_pv_3
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then click_pv end),0) as click_pv_3
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then page_pv end),0) as page_pv_3
+        ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then page_pv_20s end),0) as page_pv_20s_3
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then reply_num end),0) as reply_num_3
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then vote_num end),0) as vote_num_3
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,3) then favor_num end),0) as favor_num_3
@@ -35,6 +37,7 @@ SELECT t1.id as post_id
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then exp_pv end),0) as exp_pv_10
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then click_pv end),0) as click_pv_10
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then page_pv end),0) as page_pv_10
+        ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then page_pv_20s end),0) as page_pv_20s_10
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then reply_num end),0) as reply_num_10
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then vote_num end),0) as vote_num_10
         ,nvl(sum(case when t2.partition_date>=DATE_SUB(current_date,10) then favor_num end),0) as favor_num_10
@@ -44,6 +47,7 @@ SELECT t1.id as post_id
         ,nvl(sum(exp_pv),0) as exp_pv
         ,nvl(sum(click_pv),0) as click_pv
         ,nvl(sum(page_pv),0) as page_pv
+        ,nvl(sum(page_pv_20s),0) as page_pv_20s
         ,nvl(sum(reply_num),0) as reply_num
         ,nvl(sum(vote_num),0) as vote_num
         ,nvl(sum(favor_num),0) as favor_num
@@ -94,6 +98,7 @@ join
             ,exp_pv
             ,click_pv
             ,page_pv
+            ,page_pv_20s
             ,reply_num
             ,vote_num
             ,favor_num
@@ -147,7 +152,9 @@ join
         on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
         full join
         (--浏览pv
-            SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
+            SELECT partition_date,cl_id,params['business_id'] as business_id
+                    ,count(distinct time_str) as page_pv
+                    ,count(distinct case when page_stay>=20 then time_str end) as page_pv_20s
             FROM  online.bl_hdfs_maidian_updates
             WHERE partition_date >= '20160101'
             and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
@@ -237,4 +244,4 @@ join
 )t2
 on t1.id= t2.card_id
 where t2.partition_date>=t1.audit_date
-group by t1.id,title,audit_date,tag_list
+group by t1.id,title,audit_date,tag_list;
\ No newline at end of file

pm/daily_userpost/report/data_by_day.sql

@@ -10,6 +10,23 @@ SELECT partition_day as `日期`
 		,nvl(sum(exp_pv_10),0) as `前10日曝光`
 		,nvl(round(avg(avg_page_stay_10),2),0) as `前10日平均阅读时长(s)`
 FROM pm.tl_pm_userpost_d
-where partition_day>='20200911' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+where partition_day>='20200911' and partition_day<='20200922'
 group by partition_day
-order by `日期`;
+
\ No newline at end of file
+union all
+
+SELECT partition_day as `日期`
+        ,count(post_id) as `帖子数量`
+		,nvl(concat(round(sum(click_pv_1)/sum(exp_pv_1)*100,2),'%'),0) as `前1日ctr`
+		,nvl(sum(exp_pv_1),0) as `前1日曝光`
+		,nvl(round(avg(avg_page_stay_1),2),0) as `前1日平均阅读时长(s)`
+		,nvl(concat(round(sum(click_pv_3)/sum(exp_pv_3)*100,2),'%'),0) as `前3日ctr`
+		,nvl(sum(exp_pv_3),0) as `前3日曝光`
+		,nvl(round(avg(avg_page_stay_3),2),0) as `前3日平均阅读时长(s)`
+		,nvl(concat(round(sum(click_pv_10)/sum(exp_pv_10)*100,2),'%'),0) as `前10日ctr`
+		,nvl(sum(exp_pv_10),0) as `前10日曝光`
+		,nvl(round(avg(avg_page_stay_10),2),0) as `前10日平均阅读时长(s)`
+FROM pm.tl_pm_userpost_d_v2
+where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+group by partition_day
+order by `日期`
\ No newline at end of file

pm/daily_userpost/report/detail_data.sql

@@ -5,9 +5,11 @@ SELECT post_id as `帖子id`
 		,tag_list as `所有关联标签`
 		,nvl(concat(round((nvl(reply_num_1,0)+nvl(vote_num_1,0)+nvl(favor_num_1,0)+nvl(share_num_1,0))/page_pv_1*100,2),'%'),0) as `前1日互动率`
 		,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
+		,nvl(concat(round(page_pv_20s_1/page_pv_1*100,2),'%'),0) as `前1日浏览20s以上pv/前1日总浏览pv`
 		,nvl(click_pv_1,0) as `前1日点击`
 		,nvl(exp_pv_1,0) as `前1日曝光`
 		,nvl(page_pv_1,0) as `前1日浏览pv`
+		,nvl(page_pv_20s_1,0) as `前1日浏览20s以上pv`
 		,nvl(reply_num_1,0) as `前1日真实评论`
 		,nvl(vote_num_1,0) as `前1日真实点赞`
 		,nvl(favor_num_1,0) as `前1日收藏`
@@ -16,9 +18,11 @@ SELECT post_id as `帖子id`
 
 		,nvl(concat(round((nvl(reply_num_3,0)+nvl(vote_num_3,0)+nvl(favor_num_3,0)+nvl(share_num_3,0))/page_pv_3*100,2),'%'),0) as `前3日互动率`
 		,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
+		,nvl(concat(round(page_pv_20s_3/page_pv_3*100,2),'%'),0) as `前3日浏览20s以上pv/前3日总浏览pv`
 		,nvl(click_pv_3,0) as `前3日点击`
 		,nvl(exp_pv_3,0) as `前3日曝光`
 		,nvl(page_pv_3,0) as `前3日浏览pv`
+		,nvl(page_pv_20s_3,0) as `前3日浏览20s以上pv`
 		,nvl(reply_num_3,0) as `前3日真实评论`
 		,nvl(vote_num_3,0) as `前3日真实点赞`
 		,nvl(favor_num_3,0) as `前3日收藏`
@@ -27,9 +31,11 @@ SELECT post_id as `帖子id`
 
 		,nvl(concat(round((nvl(reply_num_10,0)+nvl(vote_num_10,0)+nvl(favor_num_10,0)+nvl(share_num_10,0))/page_pv_10*100,2),'%'),0) as `前10日互动率`
 		,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
+		,nvl(concat(round(page_pv_20s_10/page_pv_10*100,2),'%'),0) as `前10日浏览20s以上pv/前10日总浏览pv`
 		,nvl(click_pv_10,0) as `前10日点击`
 		,nvl(exp_pv_10,0) as `前10日曝光`
 		,nvl(page_pv_10,0) as `前10日浏览pv`
+		,nvl(page_pv_20s_10,0) as `前10日浏览20s以上pv`
 		,nvl(reply_num_10,0) as `前10日真实评论`
 		,nvl(vote_num_10,0) as `前10日真实点赞`
 		,nvl(favor_num_10,0) as `前10日收藏`
@@ -38,15 +44,17 @@ SELECT post_id as `帖子id`
 
 		,nvl(concat(round((nvl(reply_num,0)+nvl(vote_num,0)+nvl(favor_num,0)+nvl(share_num,0))/page_pv*100,2),'%'),0) as `历史互动率`
 		,nvl(concat(round(click_pv/exp_pv*100,2),'%'),0) as `历史ctr`
+		,nvl(concat(round(page_pv_20s/page_pv*100,2),'%'),0) as `历史浏览20s以上pv/历史总浏览pv`
 		,nvl(click_pv,0) as `历史点击`
 		,nvl(exp_pv,0) as `历史曝光`
 		,nvl(page_pv,0) as `历史浏览pv`
+		,nvl(page_pv_20s,0) as `历史浏览20s以上pv`
 		,nvl(reply_num,0) as `历史真实评论`
 		,nvl(vote_num,0) as `历史真实点赞`
 		,nvl(favor_num,0) as `历史收藏`
 		,nvl(share_num,0) as `历史转发`
 		,nvl(avg_page_stay,0) as `历史平均阅读时长(s)`
-FROM pm.tl_pm_userpost_d
+FROM pm.tl_pm_userpost_d_v2
 where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
 order by `前10日曝光` desc
 

pm/daily_userpost/report/key_data.sql

@@ -4,15 +4,18 @@ SELECT post_id as `帖子id`
 		,audit_date as `审核日期`
 		,tag_list as `所有关联标签`
 		,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
+		,nvl(concat(round(page_pv_20s_1/page_pv_1*100,2),'%'),0) as `前1日浏览20s以上pv/前1日总浏览pv`
 		,nvl(exp_pv_1,0) as `前1日曝光`
 		,nvl(avg_page_stay_1,0) as `前1日平均阅读时长(s)`
 		,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
+		,nvl(concat(round(page_pv_20s_3/page_pv_3*100,2),'%'),0) as `前3日浏览20s以上pv/前3日总浏览pv`
 		,nvl(exp_pv_3,0) as `前3日曝光`
 		,nvl(avg_page_stay_3,0) as `前3日平均阅读时长(s)`
 		,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
+		,nvl(concat(round(page_pv_20s_10/page_pv_10*100,2),'%'),0) as `前10日浏览20s以上pv/前10日总浏览pv`
 		,nvl(exp_pv_10,0) as `前10日曝光`
 		,nvl(avg_page_stay_10,0) as `前10日平均阅读时长(s)`
-FROM pm.tl_pm_userpost_d
+FROM pm.tl_pm_userpost_d_v2
 where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
 order by `前10日曝光` desc