Skip to content

B
bi-report
Commit a9ccf5ec authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent f8fe5cdf
Hide whitespace changes
Inline Side-by-side
Showing with 401 additions and 342 deletions
pm/daily_userpost/en-cn.properties
View file @ a9ccf5ec
daily_userpost=新手精选帖日报 daily_userpost=新手精选帖日报
\ No newline at end of file data_by_day=每日数据汇总
key_day=当日数据关键指标
detail_data=当日数据明细
\ No newline at end of file
pm/daily_userpost/etl/create_userpost.sql 0 → 100644
View file @ a9ccf5ec
--***************************************************************
--*脚本名称:
--*功能: 线索日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_userpost_d
(
post_id string comment '{"chs_name":"帖子id","description":"","etl":"","value":"","remark":""}',
title string comment '{"chs_name":"帖子标题","description":"","etl":"","value":"","remark":""}',
audit_date string comment '{"chs_name":"最新审核时间","description":"","etl":"","value":"","remark":""}',
tag_list string comment '{"chs_name":"关联标签","description":"","etl":"","value":"","remark":""}',
click_pv_1 bigint comment '{"chs_name":"前1日点击","description":"","etl":"","value":"","remark":""}',
exp_pv_1 bigint comment '{"chs_name":"前1日曝光","description":"","etl":"","value":"","remark":""}',
page_pv_1 bigint comment '{"chs_name":"前1日浏览pv","description":"","etl":"","value":"","remark":""}',
reply_num_1 bigint comment '{"chs_name":"前1日真实评论","description":"","etl":"","value":"","remark":""}',
vote_num_1 bigint comment '{"chs_name":"前1日真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num_1 bigint comment '{"chs_name":"前1日收藏","description":"","etl":"","value":"","remark":""}',
share_num_1 bigint comment '{"chs_name":"前1日转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay_1 double comment '{"chs_name":"前1日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
click_pv_3 bigint comment '{"chs_name":"前3日点击","description":"","etl":"","value":"","remark":""}',
exp_pv_3 bigint comment '{"chs_name":"前3日曝光","description":"","etl":"","value":"","remark":""}',
page_pv_3 bigint comment '{"chs_name":"前3日浏览pv","description":"","etl":"","value":"","remark":""}',
reply_num_3 bigint comment '{"chs_name":"前3日真实评论","description":"","etl":"","value":"","remark":""}',
vote_num_3 bigint comment '{"chs_name":"前3日真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num_3 bigint comment '{"chs_name":"前3日收藏","description":"","etl":"","value":"","remark":""}',
share_num_3 bigint comment '{"chs_name":"前3日转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay_3 double comment '{"chs_name":"前3日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
click_pv_10 bigint comment '{"chs_name":"前10日点击","description":"","etl":"","value":"","remark":""}',
exp_pv_10 bigint comment '{"chs_name":"前10日曝光","description":"","etl":"","value":"","remark":""}',
page_pv_10 bigint comment '{"chs_name":"前10日浏览pv","description":"","etl":"","value":"","remark":""}',
reply_num_10 bigint comment '{"chs_name":"前10日真实评论","description":"","etl":"","value":"","remark":""}',
vote_num_10 bigint comment '{"chs_name":"前10日真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num_10 bigint comment '{"chs_name":"前10日收藏","description":"","etl":"","value":"","remark":""}',
share_num_10 bigint comment '{"chs_name":"前10日转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay_10 double comment '{"chs_name":"前10日平均阅读时长(s)","description":"","etl":"","value":"","remark":""}',
click_pv bigint comment '{"chs_name":"历史点击","description":"","etl":"","value":"","remark":""}',
exp_pv bigint comment '{"chs_name":"历史曝光","description":"","etl":"","value":"","remark":""}',
page_pv bigint comment '{"chs_name":"历史浏览pv","description":"","etl":"","value":"","remark":""}',
reply_num bigint comment '{"chs_name":"历史真实评论","description":"","etl":"","value":"","remark":""}',
vote_num bigint comment '{"chs_name":"历史真实点赞","description":"","etl":"","value":"","remark":""}',
favor_num bigint comment '{"chs_name":"历史收藏","description":"","etl":"","value":"","remark":""}',
share_num bigint comment '{"chs_name":"历史转发","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"历史平均阅读时长(s)","description":"","etl":"","value":"","remark":""}'
)comment '新手精选帖日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_userpost/etl/daily_userpost.sql 0 → 100644
View file @ a9ccf5ec
INSERT OVERWRITE TABLE pm.tl_pm_userpost_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.id as post_id
,title
,audit_date
,tag_list
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then exp_pv end) as exp_pv_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then click_pv end) as click_pv_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then page_pv end) as page_pv_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then reply_num end) as reply_num_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then vote_num end) as vote_num_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then favor_num end) as favor_num_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then share_num end) as share_num_1
,round(avg(case when t2.partition_date>=DATE_SUB(current_date,1) then avg_page_stay end),2) as avg_page_stay_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then exp_pv end) as exp_pv_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then click_pv end) as click_pv_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then page_pv end) as page_pv_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then reply_num end) as reply_num_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then vote_num end) as vote_num_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then favor_num end) as favor_num_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then share_num end) as share_num_3
,round(avg(case when t2.partition_date>=DATE_SUB(current_date,3) then avg_page_stay end),2) as avg_page_stay_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then exp_pv end) as exp_pv_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then click_pv end) as click_pv_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then page_pv end) as page_pv_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then reply_num end) as reply_num_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then vote_num end) as vote_num_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then favor_num end) as favor_num_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then share_num end) as share_num_10
,round(avg(case when t2.partition_date>=DATE_SUB(current_date,10) then avg_page_stay end),2) as avg_page_stay_10
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,sum(reply_num) as reply_num
,sum(vote_num) as vote_num
,sum(favor_num) as favor_num
,sum(share_num) as share_num
,round(avg(avg_page_stay),2) as avg_page_stay
FROM
(
select a.id,title,a.audit_date,collect_set(d.name) as tag_list
from
(
select id,title,user_id,substr(audit_time,1,10) as audit_date
from tl.tl_mp_api_tractate
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and content_level>=3
and is_online='true'
)a
join
(
select distinct tractate_id
from tl.tl_mp_api_tractate_tag_v3
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and tag_v3_id='3315'
)b
on a.id=b.tractate_id
join
(
select tractate_id,tag_v3_id
from tl.tl_mp_api_tractate_tag_v3
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by tractate_id,tag_v3_id
)c
on a.id=c.tractate_id
left join
(
select id,name
from online.tl_hdfs_api_tag_3_0_view
where partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,name
)d
on d.id=c.tag_v3_id
group by a.id,title,a.audit_date
)t1
join
(--历史数据，指从审核时间至今的数据
SELECT nvl(concat_ws('-',substr(t1.partition_date,1,4),substr(t1.partition_date,5,2),substr(t1.partition_date,7,2))
,concat_ws('-',substr(t2.create_date,1,4),substr(t2.create_date,5,2),substr(t2.create_date,7,2))) as partition_date
,nvl(t1.card_id,t2.tractate_id) as card_id
,exp_pv
,click_pv
,page_pv
,reply_num
,vote_num
,favor_num
,share_num
,avg_page_stay
from
(
select nvl(nvl(a.card_id,e.business_id),f.business_id) as card_id
,nvl(nvl(a.partition_date,e.partition_date),f.partition_date) as partition_date
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,round(sum(page_stay)/count(distinct e.cl_id,e.partition_date),2) as avg_page_stay
from
(--曝光
select partition_date,card_id,cl_id,count(distinct app_session_id) as exp_pv
from online.ml_community_precise_exposure_detail
where partition_date>='20160101' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
group by partition_date,cl_id,card_id
)a
left join
(--点击
SELECT partition_date,params['card_id'] as card_id,cl_id,count(distinct app_session_id,cl_id) as click_pv
from online.bl_hdfs_maidian_updates
WHERE partition_date>='20160101' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
GROUP BY partition_date,params['card_id'],cl_id
)b
on a.partition_date=b.partition_date and a.card_id=b.card_id and a.cl_id=b.cl_id
full join
(--平均阅读时长
SELECT partition_date,business_id,cl_id,page_stay
from
(
SELECT partition_date,cl_id,params['business_id'] as business_id,page_stay,time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_stay>=0 and page_stay<1000
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id'],page_stay,time_str
)a
)e
on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
full join
(--浏览pv
SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id']
)f
on a.partition_date=f.partition_date and nvl(a.card_id,e.business_id)=f.business_id and nvl(a.cl_id,e.cl_id)=f.cl_id
left join
(
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_abnormal_device = 'ture'
)c
on a.cl_id=c.device_id
where c.device_id is null
group by nvl(nvl(a.card_id,e.business_id),f.business_id),nvl(nvl(a.partition_date,e.partition_date),f.partition_date)
)t1
full join
(
SELECT a.tractate_id,a.create_date
,sum(case when type='reply' then num end) as reply_num
,sum(case when type='vote' then num end) as vote_num
,sum(case when type='favor' then num end) as favor_num
,sum(case when type='share' then num end) as share_num
from
( --真实评论数
SELECT tractate_id,create_date,a.user_id,'reply' as type,sum(reply_num) as num
from
(--评论数
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
group by tractate_id,create_date,a.user_id
union all
--真实点赞数
SELECT tractate_id,create_date,a.user_id,'vote' as type,sum(vote_num) as num
FROM
(
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
GROUP BY tractate_id,create_date,a.user_id
union all
--真实收藏数
SELECT tractate_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,user_id,'favor' as type,count(distinct create_time) as num
FROM online.tl_hdfs_api_tractate_favor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by user_id,tractate_id,regexp_replace(substr(create_time,1,10),'-','')
union all
--点击分享数
SELECT params['business_id'] as tractate_id,partition_date as create_date,user_id,'share' as type,count(distinct time_str) as num
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_click_share'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by params['business_id'],partition_date,user_id
)a
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and is_abnormal_user = 'true'
)b
on a.user_id=b.user_id
where b.user_id is null
group by a.tractate_id,a.create_date
)t2
on t1.card_id=t2.tractate_id and t1.partition_date =t2.create_date
)t2
on t1.id= t2.card_id
where t2.partition_date>=t1.audit_date
group by t1.id,title,audit_date,tag_list
\ No newline at end of file
pm/daily_userpost/job/step2.job
View file @ a9ccf5ec
#step2.job #step2.job
type=command type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
command=curl -X GET http://localhost:8553/api/report/email/daily_userpost/weiyimin@igengmei.com/weiyimin@qq.com command=sh /home/bi/bi-report/lib/shell/hive daily_userpost
\ No newline at end of file \ No newline at end of file
pm/daily_userpost/job/step3.job 0 → 100644
View file @ a9ccf5ec
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_userpost/liudi@igengmei.com,zhaoyang@igengmei.com,wangxin@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com,wanglidan@igengmei.com/weiyimin@qq.com
\ No newline at end of file
pm/daily_userpost/report/daily_userpost.sql deleted 100644 → 0
View file @ f8fe5cdf
--新手精选帖子
--帖子关联标签 3315
SELECT id as `帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
,nvl(concat(round((nvl(reply_num_1,0)+nvl(vote_num_1,0)+nvl(favor_num_1,0)+nvl(share_num_1,0))/page_pv_1*100,2),'%'),0) as `前1日互动率`
,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
,nvl(click_pv_1,0) as `前1日点击`
,nvl(exp_pv_1,0) as `前1日曝光`
,nvl(page_pv_1,0) as `前1日浏览pv`
,nvl(reply_num_1,0) as `前1日真实评论`
,nvl(vote_num_1,0) as `前1日真实点赞`
,nvl(favor_num_1,0) as `前1日收藏`
,nvl(share_num_1,0) as `前1日转发`
,nvl(avg_page_stay_1,0) as `前1日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num_3,0)+nvl(vote_num_3,0)+nvl(favor_num_3,0)+nvl(share_num_3,0))/page_pv_3*100,2),'%'),0) as `前3日互动率`
,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
,nvl(click_pv_3,0) as `前3日点击`
,nvl(exp_pv_3,0) as `前3日曝光`
,nvl(page_pv_3,0) as `前3日浏览pv`
,nvl(reply_num_3,0) as `前3日真实评论`
,nvl(vote_num_3,0) as `前3日真实点赞`
,nvl(favor_num_3,0) as `前3日收藏`
,nvl(share_num_3,0) as `前3日转发`
,nvl(avg_page_stay_3,0) as `前3日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num_10,0)+nvl(vote_num_10,0)+nvl(favor_num_10,0)+nvl(share_num_10,0))/page_pv_10*100,2),'%'),0) as `前10日互动率`
,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
,nvl(click_pv_10,0) as `前10日点击`
,nvl(exp_pv_10,0) as `前10日曝光`
,nvl(page_pv_10,0) as `前10日浏览pv`
,nvl(reply_num_10,0) as `前10日真实评论`
,nvl(vote_num_10,0) as `前10日真实点赞`
,nvl(favor_num_10,0) as `前10日收藏`
,nvl(share_num_10,0) as `前10日转发`
,nvl(avg_page_stay_10,0) as `前10日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num,0)+nvl(vote_num,0)+nvl(favor_num,0)+nvl(share_num,0))/page_pv*100,2),'%'),0) as `历史互动率`
,nvl(concat(round(click_pv/exp_pv*100,2),'%'),0) as `历史ctr`
,nvl(click_pv,0) as `历史点击`
,nvl(exp_pv,0) as `历史曝光`
,nvl(page_pv,0) as `历史浏览pv`
,nvl(reply_num,0) as `历史真实评论`
,nvl(vote_num,0) as `历史真实点赞`
,nvl(favor_num,0) as `历史收藏`
,nvl(share_num,0) as `历史转发`
,nvl(avg_page_stay,0) as `历史平均阅读时长(s)`
FROM
(
SELECT t1.id,title,audit_date,tag_list
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,sum(reply_num) as reply_num
,sum(vote_num) as vote_num
,sum(favor_num) as favor_num
,sum(share_num) as share_num
,round(avg(avg_page_stay),2) as avg_page_stay
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then exp_pv end) as exp_pv_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then click_pv end) as click_pv_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then page_pv end) as page_pv_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then reply_num end) as reply_num_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then vote_num end) as vote_num_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then favor_num end) as favor_num_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,1) then share_num end) as share_num_1
,round(avg(case when t2.partition_date>=DATE_SUB(current_date,1) then avg_page_stay end),2) as avg_page_stay_1
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then exp_pv end) as exp_pv_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then click_pv end) as click_pv_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then page_pv end) as page_pv_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then reply_num end) as reply_num_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then vote_num end) as vote_num_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then favor_num end) as favor_num_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,3) then share_num end) as share_num_3
,round(avg(case when t2.partition_date>=DATE_SUB(current_date,3) then avg_page_stay end),2) as avg_page_stay_3
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then exp_pv end) as exp_pv_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then click_pv end) as click_pv_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then page_pv end) as page_pv_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then reply_num end) as reply_num_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then vote_num end) as vote_num_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then favor_num end) as favor_num_10
,sum(case when t2.partition_date>=DATE_SUB(current_date,10) then share_num end) as share_num_10
,round(avg(case when t2.partition_date>=DATE_SUB(current_date,10) then avg_page_stay end),2) as avg_page_stay_10
FROM
(
select a.id,title,a.audit_date,collect_set(d.name) as tag_list
from
(
select id,title,user_id,substr(audit_time,1,10) as audit_date
from tl.tl_mp_api_tractate
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and content_level>=3
and is_online='true'
)a
join
(
select distinct tractate_id
from tl.tl_mp_api_tractate_tag_v3
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and tag_v3_id='3315'
)b
on a.id=b.tractate_id
join
(
select tractate_id,tag_v3_id
from tl.tl_mp_api_tractate_tag_v3
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by tractate_id,tag_v3_id
)c
on a.id=c.tractate_id
left join
(
select id,name
from online.tl_hdfs_api_tag_3_0_view
where partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,name
)d
on d.id=c.tag_v3_id
group by a.id,title,a.audit_date
)t1
join
(--历史数据，指从审核时间至今的数据
SELECT nvl(concat_ws('-',substr(t1.partition_date,1,4),substr(t1.partition_date,5,2),substr(t1.partition_date,7,2))
,concat_ws('-',substr(t2.create_date,1,4),substr(t2.create_date,5,2),substr(t2.create_date,7,2))) as partition_date
,nvl(t1.card_id,t2.tractate_id) as card_id
,exp_pv
,click_pv
,page_pv
,reply_num
,vote_num
,favor_num
,share_num
,avg_page_stay
from
(
select nvl(nvl(a.card_id,e.business_id),f.business_id) as card_id
,nvl(nvl(a.partition_date,e.partition_date),f.partition_date) as partition_date
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,round(sum(page_stay)/count(distinct e.cl_id,e.partition_date),2) as avg_page_stay
from
(--曝光
select partition_date,card_id,cl_id,count(distinct app_session_id) as exp_pv
from online.ml_community_precise_exposure_detail
where partition_date>='20160101' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
group by partition_date,cl_id,card_id
)a
left join
(--点击
SELECT partition_date,params['card_id'] as card_id,cl_id,count(distinct app_session_id,cl_id) as click_pv
from online.bl_hdfs_maidian_updates
WHERE partition_date>='20160101' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
GROUP BY partition_date,params['card_id'],cl_id
)b
on a.partition_date=b.partition_date and a.card_id=b.card_id and a.cl_id=b.cl_id
full join
(--平均阅读时长
SELECT partition_date,business_id,cl_id,page_stay
from
(
SELECT partition_date,cl_id,params['business_id'] as business_id,page_stay,time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_stay>=0 and page_stay<1000
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id'],page_stay,time_str
)a
)e
on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
full join
(--浏览pv
SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id']
)f
on a.partition_date=f.partition_date and nvl(a.card_id,e.business_id)=f.business_id and nvl(a.cl_id,e.cl_id)=f.cl_id
left join
(
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_abnormal_device = 'ture'
)c
on a.cl_id=c.device_id
where c.device_id is null
group by nvl(nvl(a.card_id,e.business_id),f.business_id),nvl(nvl(a.partition_date,e.partition_date),f.partition_date)
)t1
full join
(
SELECT a.tractate_id,a.create_date
,sum(case when type='reply' then num end) as reply_num
,sum(case when type='vote' then num end) as vote_num
,sum(case when type='favor' then num end) as favor_num
,sum(case when type='share' then num end) as share_num
from
( --真实评论数
SELECT tractate_id,create_date,a.user_id,'reply' as type,sum(reply_num) as num
from
(--评论数
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
-- JOIN --限制用户是在app进行的回复
-- (
-- SELECT a.partition_date,user_id
-- FROM
-- (
-- SELECT partition_date,user_id,device_id,action
-- FROM online.bl_hdfs_operation_updates
-- WHERE partition_date>='20160101'
-- AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- )a
-- JOIN
-- (
-- SELECT code
-- FROM dim.dim_community_action_type
-- WHERE communityuserbehavior_type_name = '回帖'
-- )type
-- ON a.action = code
-- GROUP BY a.partition_date,user_id
-- )b
-- ON a.user_id = b.user_id
-- AND a.create_date = b.partition_date
group by tractate_id,create_date,a.user_id
union all
--真实点赞数
SELECT tractate_id,create_date,a.user_id,'vote' as type,sum(vote_num) as num
FROM
(
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
-- UNION ALL
-- SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
-- FROM online.tl_hdfs_api_tractate_reply_vote_view
-- WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- -- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- -- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
-- group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
-- JOIN --限制用户是在app进行的回复
-- (
-- SELECT a.partition_date,user_id
-- FROM
-- (
-- SELECT partition_date,user_id,device_id,action
-- FROM online.bl_hdfs_operation_updates
-- WHERE partition_date>='20160101'
-- AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- )a
-- JOIN
-- (
-- SELECT code
-- FROM dim.dim_community_action_type
-- WHERE communityuserbehavior_type_name = '点赞'
-- )type
-- ON a.action = code
-- GROUP BY a.partition_date,user_id
-- )b
-- ON a.user_id = b.user_id
-- AND a.create_date = b.partition_date
GROUP BY tractate_id,create_date,a.user_id
union all
--真实收藏数
SELECT tractate_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,user_id,'favor' as type,count(distinct create_time) as num
FROM online.tl_hdfs_api_tractate_favor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by user_id,tractate_id,regexp_replace(substr(create_time,1,10),'-','')
union all
--点击分享数
SELECT params['business_id'] as tractate_id,partition_date as create_date,user_id,'share' as type,count(distinct time_str) as num
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_click_share'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by params['business_id'],partition_date,user_id
)a
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and is_abnormal_user = 'true'
)b
on a.user_id=b.user_id
where b.user_id is null
group by a.tractate_id,a.create_date
)t2
on t1.card_id=t2.tractate_id and t1.partition_date =t2.create_date
)t2
on t1.id= t2.card_id
where t2.partition_date>=t1.audit_date
group by t1.id,title,audit_date,tag_list
)t4
order by `前10日曝光` desc
pm/daily_userpost/report/data_by_day.sql 0 → 100644
View file @ a9ccf5ec
SELECT partition_day as `日期`
,count(post_id) as `帖子数量`
,nvl(concat(round(sum(click_pv_1)/sum(exp_pv_1)*100,2),'%'),0) as `前1日ctr`
,nvl(sum(exp_pv_1),0) as `前1日曝光`
,nvl(round(avg(avg_page_stay_1),2),0) as `前1日平均阅读时长(s)`
,nvl(concat(round(sum(click_pv_3)/sum(exp_pv_3)*100,2),'%'),0) as `前3日ctr`
,nvl(sum(exp_pv_3),0) as `前3日曝光`
,nvl(round(avg(avg_page_stay_3),2),0) as `前3日平均阅读时长(s)`
,nvl(concat(round(sum(click_pv_10)/sum(exp_pv_10)*100,2),'%'),0) as `前10日ctr`
,nvl(sum(exp_pv_10),0) as `前10日曝光`
,nvl(round(avg(avg_page_stay_10),2),0) as `前10日平均阅读时长(s)`
FROM pm.tl_pm_userpost_d
where partition_day>='20200911' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by partition_day
order by `日期`;
\ No newline at end of file
pm/daily_userpost/report/detail_data.sql 0 → 100644
View file @ a9ccf5ec
SELECT post_id as `帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
,nvl(concat(round((nvl(reply_num_1,0)+nvl(vote_num_1,0)+nvl(favor_num_1,0)+nvl(share_num_1,0))/page_pv_1*100,2),'%'),0) as `前1日互动率`
,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
,nvl(click_pv_1,0) as `前1日点击`
,nvl(exp_pv_1,0) as `前1日曝光`
,nvl(page_pv_1,0) as `前1日浏览pv`
,nvl(reply_num_1,0) as `前1日真实评论`
,nvl(vote_num_1,0) as `前1日真实点赞`
,nvl(favor_num_1,0) as `前1日收藏`
,nvl(share_num_1,0) as `前1日转发`
,nvl(avg_page_stay_1,0) as `前1日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num_3,0)+nvl(vote_num_3,0)+nvl(favor_num_3,0)+nvl(share_num_3,0))/page_pv_3*100,2),'%'),0) as `前3日互动率`
,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
,nvl(click_pv_3,0) as `前3日点击`
,nvl(exp_pv_3,0) as `前3日曝光`
,nvl(page_pv_3,0) as `前3日浏览pv`
,nvl(reply_num_3,0) as `前3日真实评论`
,nvl(vote_num_3,0) as `前3日真实点赞`
,nvl(favor_num_3,0) as `前3日收藏`
,nvl(share_num_3,0) as `前3日转发`
,nvl(avg_page_stay_3,0) as `前3日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num_10,0)+nvl(vote_num_10,0)+nvl(favor_num_10,0)+nvl(share_num_10,0))/page_pv_10*100,2),'%'),0) as `前10日互动率`
,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
,nvl(click_pv_10,0) as `前10日点击`
,nvl(exp_pv_10,0) as `前10日曝光`
,nvl(page_pv_10,0) as `前10日浏览pv`
,nvl(reply_num_10,0) as `前10日真实评论`
,nvl(vote_num_10,0) as `前10日真实点赞`
,nvl(favor_num_10,0) as `前10日收藏`
,nvl(share_num_10,0) as `前10日转发`
,nvl(avg_page_stay_10,0) as `前10日平均阅读时长(s)`
,nvl(concat(round((nvl(reply_num,0)+nvl(vote_num,0)+nvl(favor_num,0)+nvl(share_num,0))/page_pv*100,2),'%'),0) as `历史互动率`
,nvl(concat(round(click_pv/exp_pv*100,2),'%'),0) as `历史ctr`
,nvl(click_pv,0) as `历史点击`
,nvl(exp_pv,0) as `历史曝光`
,nvl(page_pv,0) as `历史浏览pv`
,nvl(reply_num,0) as `历史真实评论`
,nvl(vote_num,0) as `历史真实点赞`
,nvl(favor_num,0) as `历史收藏`
,nvl(share_num,0) as `历史转发`
,nvl(avg_page_stay,0) as `历史平均阅读时长(s)`
FROM pm.tl_pm_userpost_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `前10日曝光` desc
pm/daily_userpost/report/key_data.sql 0 → 100644
View file @ a9ccf5ec
SELECT post_id as `帖子id`
,title as `帖子标题`
,audit_date as `审核日期`
,tag_list as `所有关联标签`
,nvl(concat(round(click_pv_1/exp_pv_1*100,2),'%'),0) as `前1日ctr`
,nvl(exp_pv_1,0) as `前1日曝光`
,nvl(avg_page_stay_1,0) as `前1日平均阅读时长(s)`
,nvl(concat(round(click_pv_3/exp_pv_3*100,2),'%'),0) as `前3日ctr`
,nvl(exp_pv_3,0) as `前3日曝光`
,nvl(avg_page_stay_3,0) as `前3日平均阅读时长(s)`
,nvl(concat(round(click_pv_10/exp_pv_10*100,2),'%'),0) as `前10日ctr`
,nvl(exp_pv_10,0) as `前10日曝光`
,nvl(avg_page_stay_10,0) as `前10日平均阅读时长(s)`
FROM pm.tl_pm_userpost_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `前10日曝光` desc
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment