Commit a4ee2822 authored by 魏艺敏's avatar 魏艺敏

push codes

parent f21b3690
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaoyang@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
daily_userpost=精华帖日报
\ No newline at end of file
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_vote_view
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_vote_view
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_favor_view
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_favor_view
\ No newline at end of file
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_mp_api_tractate_tag_v3
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_mp_api_tractate_tag_v3
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_3_0_view
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ML_D_CT_DV_DEVICECLEAN_DIMEN_D
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ML_D_CT_UI_USERCLEAN_DIMEN_D
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,zhaoyang@igengmei.com,duanyingrong@igengmei.com,wanglidan@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
--帖子关联标签 3315
SELECT t1.id as `帖子id`
,content as `帖子标题`
,t1.create_date as `上线日期`
,t1.tag_list as `所有关联标签`
,nvl(concat(round((nvl(t2.reply_num,0)+nvl(t2.vote_num,0)+nvl(t2.favor_num,0)+nvl(t2.share_num,0))/t2.page_pv*100,2),'%'),0) as `今日互动率`
,nvl(concat(round(t2.click_pv/t2.exp_pv*100,2),'%'),0) as `今日ctr`
,nvl(t2.click_pv,0) as `今日点击pv`
,nvl(t2.exp_pv,0) as `今日曝光pv`
,nvl(t2.page_pv,0) as `今日浏览pv`
,nvl(t2.reply_num,0) as `今日真实评论数`
,nvl(t2.vote_num,0) as `今日真实点赞数`
,nvl(t2.favor_num,0) as `今日收藏数`
,nvl(t2.share_num,0) as `今日转发数`
,nvl(t2.avg_page_stay,0) as `今日平均阅读时长(s)`
,nvl(concat(round((nvl(t3.reply_num,0)+nvl(t3.vote_num,0)+nvl(t3.favor_num,0)+nvl(t3.share_num,0))/t3.page_pv*100,2),'%'),0) as `3日互动率`
,nvl(concat(round(t3.click_pv/t3.exp_pv*100,2),'%'),0) as `3日ctr`
,nvl(t3.click_pv,0) as `3日点击pv`
,nvl(t3.exp_pv,0) as `3日曝光pv`
,nvl(t3.page_pv,0) as `3日浏览pv`
,nvl(t3.reply_num,0) as `3日真实评论数`
,nvl(t3.vote_num,0) as `3日真实点赞数`
,nvl(t3.favor_num,0) as `3日收藏数`
,nvl(t3.share_num,0) as `3日转发数`
,nvl(t3.avg_page_stay,0) as `3日平均阅读时长(s)`
,nvl(concat(round((nvl(t5.reply_num,0)+nvl(t5.vote_num,0)+nvl(t5.favor_num,0)+nvl(t5.share_num,0))/t5.page_pv*100,2),'%'),0) as `10日互动率`
,nvl(concat(round(t5.click_pv/t5.exp_pv*100,2),'%'),0) as `10日ctr`
,nvl(t5.click_pv,0) as `10日点击pv`
,nvl(t5.exp_pv,0) as `10日曝光pv`
,nvl(t5.page_pv,0) as `10日浏览pv`
,nvl(t5.reply_num,0) as `10日真实评论数`
,nvl(t5.vote_num,0) as `10日真实点赞数`
,nvl(t5.favor_num,0) as `10日收藏数`
,nvl(t5.share_num,0) as `10日转发数`
,nvl(t5.avg_page_stay,0) as `10日平均阅读时长(s)`
,nvl(concat(round((nvl(t4.reply_num,0)+nvl(t4.vote_num,0)+nvl(t4.favor_num,0)+nvl(t4.share_num,0))/t4.page_pv*100,2),'%'),0) as `历史互动率`
,nvl(concat(round(t4.click_pv/t4.exp_pv*100,2),'%'),0) as `历史ctr`
,nvl(t4.click_pv,0) as `历史点击pv`
,nvl(t4.exp_pv,0) as `历史曝光pv`
,nvl(t4.page_pv,0) as `历史浏览pv`
,nvl(t4.reply_num,0) as `历史真实评论数`
,nvl(t4.vote_num,0) as `历史真实点赞数`
,nvl(t4.favor_num,0) as `历史收藏数`
,nvl(t4.share_num,0) as `历史转发数`
,nvl(t4.avg_page_stay,0) as `历史平均阅读时长(s)`
FROM
(
select a.id,content,a.create_date,collect_set(d.name) as tag_list
from
(
select id,content,user_id,substr(create_time,1,10) as create_date
from online.tl_hdfs_api_tractate_view
where partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and content_level>=3
and is_online='true'
)a
join
(
select distinct tractate_id
from tl.tl_mp_api_tractate_tag_v3
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and tag_v3_id='3315'
)b
on a.id=b.tractate_id
join
(
select tractate_id,tag_v3_id
from tl.tl_mp_api_tractate_tag_v3
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by tractate_id,tag_v3_id
)c
on a.id=c.tractate_id
left join
(
select id,name
from online.tl_hdfs_api_tag_3_0_view
where partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,name
)d
on d.id=c.tag_v3_id
group by a.id,content,a.create_date
)t1
left join
(
SELECT t1.card_id
,exp_pv
,click_pv
,page_pv
,reply_num
,vote_num
,favor_num
,share_num
,avg_page_stay
from
(
select a.card_id
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,round(sum(page_stay)/count(distinct e.cl_id,e.partition_date),2) as avg_page_stay
from
(--曝光
select partition_date,card_id,cl_id,count(distinct app_session_id) as exp_pv
from online.ml_community_precise_exposure_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
group by partition_date,cl_id,card_id
)a
left join
(--点击
SELECT partition_date,params['card_id'] as card_id,cl_id,count(distinct app_session_id,cl_id) as click_pv
from online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
GROUP BY partition_date,params['card_id'],cl_id
)b
on a.partition_date=b.partition_date and a.card_id=b.card_id and a.cl_id=b.cl_id
left join
(--平均阅读时长
SELECT partition_date,business_id,cl_id,page_stay
from
(
SELECT partition_date,cl_id,params['business_id'] as business_id,page_stay,time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_stay>=0 and page_stay<1000
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id'],page_stay,time_str
)a
)e
on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
left join
(--浏览pv
SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id']
)f
on a.partition_date=f.partition_date and a.card_id=f.business_id and a.cl_id=f.cl_id
left join
(
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (IS_MORE_USER = 'true'
OR IS_STAFF = 'true'
OR IS_SPAM_CHANNEL = 'true'
OR IS_SUSPICIOUS = 'true'
OR IS_ASSOCIATED_ISSUEUSER = 'ture')
)c
on a.cl_id=c.device_id
left join
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
)d
on a.cl_id=d.device_id and a.partition_date=d.partition_date
where c.device_id is null and d.device_id is null
group by a.card_id
)t1
left join
(
SELECT a.tractate_id
,sum(case when type='reply' then num end) as reply_num
,sum(case when type='vote' then num end) as vote_num
,sum(case when type='favor' then num end) as favor_num
,sum(case when type='share' then num end) as share_num
from
( --真实评论数
SELECT tractate_id,create_date,a.user_id,'reply' as type,sum(reply_num) as num
from
(--评论数
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(substr(create_time,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
GROUP by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
group by tractate_id,create_date,a.user_id
union all
--真实点赞数
SELECT tractate_id,create_date,a.user_id,'vote' as type,sum(vote_num) as num
FROM
(
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
UNION ALL
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_reply_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '点赞'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
GROUP BY tractate_id,create_date,a.user_id
union all
--真实收藏数
SELECT tractate_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,user_id,'favor' as type,count(distinct create_time) as num
FROM online.tl_hdfs_api_tractate_favor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by user_id,tractate_id,regexp_replace(substr(create_time,1,10),'-','')
union all
--点击分享数
SELECT params['business_id'] as tractate_id,partition_date as create_date,user_id,'share' as type,count(distinct time_str) as num
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_click_share'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by params['business_id'],partition_date,user_id
)a
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
where b.user_id is null
group by a.tractate_id
)t2
on t1.card_id=t2.tractate_id
)t2
on t1.id= t2.card_id
left join
(
SELECT t1.card_id
,exp_pv
,click_pv
,page_pv
,reply_num
,vote_num
,favor_num
,share_num
,avg_page_stay
from
(
select a.card_id
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,round(sum(page_stay)/count(distinct e.cl_id,e.partition_date),2) as avg_page_stay
from
(--曝光
select partition_date,card_id,cl_id,count(distinct app_session_id) as exp_pv
from online.ml_community_precise_exposure_detail
where partition_date>=regexp_replace(DATE_SUB(current_date,3) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
group by partition_date,cl_id,card_id
)a
left join
(--点击
SELECT partition_date,params['card_id'] as card_id,cl_id,count(distinct app_session_id,cl_id) as click_pv
from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,3) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
GROUP BY partition_date,params['card_id'],cl_id
)b
on a.partition_date=b.partition_date and a.card_id=b.card_id and a.cl_id=b.cl_id
left join
(--平均阅读时长
SELECT partition_date,business_id,cl_id,page_stay
from
(
SELECT partition_date,cl_id,params['business_id'] as business_id,page_stay,time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,3) ,'-','')
and action='page_view'
and page_stay>=0 and page_stay<1000
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id'],page_stay,time_str
)a
)e
on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
left join
(--浏览pv
SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,3) ,'-','')
and action='page_view'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id']
)f
on a.partition_date=f.partition_date and a.card_id=f.business_id and a.cl_id=f.cl_id
left join
(
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (IS_MORE_USER = 'true'
OR IS_STAFF = 'true'
OR IS_SPAM_CHANNEL = 'true'
OR IS_SUSPICIOUS = 'true'
OR IS_ASSOCIATED_ISSUEUSER = 'ture')
)c
on a.cl_id=c.device_id
left join
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,3) ,'-','')
)a
join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
)d
on a.cl_id=d.device_id and a.partition_date=d.partition_date
where c.device_id is null and d.device_id is null
group by a.card_id
)t1
left join
(
SELECT a.tractate_id
,sum(case when type='reply' then num end) as reply_num
,sum(case when type='vote' then num end) as vote_num
,sum(case when type='favor' then num end) as favor_num
,sum(case when type='share' then num end) as share_num
from
( --真实评论数
SELECT tractate_id,create_date,a.user_id,'reply' as type,sum(reply_num) as num
from
(--评论数
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,3) ,'-','')
GROUP by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,3) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
group by tractate_id,create_date,a.user_id
union all
--真实点赞数
SELECT tractate_id,create_date,a.user_id,'vote' as type,sum(vote_num) as num
FROM
(
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,3) ,'-','')
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
UNION ALL
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_reply_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,3) ,'-','')
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,3) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '点赞'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
GROUP BY tractate_id,create_date,a.user_id
union all
--真实收藏数
SELECT tractate_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,user_id,'favor' as type,count(distinct create_time) as num
FROM online.tl_hdfs_api_tractate_favor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >=regexp_replace(DATE_SUB(current_date,3) ,'-','')
group by user_id,tractate_id,regexp_replace(substr(create_time,1,10),'-','')
union all
--点击分享数
SELECT params['business_id'] as tractate_id,partition_date as create_date,user_id,'share' as type,count(distinct time_str) as num
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,3) ,'-','')
and action='page_click_share'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by params['business_id'],partition_date,user_id
)a
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
where b.user_id is null
group by a.tractate_id
)t2
on t1.card_id=t2.tractate_id
)t3
on t1.id= t3.card_id
left join
(
SELECT t1.card_id
,exp_pv
,click_pv
,page_pv
,reply_num
,vote_num
,favor_num
,share_num
,avg_page_stay
from
(
select a.card_id
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,round(sum(page_stay)/count(distinct e.cl_id,e.partition_date),2) as avg_page_stay
from
(--曝光
select partition_date,card_id,cl_id,count(distinct app_session_id) as exp_pv
from online.ml_community_precise_exposure_detail
where partition_date>='20160101' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
group by partition_date,cl_id,card_id
)a
left join
(--点击
SELECT partition_date,params['card_id'] as card_id,cl_id,count(distinct app_session_id,cl_id) as click_pv
from online.bl_hdfs_maidian_updates
WHERE partition_date>='20160101' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
GROUP BY partition_date,params['card_id'],cl_id
)b
on a.partition_date=b.partition_date and a.card_id=b.card_id and a.cl_id=b.cl_id
left join
(--平均阅读时长
SELECT partition_date,business_id,cl_id,page_stay
from
(
SELECT partition_date,cl_id,params['business_id'] as business_id,page_stay,time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_stay>=0 and page_stay<1000
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id'],page_stay,time_str
)a
)e
on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
left join
(--浏览pv
SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id']
)f
on a.partition_date=f.partition_date and a.card_id=f.business_id and a.cl_id=f.cl_id
left join
(
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (IS_MORE_USER = 'true'
OR IS_STAFF = 'true'
OR IS_SPAM_CHANNEL = 'true'
OR IS_SUSPICIOUS = 'true'
OR IS_ASSOCIATED_ISSUEUSER = 'ture')
)c
on a.cl_id=c.device_id
left join
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
)d
on a.cl_id=d.device_id and a.partition_date=d.partition_date
where c.device_id is null and d.device_id is null
group by a.card_id
)t1
left join
(
SELECT a.tractate_id
,sum(case when type='reply' then num end) as reply_num
,sum(case when type='vote' then num end) as vote_num
,sum(case when type='favor' then num end) as favor_num
,sum(case when type='share' then num end) as share_num
from
( --真实评论数
SELECT tractate_id,create_date,a.user_id,'reply' as type,sum(reply_num) as num
from
(--评论数
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>='20160101'
AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
group by tractate_id,create_date,a.user_id
union all
--真实点赞数
SELECT tractate_id,create_date,a.user_id,'vote' as type,sum(vote_num) as num
FROM
(
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
UNION ALL
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_reply_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '20200101'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>='20160101'
AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '点赞'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
GROUP BY tractate_id,create_date,a.user_id
union all
--真实收藏数
SELECT tractate_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,user_id,'favor' as type,count(distinct create_time) as num
FROM online.tl_hdfs_api_tractate_favor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
-- and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
-- and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by user_id,tractate_id,regexp_replace(substr(create_time,1,10),'-','')
union all
--点击分享数
SELECT params['business_id'] as tractate_id,partition_date as create_date,user_id,'share' as type,count(distinct time_str) as num
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20160101'
and partition_date <=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_click_share'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by params['business_id'],partition_date,user_id
)a
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
where b.user_id is null
group by a.tractate_id
)t2
on t1.card_id=t2.tractate_id
)t4
on t1.id= t4.card_id
left join
(
SELECT t1.card_id
,exp_pv
,click_pv
,page_pv
,reply_num
,vote_num
,favor_num
,share_num
,avg_page_stay
from
(
select a.card_id
,sum(exp_pv) as exp_pv
,sum(click_pv) as click_pv
,sum(page_pv) as page_pv
,round(sum(page_stay)/count(distinct e.cl_id,e.partition_date),2) as avg_page_stay
from
(--曝光
select partition_date,card_id,cl_id,count(distinct app_session_id) as exp_pv
from online.ml_community_precise_exposure_detail
where partition_date>=regexp_replace(DATE_SUB(current_date,10) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
group by partition_date,cl_id,card_id
)a
left join
(--点击
SELECT partition_date,params['card_id'] as card_id,cl_id,count(distinct app_session_id,cl_id) as click_pv
from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,10) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
GROUP BY partition_date,params['card_id'],cl_id
)b
on a.partition_date=b.partition_date and a.card_id=b.card_id and a.cl_id=b.cl_id
left join
(--平均阅读时长
SELECT partition_date,business_id,cl_id,page_stay
from
(
SELECT partition_date,cl_id,params['business_id'] as business_id,page_stay,time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,10) ,'-','')
and action='page_view'
and page_stay>=0 and page_stay<1000
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id'],page_stay,time_str
)a
)e
on a.partition_date=e.partition_date and a.card_id=e.business_id and a.cl_id=e.cl_id
left join
(--浏览pv
SELECT partition_date,cl_id,params['business_id'] as business_id,count(distinct time_str) as page_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,10) ,'-','')
and action='page_view'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by partition_date,cl_id,params['business_id']
)f
on a.partition_date=f.partition_date and a.card_id=f.business_id and a.cl_id=f.cl_id
left join
(
select distinct device_id
from ML.ML_D_CT_DV_DEVICECLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (IS_MORE_USER = 'true'
OR IS_STAFF = 'true'
OR IS_SPAM_CHANNEL = 'true'
OR IS_SUSPICIOUS = 'true'
OR IS_ASSOCIATED_ISSUEUSER = 'ture')
)c
on a.cl_id=c.device_id
left join
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,3) ,'-','')
)a
join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
)d
on a.cl_id=d.device_id and a.partition_date=d.partition_date
where c.device_id is null and d.device_id is null
group by a.card_id
)t1
left join
(
SELECT a.tractate_id
,sum(case when type='reply' then num end) as reply_num
,sum(case when type='vote' then num end) as vote_num
,sum(case when type='favor' then num end) as favor_num
,sum(case when type='share' then num end) as share_num
from
( --真实评论数
SELECT tractate_id,create_date,a.user_id,'reply' as type,sum(reply_num) as num
from
(--评论数
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,10) ,'-','')
GROUP by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,10) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
group by tractate_id,create_date,a.user_id
union all
--真实点赞数
SELECT tractate_id,create_date,a.user_id,'vote' as type,sum(vote_num) as num
FROM
(
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,10) ,'-','')
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
UNION ALL
SELECT tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,count(distinct create_time) as vote_num
FROM online.tl_hdfs_api_tractate_reply_vote_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(DATE_SUB(current_date,10) ,'-','')
group by tractate_id,user_id,regexp_replace(substr(create_time,1,10),'-','')
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,10) ,'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '点赞'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
GROUP BY tractate_id,create_date,a.user_id
union all
--真实收藏数
SELECT tractate_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,user_id,'favor' as type,count(distinct create_time) as num
FROM online.tl_hdfs_api_tractate_favor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >=regexp_replace(DATE_SUB(current_date,10) ,'-','')
group by user_id,tractate_id,regexp_replace(substr(create_time,1,10),'-','')
union all
--点击分享数
SELECT params['business_id'] as tractate_id,partition_date as create_date,user_id,'share' as type,count(distinct time_str) as num
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,10) ,'-','')
and action='page_click_share'
and page_name in ('post_detail','user_post_detail','doctor_post_detail')
group by params['business_id'],partition_date,user_id
)a
left join
(
select distinct user_id
from ml.ML_D_CT_UI_USERCLEAN_DIMEN_D
where PARTITION_DAY = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and (is_doctor = 'true'
or is_puppet = 'true'
or is_classify_user = 'true'
or is_reply_fake = 'true'
or is_staff = 'true'
or is_associated_issuedevice = 'true')
)b
on a.user_id=b.user_id
where b.user_id is null
group by a.tractate_id
)t2
on t1.card_id=t2.tractate_id
)t5
on t1.id= t5.card_id
order by `3日曝光pv` desc
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,dailiang@igengmei.com,zhanghaoyu@igengmei.com,cp-sunyinghe@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,dailiang@igengmei.com,zhanghaoyu@igengmei.com,cp-sunyinghe@igengmei.com,zhaofei@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment