Skip to content

B
bi-report
Commit 7f7d52cf authored by 赵建伟's avatar 赵建伟
Browse files
Options

Merge branch 'weiyimin' into 'master' 

Weiyimin

See merge request !25
parents dc0a590c deb9c561
Show whitespace changes
Inline Side-by-side
Showing with 402 additions and 97 deletions
pm/daily_recommend_strategy/en-cn.properties
View file @ 7f7d52cf
daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file
daily_recommend_strategy=首页推荐策略及内容页ctr日报
home_recommend_strategy=首页推荐策略日报
content_ctr=内容详情页灰度ctr
\ No newline at end of file
pm/daily_recommend_strategy/etl/create_content_grey.sql 0 → 100644
View file @ 7f7d52cf
--***************************************************************
--*脚本名称:
--*功能: 首页推荐策略日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_contentpage_ctr
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
page_name string comment '{"chs_name":"页面名称","description":"","etl":"","value":"","remark":""}',
content_pv bigint comment '{"chs_name":"内容详情页pv","description":"","etl":"","value":"","remark":""}',
content_uv bigint comment '{"chs_name":"内容详情页uv","description":"","etl":"","value":"","remark":""}',
wel_exp_pv bigint comment '{"chs_name":"你可能喜欢美购卡片曝光pv","description":"","etl":"","value":"","remark":""}',
content_exp_pv bigint comment '{"chs_name":"你可能喜欢内容卡片曝光pv","description":"","etl":"","value":"","remark":""}',
wel_click_pv bigint comment '{"chs_name":"你可能喜欢美购卡片点击pv","description":"","etl":"","value":"","remark":""}',
content_click_pv bigint comment '{"chs_name":"你可能喜欢内容卡片点击pv","description":"","etl":"","value":"","remark":""}',
slide_wel_click_pv bigint comment '{"chs_name":"横划卡片点击pv","description":"","etl":"","value":"","remark":""}',
self_wel_click_pv bigint comment '{"chs_name":"关联卡片点击pv","description":"","etl":"","value":"","remark":""}'
)comment '内容详情页卡片ctr'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_recommend_strategy/etl/create_daily_recommend_strategy.sql
View file @ 7f7d52cf
......@@ -14,94 +14,25 @@ SET mapreduce.job.queuename=data;
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_content_d
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
is_ai_channel string comment '{"chs_name":"是否AI渠道","description":"","etl":"","value":"","remark":""}',
dau int comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
content_uv int comment '{"chs_name":"内容详情页uv","description":"","etl":"","value":"","remark":""}',
content_pv int comment '{"chs_name":"内容详情页PV","description":"","etl":"","value":"","remark":""}',
per_content_uv string comment '{"chs_name":"内容详情页uv/dau","description":"","etl":"","value":"","remark":""}',
per_content_pv string comment '{"chs_name":"内容详情页pv/dau","description":"","etl":"","value":"","remark":""}',
retention_1 string comment '{"chs_name":"内容用户APP次留","description":"","etl":"","value":"","remark":""}',
retention_7 string comment '{"chs_name":"内容用户APP7留","description":"","etl":"","value":"","remark":""}',
retention_30 string comment '{"chs_name":"内容用户APP30留","description":"","etl":"","value":"","remark":""}',
avg_app_duration double comment '{"chs_name":"内容用户单设备App时长(m)","description":"","etl":"","value":"","remark":""}',
avg_content_stay double comment '{"chs_name":"内容用户单设备内容时长(m)","description":"","etl":"","value":"","remark":""}',
avg_open_times double comment '{"chs_name":"内容用户单设备打开次数","description":"","etl":"","value":"","remark":""}',
search_related_stay double comment '{"chs_name":"内容用户搜索相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
welfare_stay double comment '{"chs_name":"内容用户美购详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
content_question_stay double comment '{"chs_name":"内容用户问题详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
ai_related_stay double comment '{"chs_name":"内容用户AI相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
content_diary_stay double comment '{"chs_name":"内容用户日记详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
home_stay double comment '{"chs_name":"内容用户首页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
conv_related_stay double comment '{"chs_name":"内容用户咨询相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
recommend_rate double comment '{"chs_name":"首页feeds推荐进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_recommend_pv double comment '{"chs_name":"首页feeds推荐进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
feeds_rate double comment '{"chs_name":"首页feeds非推荐进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_feeds_pv double comment '{"chs_name":"首页feeds非推荐进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
search_rate double comment '{"chs_name":"搜索进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_search_pv double comment '{"chs_name":"搜索进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
zone_rate double comment '{"chs_name":"内容聚合页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_zone_pv double comment '{"chs_name":"内容聚合页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
content_rate double comment '{"chs_name":"内容详情页推荐板块进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_from_content_pv double comment '{"chs_name":"内容详情页推荐板块进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
blank_rate double comment '{"chs_name":"无来源页面(大多数为push)进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_blank_pv double comment '{"chs_name":"无来源页面(大多数为push)进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
comment_rate double comment '{"chs_name":"评论列表页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_comment_pv double comment '{"chs_name":"评论列表页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
org_rate double comment '{"chs_name":"医生医院主页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_org_pv double comment '{"chs_name":"医生医院主页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
category_rate double comment '{"chs_name":"品类聚合页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_category_pv double comment '{"chs_name":"品类聚合页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
my_diary_rate double comment '{"chs_name":"我的日记页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_my_diary_pv double comment '{"chs_name":"我的日记页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
ai_rate double comment '{"chs_name":"AI报告页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_ai_pv double comment '{"chs_name":"AI报告页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
create_topic_num int comment '{"chs_name":"真实发帖数","description":"","etl":"","value":"","remark":""}',
create_reply_num int comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
diary_uv int comment '{"chs_name":"日记UV","description":"","etl":"","value":"","remark":""}',
diary_pv int comment '{"chs_name":"日记PV","description":"","etl":"","value":"","remark":""}',
diary_rate double comment '{"chs_name":"日记UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_diary_pv double comment '{"chs_name":"日记PV/日记UV","description":"","etl":"","value":"","remark":""}',
diary_stay double comment '{"chs_name":"日记单设备时长(m)","description":"","etl":"","value":"","remark":""}',
post_uv int comment '{"chs_name":"帖子UV","description":"","etl":"","value":"","remark":""}',
post_pv int comment '{"chs_name":"帖子PV","description":"","etl":"","value":"","remark":""}',
post_rate double comment '{"chs_name":"帖子UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_post_pv double comment '{"chs_name":"帖子PV/帖子UV","description":"","etl":"","value":"","remark":""}',
post_stay double comment '{"chs_name":"帖子单设备时长(m)","description":"","etl":"","value":"","remark":""}',
question_uv int comment '{"chs_name":"问题UV","description":"","etl":"","value":"","remark":""}',
question_pv int comment '{"chs_name":"问题PV","description":"","etl":"","value":"","remark":""}',
question_rate double comment '{"chs_name":"问题UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_question_pv double comment '{"chs_name":"问题PV/问题UV","description":"","etl":"","value":"","remark":""}',
question_stay double comment '{"chs_name":"问题单设备时长(m)","description":"","etl":"","value":"","remark":""}',
question_answer_uv int comment '{"chs_name":"问答UV","description":"","etl":"","value":"","remark":""}',
question_answer_pv int comment '{"chs_name":"问答PV","description":"","etl":"","value":"","remark":""}',
question_answer_rate double comment '{"chs_name":"问答UV/问答UV","description":"","etl":"","value":"","remark":""}',
per_question_answer_pv double comment '{"chs_name":"问答PV/问答UV","description":"","etl":"","value":"","remark":""}',
question_answer_stay double comment '{"chs_name":"问答单设备时长(m)","description":"","etl":"","value":"","remark":""}',
answer_uv int comment '{"chs_name":"回答UV","description":"","etl":"","value":"","remark":""}',
answer_pv int comment '{"chs_name":"回答PV","description":"","etl":"","value":"","remark":""}',
answer_rate double comment '{"chs_name":"回答UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_answer_pv double comment '{"chs_name":"回答PV/回答UV","description":"","etl":"","value":"","remark":""}',
answer_stay double comment '{"chs_name":"回答单设备时长(m)","description":"","etl":"","value":"","remark":""}',
video_uv int comment '{"chs_name":"视频UV","description":"","etl":"","value":"","remark":""}',
video_pv int comment '{"chs_name":"视频PV","description":"","etl":"","value":"","remark":""}',
video_rate double comment '{"chs_name":"视频UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_video_pv double comment '{"chs_name":"视频PV/视频UV","description":"","etl":"","value":"","remark":""}',
video_stay double comment '{"chs_name":"视频单设备时长(m)","description":"","etl":"","value":"","remark":""}',
wiki_uv int comment '{"chs_name":"百科UV","description":"","etl":"","value":"","remark":""}',
wiki_pv int comment '{"chs_name":"百科PV","description":"","etl":"","value":"","remark":""}',
wiki_rate double comment '{"chs_name":"百科UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_wiki_pv double comment '{"chs_name":"百科PV/百科UV","description":"","etl":"","value":"","remark":""}',
wiki_stay double comment '{"chs_name":"百科单设备时长(m)","description":"","etl":"","value":"","remark":""}',
article_uv int comment '{"chs_name":"专栏UV","description":"","etl":"","value":"","remark":""}',
article_pv int comment '{"chs_name":"专栏PV","description":"","etl":"","value":"","remark":""}',
article_rate double comment '{"chs_name":"专栏UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_article_pv double comment '{"chs_name":"专栏PV/专栏UV","description":"","etl":"","value":"","remark":""}',
article_stay double comment '{"chs_name":"专栏单设备时长(m)","description":"","etl":"","value":"","remark":""}'
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
card_click bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
card_exposure bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
navbar_search bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
highlight_word bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
self_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_content_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_special_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
transfer_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
video_consultation bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}'
)comment '内容日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ 7f7d52cf
......@@ -7,7 +7,9 @@ set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';--进制转换函数，支持各种进制转换
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT
......@@ -25,8 +27,8 @@ SELECT
NVL(sum(recom_wel_pv),0)-NVL(sum(self_wel_pv),0) as recommend_welfare_card,--需要排除关联的商品卡片点击
NVL(sum(recom_content_pv),0) as recommend_content_card,
NULL as recommend_special_card,
NULL as transfer_card,
NULL as video_consultation
NVL(sum(referral_pv),0) as transfer_card,
NVL(sum(video_pv),0) as video_consultation
FROM
(
SELECT partition_date
......@@ -310,6 +312,54 @@ FROM
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(--转诊点击，视频面诊点击
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as video_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button'
and params['button_name']='video_interview'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t10
on t10.partition_date=t3.partition_date
and t10.cl_id=t3.cl_id
and t10.business_id=t3.card_id
and t10.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as referral_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button'
and params['button_name']='referral'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t11
on t11.partition_date=t3.partition_date
and t11.cl_id=t3.cl_id
and t11.business_id=t3.card_id
and t11.page_name=t3.card_content_type
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
......@@ -376,3 +426,265 @@ WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT mas.partition_date day_id
,device_os_type
,active_type
,grey_type
,case when t1.page_name in ('post_detail','user_post_detail','doctor_post_detail') then '帖子详情页'
when t1.page_name in ('diary_detail') then '日记本详情页'
when t1.page_name in ('topic_detail') then '日记帖详情页'
when t1.page_name in ('question_answer_detail') then '问答详情页' end as page_name
,nvl(sum(t1.pv),0) as content_pv
,nvl(count(distinct t1.cl_id),0) as content_uv
,nvl(sum(t4.pv),0) as wel_exp_pv
,nvl(sum(t5.pv),0) as content_exp_pv
,nvl(sum(t2.pv),0) as wel_click_pv
,nvl(sum(t3.pv),0) as content_click_pv
,nvl(sum(t6.pv),0) as slide_wel_click_pv
,nvl(sum(t7.pv),0) as self_wel_click_pv
FROM
(
SELECT
partition_date,device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,CASE WHEN substr(convup(setencryption(device_id,'sha-1'),16,10),-1) in ('0','1','2','3','4') THEN '灰度' ELSE '非灰' END AS grey_type
FROM online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
join
(
SELECT partition_date,cl_id,page_name,count(cl_id) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
AND action='page_view'
AND int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
left join
(--你可能喜欢美购卡片点击
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type']='service'
and params['in_page_pos']='bottom'
and params['tab_name'] is null--与横划区域区分
and ((params['position'] is not null and cl_type='android')
or (params['card_but_pos'] is null and cl_type='ios')) --与关联卡片区分
AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
AND int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t2
on t1.partition_date=t2.partition_date
and t1.cl_id=t2.cl_id
and t1.page_name=t2.page_name
left join
(--你可能喜欢内容卡片点击
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type']<>'service'
and params['in_page_pos']='bottom'
and params['tab_name'] is null
AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
AND int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t3
on t1.partition_date=t3.partition_date
and t1.cl_id=t3.cl_id
and t1.page_name=t3.page_name
left join
(--你可能喜欢美购卡片曝光
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
select partition_date,cl_id,page_name,business_id,card_id,app_session_id
from online.ml_community_precise_exposure_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
and card_content_type='service'
and action ='page_precise_exposure'
and params['tab_name']='feed_recommend'
and params['in_page_pos']='bottom'
and int(split(app_version,'\\.')[1]) >= 30
and is_exposure='1'
group by partition_date,cl_id,page_name,business_id,card_id,app_session_id
)a
group by partition_date,cl_id,page_name
)t4
on t1.partition_date=t4.partition_date
and t1.cl_id=t4.cl_id
and t1.page_name=t4.page_name
left join
(--你可能喜欢内容卡片曝光
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
select partition_date,cl_id,page_name,business_id,card_id,app_session_id
from online.ml_community_precise_exposure_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
and card_content_type in ('diary','user_post','answer')
and action ='page_precise_exposure'
and params['tab_name']='feed_recommend'
and params['in_page_pos']='bottom'
and int(split(app_version,'\\.')[1]) >= 30
and is_exposure='1'
group by partition_date,cl_id,page_name,business_id,card_id,app_session_id
)a
group by partition_date,cl_id,page_name
)t5
on t1.partition_date=t5.partition_date
and t1.cl_id=t5.cl_id
and t1.page_name=t5.page_name
left join
(--横划美购卡片点击
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
and (( action='on_click_button' and params['button_name'] = 'more_recommendations')
or (action ='on_click_card' and params['in_page_pos']='recommand' and params['card_content_type']='service')
or (action ='on_click_card' and params['tab_name']='hot_recommendation' and params['card_content_type']='service'))
and int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t6
on t1.partition_date=t6.partition_date
and t1.cl_id=t6.cl_id
and t1.page_name=t6.page_name
left join
(--关联卡片
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top'
or (params['in_page_pos']='bottom' and params['position'] is null and cl_type='android')
or (params['in_page_pos']='bottom' and params['card_but_pos'] is not null and cl_type='ios'))
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name in ('diary_detail','topic_detail')
AND params['position'] is null
and int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t7
on t1.partition_date=t7.partition_date
and t1.cl_id=t7.cl_id
and t1.page_name=t7.page_name
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=mas.device_id
LEFT JOIN
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)t1
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
)dev
on mas.partition_date=dev.partition_date and mas.device_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
group by mas.partition_date
,device_os_type,active_type,grey_type
,case when t1.page_name in ('post_detail','user_post_detail','doctor_post_detail') then '帖子详情页'
when t1.page_name in ('diary_detail') then '日记本详情页'
when t1.page_name in ('topic_detail') then '日记帖详情页'
when t1.page_name in ('question_answer_detail') then '问答详情页' end
order by 1,2,3,4,5;
pm/daily_recommend_strategy/report/content_ctr.sql 0 → 100644
View file @ 7f7d52cf
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
grey_type as `灰度`,
page_name as `页面`,
NVL(CONCAT(ROUND(wel_click_pv/wel_exp_pv*100,2),'%'),0) as `内容页你可能喜欢美购卡片点击PV/该类卡片曝光PV`,
NVL(CONCAT(ROUND(content_click_pv/content_exp_pv*100,2),'%'),0) as `内容页你可能喜欢内容卡片点击PV/该类卡片曝光PV`,
content_pv as `内容详情页pv`,
content_uv as `内容详情页uv`,
wel_exp_pv as `你可能喜欢美购卡片曝光pv`,
content_exp_pv as `你可能喜欢内容卡片曝光pv`,
wel_click_pv as `你可能喜欢美购卡片点击pv`,
content_click_pv as `你可能喜欢内容卡片点击pv`,
slide_wel_click_pv as `横划卡片点击pv`,
self_wel_click_pv as `关联卡片点击pv`
FROM pm.tl_pm_contentpage_ctr
WHERE partition_day>='20200730' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by 1 desc,2,3,4,5
\ No newline at end of file
pm/daily_recommend_strategy/report/daily_recommend_strategy.sql → pm/daily_recommend_strategy/report/home_recommend_strategy.sql
View file @ 7f7d52cf
......@@ -4,12 +4,12 @@ SELECT
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)as `有效二跳pv`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
......@@ -17,8 +17,7 @@ SELECT
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
transfer_card as `来自I的转诊点击pv`,
video_consultation as `来自I的视频面诊点击pv`FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/meigou_detail_page/job/step2.job
View file @ 7f7d52cf
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,dailiang@igengmei.com,zhanghaoyu@igengmei.com,cp-sunyinghe@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment