Skip to content

B
bi-report
Commit 040750af authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent 1715d860
Expand all Show whitespace changes
Inline Side-by-side
Showing with 199 additions and 20 deletions
pm/daily_push/en-cn.properties 0 → 100644
View file @ 040750af
daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file
pm/daily_push/etl/create_push_click.sql 0 → 100644
View file @ 040750af
--***************************************************************
--*脚本名称:
--*功能: push日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
dau BIGINT comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
is_open_uv BIGINT comment '{"chs_name":"push开启设备数","description":"","etl":"","value":"","remark":""}',
push_open_uv BIGINT comment '{"chs_name":"push拉起设备数","description":"","etl":"","value":"","remark":""}',
push_open_pv BIGINT comment '{"chs_name":"push拉起人次","description":"","etl":"","value":"","remark":""}',
push_open_2_uv BIGINT comment '{"chs_name":"push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
push_open_2_pv BIGINT comment '{"chs_name":"push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
op_push_open_uv BIGINT comment '{"chs_name":"运营push拉起设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_pv BIGINT comment '{"chs_name":"运营push拉起人次","description":"","etl":"","value":"","remark":""}',
op_push_open_2_uv BIGINT comment '{"chs_name":"运营push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_2_pv BIGINT comment '{"chs_name":"运营push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
will_push_open_uv BIGINT comment '{"chs_name":"意愿探索push拉起设备数","description":"","etl":"","value":"","remark":""}',
will_push_open_pv BIGINT comment '{"chs_name":"意愿探索push拉起人次","description":"","etl":"","value":"","remark":""}',
will_push_open_2_uv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
will_push_open_2_pv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv BIGINT comment '{"chs_name":"签到push拉起设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_pv BIGINT comment '{"chs_name":"签到push拉起人次","description":"","etl":"","value":"","remark":""}',
sign_push_open_2_uv BIGINT comment '{"chs_name":"签到push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_2_pv BIGINT comment '{"chs_name":"签到push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv BIGINT comment '{"chs_name":"个性化push拉起设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_pv BIGINT comment '{"chs_name":"个性化push拉起人次","description":"","etl":"","value":"","remark":""}',
personal_push_open_2_uv BIGINT comment '{"chs_name":"个性化push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_2_pv BIGINT comment '{"chs_name":"个性化push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
ctr_push_open_uv BIGINT comment '{"chs_name":"ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
ctr_push_open_pv BIGINT comment '{"chs_name":"ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
ctr_push_open_2_uv BIGINT comment '{"chs_name":"ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
ctr_push_open_2_pv BIGINT comment '{"chs_name":"ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_uv BIGINT comment '{"chs_name":"非ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_pv BIGINT comment '{"chs_name":"非ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_2_uv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_2_pv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
AI_push_open_uv BIGINT comment '{"chs_name":"AI_push拉起设备数","description":"","etl":"","value":"","remark":""}',
AI_push_open_pv BIGINT comment '{"chs_name":"AI_push拉起人次","description":"","etl":"","value":"","remark":""}',
AI_push_open_2_uv BIGINT comment '{"chs_name":"AI_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
AI_push_open_2_pv BIGINT comment '{"chs_name":"AI_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
orignal_auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}',
orignal_auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}',
orginal_auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
orginal_auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
)comment 'push日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/create_push_receive.sql 0 → 100644
View file @ 040750af
--***************************************************************
--*脚本名称:
--*功能: push接收数据
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_receive_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
push_type BIGINT comment '{"chs_name":"push类型","description":"","etl":"","value":"","remark":""}',
received_dev_num BIGINT comment '{"chs_name":"push接收设备数","description":"","etl":"","value":"","remark":""}',
received_num BIGINT comment '{"chs_name":"push接收人次","description":"","etl":"","value":"","remark":""}',
click_dev_num BIGINT comment '{"chs_name":"push点击设备数","description":"","etl":"","value":"","remark":""}',
click_num BIGINT comment '{"chs_name":"push点击人次","description":"","etl":"","value":"","remark":""}'
)comment 'push接收数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/daily_push_data.sql 0 → 100644
View file @ 040750af
This diff is collapsed.
pm/daily_push/job/step1_1.job 0 → 100644
View file @ 040750af
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/daily_push/job/step1_2.job 0 → 100644
View file @ 040750af
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file
pm/daily_push/job/step1_3.job 0 → 100644
View file @ 040750af
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_push/job/step1_4.job 0 → 100644
View file @ 040750af
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/daily_push/job/step1_5.job 0 → 100644
View file @ 040750af
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_pushtask2_view
\ No newline at end of file
pm/daily_push/job/step2.job 0 → 100644
View file @ 040750af
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file
pm/daily_push/job/step3.job 0 → 100644
View file @ 040750af
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com
\ No newline at end of file
pm/daily_push/report/daily_push.sql 0 → 100644
View file @ 040750af
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/daily_push/report/personal_push_click_rate.sql 0 → 100644
View file @ 040750af
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ 040750af
......@@ -37,8 +37,7 @@ FROM
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
......@@ -71,8 +70,7 @@ FROM
card_id,
app_session_id
from online.ml_community_precise_exposure_detail
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
......@@ -115,8 +113,7 @@ FROM
params['card_id'] as card_id,
app_session_id
from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
......@@ -148,8 +145,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND referrer='home'
......@@ -171,8 +167,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
......@@ -195,8 +190,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
......@@ -223,8 +217,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top' or params['in_page_pos']='bottom')
......@@ -256,8 +249,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
......@@ -286,8 +278,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
......@@ -325,8 +316,7 @@ FROM
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)t1
JOIN
( --医生账号
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment