Skip to content

B
bi-report
Commit 040750af authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent 1715d860
Show whitespace changes
Inline Side-by-side
Showing with 473 additions and 20 deletions
pm/daily_push/en-cn.properties 0 → 100644
View file @ 040750af
daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file
pm/daily_push/etl/create_push_click.sql 0 → 100644
View file @ 040750af
--***************************************************************
--*脚本名称:
--*功能: push日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
dau BIGINT comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
is_open_uv BIGINT comment '{"chs_name":"push开启设备数","description":"","etl":"","value":"","remark":""}',
push_open_uv BIGINT comment '{"chs_name":"push拉起设备数","description":"","etl":"","value":"","remark":""}',
push_open_pv BIGINT comment '{"chs_name":"push拉起人次","description":"","etl":"","value":"","remark":""}',
push_open_2_uv BIGINT comment '{"chs_name":"push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
push_open_2_pv BIGINT comment '{"chs_name":"push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
op_push_open_uv BIGINT comment '{"chs_name":"运营push拉起设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_pv BIGINT comment '{"chs_name":"运营push拉起人次","description":"","etl":"","value":"","remark":""}',
op_push_open_2_uv BIGINT comment '{"chs_name":"运营push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_2_pv BIGINT comment '{"chs_name":"运营push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
will_push_open_uv BIGINT comment '{"chs_name":"意愿探索push拉起设备数","description":"","etl":"","value":"","remark":""}',
will_push_open_pv BIGINT comment '{"chs_name":"意愿探索push拉起人次","description":"","etl":"","value":"","remark":""}',
will_push_open_2_uv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
will_push_open_2_pv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv BIGINT comment '{"chs_name":"签到push拉起设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_pv BIGINT comment '{"chs_name":"签到push拉起人次","description":"","etl":"","value":"","remark":""}',
sign_push_open_2_uv BIGINT comment '{"chs_name":"签到push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_2_pv BIGINT comment '{"chs_name":"签到push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv BIGINT comment '{"chs_name":"个性化push拉起设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_pv BIGINT comment '{"chs_name":"个性化push拉起人次","description":"","etl":"","value":"","remark":""}',
personal_push_open_2_uv BIGINT comment '{"chs_name":"个性化push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_2_pv BIGINT comment '{"chs_name":"个性化push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
ctr_push_open_uv BIGINT comment '{"chs_name":"ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
ctr_push_open_pv BIGINT comment '{"chs_name":"ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
ctr_push_open_2_uv BIGINT comment '{"chs_name":"ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
ctr_push_open_2_pv BIGINT comment '{"chs_name":"ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_uv BIGINT comment '{"chs_name":"非ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_pv BIGINT comment '{"chs_name":"非ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_2_uv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_2_pv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
AI_push_open_uv BIGINT comment '{"chs_name":"AI_push拉起设备数","description":"","etl":"","value":"","remark":""}',
AI_push_open_pv BIGINT comment '{"chs_name":"AI_push拉起人次","description":"","etl":"","value":"","remark":""}',
AI_push_open_2_uv BIGINT comment '{"chs_name":"AI_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
AI_push_open_2_pv BIGINT comment '{"chs_name":"AI_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
orignal_auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}',
orignal_auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}',
orginal_auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
orginal_auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
)comment 'push日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/create_push_receive.sql 0 → 100644
View file @ 040750af
--***************************************************************
--*脚本名称:
--*功能: push接收数据
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_receive_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
push_type BIGINT comment '{"chs_name":"push类型","description":"","etl":"","value":"","remark":""}',
received_dev_num BIGINT comment '{"chs_name":"push接收设备数","description":"","etl":"","value":"","remark":""}',
received_num BIGINT comment '{"chs_name":"push接收人次","description":"","etl":"","value":"","remark":""}',
click_dev_num BIGINT comment '{"chs_name":"push点击设备数","description":"","etl":"","value":"","remark":""}',
click_num BIGINT comment '{"chs_name":"push点击人次","description":"","etl":"","value":"","remark":""}'
)comment 'push接收数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/daily_push_data.sql 0 → 100644
View file @ 040750af
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--push点击数据
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_date as day_id
,device_os_type
,active_type
,count(distinct t1.device_id) as dau
,count(distinct is_open.cl_id) as is_open_uv --push开启设备数
,count(distinct t2.cl_id) as push_open_uv --push拉起设备数
,sum(NVL(t2.push_open_pv,0)) as push_open_pv --push拉起人次
,count(distinct t5.cl_id) as push_open_2_uv --push拉起有2跳设备数
,sum(case WHEN t5.cl_id is not null then push_open_2_pv END) as push_open_2_pv --push拉起有2跳人次
,count(distinct case when op_push_open_pv is not null and op_push_open_pv>0 then t2.cl_id end) as op_push_open_uv --运营push拉起设备数
,sum(NVL(t2.op_push_open_pv,0)) as op_push_open_pv --运营push拉起人次
,count(distinct case when op_push_open_2_pv is not null and op_push_open_2_pv>0 then t5.cl_id end) as op_push_open_2_uv --运营push拉起有2跳设备数
,sum(NVL(t5.op_push_open_2_pv,0)) as op_push_open_2_pv --运营push拉起有2跳人次
,count(distinct case when auto_push_open_pv is not null and auto_push_open_pv>0 then t2.cl_id end) as auto_push_open_uv --自动push拉起设备数
,sum(NVL(t2.auto_push_open_pv,0)) as auto_push_open_pv --自动push拉起人次
,count(distinct case when auto_push_open_2_pv is not null and auto_push_open_2_pv>0 then t5.cl_id end) as auto_push_open_2_uv --自动push拉起有2跳设备数
,sum(NVL(t5.auto_push_open_2_pv,0)) as auto_push_open_2_pv --自动push拉起有2跳人次
,count(distinct case when will_push_open_pv is not null and will_push_open_pv>0 then t2.cl_id end) as will_push_open_uv --意愿探索push拉起设备数
,sum(NVL(t2.will_push_open_pv,0)) as will_push_open_pv --意愿探索push拉起人次
,count(distinct case when will_push_open_2_pv is not null and will_push_open_2_pv>0 then t5.cl_id end) as will_push_open_2_uv --意愿探索push拉起有2跳设备数
,sum(NVL(t5.will_push_open_2_pv,0)) as will_push_open_2_pv
,count(distinct case when sign_push_open_pv is not null and sign_push_open_pv>0 then t2.cl_id end) as sign_push_open_uv --签到push拉起设备数
,sum(NVL(t2.sign_push_open_pv,0)) as sign_push_open_pv --签到push拉起人次
,count(distinct case when sign_push_open_2_pv is not null and sign_push_open_2_pv>0 then t5.cl_id end) as sign_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.sign_push_open_2_pv,0)) as sign_push_open_2_pv
,count(distinct case when t2.ctr_push_open_pv+t2.non_ctr_push_open_pv>0 then t2.cl_id end) as personal_push_open_uv --个性化push拉起设备数
,sum(NVL(t2.ctr_push_open_pv+t2.non_ctr_push_open_pv,0)) as personal_push_open_pv --个性化push拉起人次
,count(distinct case when t5.ctr_push_open_2_pv+t5.non_ctr_push_open_2_pv>0 then t5.cl_id end) as personal_push_open_2_uv --个性化push拉起设备数
,sum(NVL(t5.ctr_push_open_2_pv+t5.non_ctr_push_open_2_pv,0)) as personal_push_open_2_pv --个性化push拉起有2跳人次
,count(distinct case when ctr_push_open_pv is not null and ctr_push_open_pv>0 then t2.cl_id end) as ctr_push_open_uv --签到push拉起设备数
,sum(NVL(t2.ctr_push_open_pv,0)) as ctr_push_open_pv --签到push拉起人次
,count(distinct case when ctr_push_open_2_pv is not null and ctr_push_open_2_pv>0 then t5.cl_id end) as ctr_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.ctr_push_open_2_pv,0)) as ctr_push_open_2_pv
,count(distinct case when non_ctr_push_open_pv is not null and non_ctr_push_open_pv>0 then t2.cl_id end) as non_ctr_push_open_uv --签到push拉起设备数
,sum(NVL(t2.non_ctr_push_open_pv,0)) as non_ctr_push_open_pv --签到push拉起人次
,count(distinct case when non_ctr_push_open_2_pv is not null and non_ctr_push_open_2_pv>0 then t5.cl_id end) as non_ctr_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.non_ctr_push_open_2_pv,0)) as non_ctr_push_open_2_pv
,count(distinct case when AI_push_open_pv is not null and AI_push_open_pv>0 then t2.cl_id end) as AI_push_open_uv --签到push拉起设备数
,sum(NVL(t2.AI_push_open_pv,0)) as AI_push_open_pv --签到push拉起人次
,count(distinct case when AI_push_open_2_pv is not null and AI_push_open_2_pv>0 then t5.cl_id end) as AI_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.AI_push_open_2_pv,0)) as AI_push_open_2_pv
,count(distinct case when orignal_auto_push_open_pv is not null and orignal_auto_push_open_pv>0 then t2.cl_id end) as orignal_auto_push_open_uv --签到push拉起设备数
,sum(NVL(t2.orignal_auto_push_open_pv,0)) as orignal_auto_push_open_pv --签到push拉起人次
,count(distinct case when orginal_auto_push_open_2_pv is not null and orginal_auto_push_open_2_pv>0 then t5.cl_id end) as orginal_auto_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.orginal_auto_push_open_2_pv,0)) as orginal_auto_push_open_2_pv
FROM
(
select
partition_date
,device_id
,device_os_type
,case WHEN active_type = '4' THEN '老活' WHEN active_type in ('1','2') then '新增' END as active_type
from online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('4','1','2')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)t1
LEFT JOIN
(
select a.partition_date,a.cl_id --开启push设备
from
(
SELECT partition_date,cl_id,max(time_str) as max_time_str
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='is_open_push'
group by partition_date,cl_id
)a
left join
(
SELECT partition_date,
cl_id,time_str,COALESCE(params['type'],params['tyep']) as type
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='is_open_push'
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.time_str=a.max_time_str
where type=1
group by a.partition_date,a.cl_id
)is_open
on is_open.partition_date=t1.partition_date and is_open.cl_id=t1.device_id
LEFT JOIN
(
SELECT a.partition_date,a.cl_id,
count(distinct a.message_id) as push_open_pv,
count(distinct case when push_type in ('70','71') then a.message_id end) as op_push_open_pv, --运营push push_type in ('70','71')
count(distinct case when push_type in ('101','102','103','104','105') then a.message_id end) as ctr_push_open_pv, --个性化push
count(distinct case when push_type in ('111','112','113') then a.message_id end) as non_ctr_push_open_pv, --个性化push
count(distinct case when push_type in ('106','107','108') then a.message_id end) as will_push_open_pv, --意愿push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv, --签到push
count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.message_id end) as auto_push_open_pv, --自动push
count(distinct case when push_type in ('93','94') then a.message_id end) as AI_push_open_pv,
count(distinct case when push_type is not null and push_type not in ('70','71') then a.message_id end) as orignal_auto_push_open_pv --自动push
FROM
(
SELECT a.partition_date,b.cl_id,a.app_session_id,b.message_id
FROM
(
--设备开启
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,app_session_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='device_opened'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),app_session_id
)a
JOIN
(
--推送点击
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,params['message_id'] as message_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='notification_open'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),params['message_id']
)b
on a.partition_date = b.partition_date
AND a.cl_id = b.cl_id
where abs(a.time_stamp-b.time_stamp)<=2
)a
LEFT JOIN
(
SELECT a.msg_id,case when b.msg_id is not null then 'ex' else push_type end as push_type
FROM
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='20190801' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)a
left join
(
SELECT substr(jiguang_id,2) as msg_id
FROM online.tl_hdfs_api_pushtask2_view --mars后台消息推送表
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND approver_status = 'true' --审核状态，true表示审核通过
AND status = '2' --推送状态，2表示发送成功，1表示准备发送
AND creator_id in ('32018153','32719681','32264790','31779749','31436904')
)b
on a.msg_id=b.msg_id
)e
on e.msg_id=a.message_id
group by a.partition_date,a.cl_id
)t2
on t2.cl_id=t1.device_id and t2.partition_date=t1.partition_date
LEFT JOIN
(
--push拉起后有2跳的设备
SELECT partition_date,
a.cl_id,
count(distinct a.app_session_id) as push_open_2_pv,
count(distinct case when push_type in ('70','71') then a.app_session_id end) as op_push_open_2_pv, --运营push push_type in ('70','71')
count(distinct case when push_type in ('101','102','103','104','105') then a.app_session_id end) as ctr_push_open_2_pv, --个性化push
count(distinct case when push_type in ('111','112','113') then a.app_session_id end) as non_ctr_push_open_2_pv, --个性化push
count(distinct case when push_type in ('106','107','108') then a.app_session_id end) as will_push_open_2_pv, --意愿push
count(distinct case when push_type in ('25') then a.app_session_id end) as sign_push_open_2_pv, --签到push
count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.app_session_id end) as auto_push_open_2_pv, --自动push
count(distinct case when push_type in ('93','94') then a.app_session_id end) as AI_push_open_2_pv, --签到push
count(distinct case when push_type is not null and push_type not in ('70','71') then a.app_session_id end) as orginal_auto_push_open_2_pv --自动push
FROM
(
SELECT a.partition_date,b.cl_id,a.app_session_id,b.message_id
FROM
(
--设备开启
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,app_session_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='device_opened'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),app_session_id
)a
JOIN
(
--推送点击
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,params['message_id'] as message_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='notification_open'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),params['message_id']
)b
on a.partition_date = b.partition_date
AND a.cl_id = b.cl_id
where abs(a.time_stamp-b.time_stamp)<=2
)a
LEFT JOIN
(
SELECT a.msg_id,case when b.msg_id is not null then 'ex' else push_type end as push_type
FROM
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='20190801' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)a
left join
(
SELECT substr(jiguang_id,2) as msg_id
FROM online.tl_hdfs_api_pushtask2_view --mars后台消息推送表
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND approver_status = 'true' --审核状态，true表示审核通过
AND status = '2' --推送状态，2表示发送成功，1表示准备发送
AND creator_id in ('32018153','32719681','32264790','31779749','31436904')
)b
on a.msg_id=b.msg_id
)e
on e.msg_id=a.message_id
JOIN
(
--通过push拉起的一次会话中，有2跳，包括回退
SELECT a.partition_date,a.cl_id,a.app_session_id,a.cl_type
from
(
SELECT partition_date,cl_id,app_session_id,cl_type,if(page_name='home','home','non_home') as is_home
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
GROUP BY partition_date,cl_id,app_session_id,cl_type,if(page_name='home','home','non_home')
)a
join
(
SELECT partition_date,cl_id,app_session_id,cl_type
,count(distinct page_name) as page_num
,count(distinct array(time_str,page_name)) as page_array
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
GROUP BY partition_date,cl_id,app_session_id,cl_type
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.app_session_id=a.app_session_id
left join
(
select device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam
on spam.device_id=a.cl_id
where spam.device_id IS NULL and
(a.cl_type='ios' and is_home='home' and page_num>=2 and page_array>=3 or
a.cl_type='ios' and is_home='non_home' and page_array>=2 or
a.cl_type='android' and page_array>=2)
GROUP BY a.partition_date,a.cl_id,a.app_session_id,a.cl_type
)c
on a.partition_date = c.partition_date
AND a.cl_id = c.cl_id
AND a.app_session_id = c.app_session_id
GROUP BY a.cl_id,a.partition_date
)t5
on t1.partition_date=t5.partition_date and t1.device_id=t5.cl_id
GROUP BY t1.partition_date,device_os_type,active_type
order by day_id,device_os_type,active_type;
--push接收数据
pm/daily_push/job/step1_1.job 0 → 100644
View file @ 040750af
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/daily_push/job/step1_2.job 0 → 100644
View file @ 040750af
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file
pm/daily_push/job/step1_3.job 0 → 100644
View file @ 040750af
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_push/job/step1_4.job 0 → 100644
View file @ 040750af
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/daily_push/job/step1_5.job 0 → 100644
View file @ 040750af
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_pushtask2_view
\ No newline at end of file
pm/daily_push/job/step2.job 0 → 100644
View file @ 040750af
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file
pm/daily_push/job/step3.job 0 → 100644
View file @ 040750af
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com
\ No newline at end of file
pm/daily_push/report/daily_push.sql 0 → 100644
View file @ 040750af
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/daily_push/report/personal_push_click_rate.sql 0 → 100644
View file @ 040750af
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ 040750af
...@@ -37,8 +37,7 @@ FROM ...@@ -37,8 +37,7 @@ FROM
WHEN active_type IN ('1','2') THEN '新增' END AS active_type WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id ,device_id
FROM online.ml_device_day_active_status FROM online.ml_device_day_active_status
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4') AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3' AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang' ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
...@@ -71,8 +70,7 @@ FROM ...@@ -71,8 +70,7 @@ FROM
card_id, card_id,
app_session_id app_session_id
from online.ml_community_precise_exposure_detail from online.ml_community_precise_exposure_detail
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光 AND is_exposure = '1' ----精准曝光
AND page_name ='home' AND page_name ='home'
...@@ -115,8 +113,7 @@ FROM ...@@ -115,8 +113,7 @@ FROM
params['card_id'] as card_id, params['card_id'] as card_id,
app_session_id app_session_id
from online.bl_hdfs_maidian_updates from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card' AND action='on_click_card'
AND params['page_name'] ='home' AND params['page_name'] ='home'
AND params['tab_name'] = '精选' AND params['tab_name'] = '精选'
...@@ -148,8 +145,7 @@ FROM ...@@ -148,8 +145,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
sum(page_stay) as page_stay sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='page_view' AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND referrer='home' AND referrer='home'
...@@ -171,8 +167,7 @@ FROM ...@@ -171,8 +167,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as navbar_pv count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('on_click_navbar_search','do_search') AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or AND (referrer='home' or
...@@ -195,8 +190,7 @@ FROM ...@@ -195,8 +190,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as highlight_pv count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card' AND action='on_click_card'
and params['card_type']='highlight_word' and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
...@@ -223,8 +217,7 @@ FROM ...@@ -223,8 +217,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as pv count(1) as pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏' AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link' or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top' or params['in_page_pos']='bottom') or params['in_page_pos']='top' or params['in_page_pos']='bottom')
...@@ -256,8 +249,7 @@ FROM ...@@ -256,8 +249,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv count(1) as service_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (action='on_click_card'and params['card_content_type']='service' AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold') or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
...@@ -286,8 +278,7 @@ FROM ...@@ -286,8 +278,7 @@ FROM
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv count(1) as service_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card' AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer') and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
...@@ -325,8 +316,7 @@ FROM ...@@ -325,8 +316,7 @@ FROM
SELECT user_id,partition_date, SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
)t1 )t1
JOIN JOIN
( --医生账号 ( --医生账号
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment