Skip to content

B
bi-report
Commit 7df42ff4 authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent 6e148cd0
Hide whitespace changes
Inline Side-by-side
Showing with 116 additions and 14 deletions
pm/daily_push/etl/create_push_receive.sql
View file @ 7df42ff4
......@@ -21,9 +21,9 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_push_receive_d
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
push_type BIGINT comment '{"chs_name":"push类型","description":"","etl":"","value":"","remark":""}',
received_dev_num BIGINT comment '{"chs_name":"push接收设备数","description":"","etl":"","value":"","remark":""}',
received_num BIGINT comment '{"chs_name":"push接收人次","description":"","etl":"","value":"","remark":""}',
received_msg_num BIGINT comment '{"chs_name":"push接收消息数","description":"","etl":"","value":"","remark":""}',
click_dev_num BIGINT comment '{"chs_name":"push点击设备数","description":"","etl":"","value":"","remark":""}',
click_num BIGINT comment '{"chs_name":"push点击人次","description":"","etl":"","value":"","remark":""}'
click_msg_num BIGINT comment '{"chs_name":"push点击消息数","description":"","etl":"","value":"","remark":""}'
)comment 'push接收数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/daily_push/etl/daily_push_data.sql → pm/daily_push/etl/daily_push.sql
View file @ 7df42ff4
......@@ -8,7 +8,7 @@ SET mapred.reduce.tasks=20;
SET role admin;
--push点击数据
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY = ${partition_day})
INSERT OVERWRITE TABLE pm.tl_pm_push_click_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_date as day_id
,device_os_type
......@@ -143,15 +143,16 @@ LEFT JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='20190801' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
where partition_date>='20190801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)a
left join
(
SELECT substr(jiguang_id,2) as msg_id
SELECT if(substr(jiguang_id,1,1)=',',substr(jiguang_id,2),jiguang_id) as msg_id
FROM online.tl_hdfs_api_pushtask2_view --mars后台消息推送表
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') >= '20200629'
AND regexp_replace(substr(pushtime,1,10),'-','') <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND approver_status = 'true' --审核状态，true表示审核通过
AND status = '2' --推送状态，2表示发送成功，1表示准备发送
AND creator_id in ('32018153','32719681','32264790','31779749','31436904')
......@@ -210,15 +211,16 @@ LEFT JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='20190801' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
where partition_date>='20190801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)a
left join
(
SELECT substr(jiguang_id,2) as msg_id
SELECT if(substr(jiguang_id,1,1)=',',substr(jiguang_id,2),jiguang_id) as msg_id
FROM online.tl_hdfs_api_pushtask2_view --mars后台消息推送表
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') >= '20200629'
AND regexp_replace(substr(pushtime,1,10),'-','') <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND approver_status = 'true' --审核状态，true表示审核通过
AND status = '2' --推送状态，2表示发送成功，1表示准备发送
AND creator_id in ('32018153','32719681','32264790','31779749','31436904')
......@@ -272,3 +274,94 @@ GROUP BY t1.partition_date,device_os_type,active_type
order by day_id,device_os_type,active_type;
--push接收数据
INSERT OVERWRITE TABLE pm.tl_pm_push_receive_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_day as day_id
,t1.device_os_type as device_os_type
,'老活' as active_type
,t1.push_type as push_type
,count(distinct t1.device_id) as received_dev_num
,count(t1.msg_id) as received_msg_num
,count(distinct t2.cl_id) as click_dev_num
,count(t2.msg_id) as click_msg_num
FROM
(--接收设备数
SELECT partition_day
,device_os_type
,push_type
,a.device_id
,a.msg_id
FROM
(
select partition_day
,device_id,msg_id
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by partition_day,device_id,msg_id
)a
JOIN
(
select device_id,device_os_type
from online.ml_device_history_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
on a.device_id=b.device_id
JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
union all
SELECT msg_id,regexp_replace(labels['event'], '\\s+', '') AS push_type,time_stamp
FROM online.tl_hdfs_push2_task_view
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND labels['event_type'] = 'push'
group by msg_id,regexp_replace(labels['event'], '\\s+', ''),time_stamp
)c
on a.msg_id=c.msg_id
left join
(
select
partition_date
,device_id
from online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','3')
and device_os_type='ios'
)d
on a.partition_day=d.partition_date and a.device_id=d.device_id
where d.device_id is null
group by partition_day
,device_os_type
,push_type
,a.device_id
,a.msg_id
)t1
left join
(--点击设备数
select partition_date
,cl_id,params['message_id'] as msg_id,time_stamp
from online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='notification_open'
group by partition_date,cl_id,params['message_id'],time_stamp
)t2
on t2.partition_date=t1.partition_day
and t2.msg_id=t1.msg_id
and t2.cl_id=t1.device_id
group by t1.partition_day
,t1.device_os_type
,t1.push_type;
\ No newline at end of file
pm/daily_push/job/step1_6.job 0 → 100644
View file @ 7df42ff4
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/daily_push/job/step1_7.job 0 → 100644
View file @ 7df42ff4
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive bl bl_et_bg_trackingpushlog_inc_d
\ No newline at end of file
pm/daily_push/job/step2.job
View file @ 7df42ff4
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7
command=/home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file
pm/daily_push/job/step3.job
View file @ 7df42ff4
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_push/weiyimin@igengmei.com/zhaojianwei@igengmei.com
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment