Skip to content

B
bi-report
Commit 9fd06823 authored by 赵建伟's avatar 赵建伟
Browse files
Options

Merge branch 'weiyimin' into 'master' 

Weiyimin

See merge request !20
parents 08277965 ca7669e6
Hide whitespace changes
Inline Side-by-side
Showing with 54 additions and 28 deletions
pm/daily_content_data/job/step2.job
View file @ 9fd06823
#step2.job #step2.job
type=command type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
command=/home/bi/bi-report/lib/shell/hive daily_content_data command=sh /home/bi/bi-report/lib/shell/hive daily_content_data
\ No newline at end of file \ No newline at end of file
pm/daily_push/etl/create_push.sql
View file @ 9fd06823
...@@ -80,7 +80,19 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d ...@@ -80,7 +80,19 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d
auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}', auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}', auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}', auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}' auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
location_receive_uv BIGINT comment '{"chs_name":"地狱push拉起设备数","description":"","etl":"","value":"","remark":""}',
location_receive_pv BIGINT comment '{"chs_name":"地狱push拉起人次","description":"","etl":"","value":"","remark":""}',
location_push_open_uv BIGINT comment '{"chs_name":"地狱push拉起设备数","description":"","etl":"","value":"","remark":""}',
location_push_open_pv BIGINT comment '{"chs_name":"地狱push拉起人次","description":"","etl":"","value":"","remark":""}',
location_push_open_2_uv BIGINT comment '{"chs_name":"地狱push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
location_push_open_2_pv BIGINT comment '{"chs_name":"地狱push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
qa_receive_uv BIGINT comment '{"chs_name":"邀请问答push拉起设备数","description":"","etl":"","value":"","remark":""}',
qa_receive_pv BIGINT comment '{"chs_name":"邀请问答push拉起人次","description":"","etl":"","value":"","remark":""}',
qa_push_open_uv BIGINT comment '{"chs_name":"邀请问答push拉起设备数","description":"","etl":"","value":"","remark":""}',
qa_push_open_pv BIGINT comment '{"chs_name":"邀请问答push拉起人次","description":"","etl":"","value":"","remark":""}',
qa_push_open_2_uv BIGINT comment '{"chs_name":"邀请问答push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
qa_push_open_2_pv BIGINT comment '{"chs_name":"邀请问答push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
)comment 'push日报' )comment 'push日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期') PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED ROW FORMAT DELIMITED
......
pm/daily_push/etl/daily_push.sql
View file @ 9fd06823
...@@ -8,8 +8,7 @@ SET mapred.reduce.tasks=20; ...@@ -8,8 +8,7 @@ SET mapred.reduce.tasks=20;
SET role admin; SET role admin;
--push点击数据 --push点击数据
INSERT OVERWRITE TABLE pm.tl_pm_push_d PARTITION (PARTITION_DAY = ${partition_day}) INSERT OVERWRITE TABLE pm.tl_pm_push_detail_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT NVL(click.partition_date,received.partition_date) as day_id SELECT NVL(click.partition_date,received.partition_date) as day_id
,NVL(click.device_os_type,received.device_os_type) as device_os_type ,NVL(click.device_os_type,received.device_os_type) as device_os_type
...@@ -57,25 +56,37 @@ SELECT NVL(click.partition_date,received.partition_date) as day_id ...@@ -57,25 +56,37 @@ SELECT NVL(click.partition_date,received.partition_date) as day_id
,ctr_push_open_uv ,ctr_push_open_uv
,ctr_push_open_pv ,ctr_push_open_pv
,ctr_push_open_2_uv ,ctr_push_open_2_uv
,ctr_push_open_2_uv ,ctr_push_open_2_pv
,non_ctr_receive_uv ,non_ctr_receive_uv
,non_ctr_receive_pv ,non_ctr_receive_pv
,non_ctr_push_open_uv ,non_ctr_push_open_uv
,non_ctr_push_open_pv ,non_ctr_push_open_pv
,non_ctr_push_open_2_uv ,non_ctr_push_open_2_uv
,non_ctr_push_open_2_uv ,non_ctr_push_open_2_pv
,AI_receive_uv ,AI_receive_uv
,AI_receive_pv ,AI_receive_pv
,AI_push_open_pv ,AI_push_open_pv
,AI_push_open_uv ,AI_push_open_uv
,AI_push_open_2_uv ,AI_push_open_2_uv
,AI_push_open_2_uv ,AI_push_open_2_pv
,auto_receive_uv ,auto_receive_uv
,auto_receive_pv ,auto_receive_pv
,auto_push_open_uv ,auto_push_open_uv
,auto_push_open_pv ,auto_push_open_pv
,auto_push_open_2_uv ,auto_push_open_2_uv
,auto_push_open_2_pv ,auto_push_open_2_pv
,location_receive_uv
,location_receive_pv
,location_push_open_uv
,location_push_open_pv
,location_push_open_2_uv
,location_push_open_2_pv
,qa_receive_uv
,qa_receive_pv
,qa_push_open_uv
,qa_push_open_pv
,qa_push_open_2_uv
,qa_push_open_2_pv
FROM FROM
( (
SELECT t1.partition_date SELECT t1.partition_date
...@@ -123,6 +134,14 @@ FROM ...@@ -123,6 +134,14 @@ FROM
,sum(NVL(t2.auto_push_open_pv,0)) as auto_push_open_pv --签到push拉起人次 ,sum(NVL(t2.auto_push_open_pv,0)) as auto_push_open_pv --签到push拉起人次
,count(distinct case when auto_push_open_2_pv is not null and auto_push_open_2_pv>0 then t5.cl_id end) as auto_push_open_2_uv --签到push拉起有2跳设备数 ,count(distinct case when auto_push_open_2_pv is not null and auto_push_open_2_pv>0 then t5.cl_id end) as auto_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.auto_push_open_2_pv,0)) as auto_push_open_2_pv ,sum(NVL(t5.auto_push_open_2_pv,0)) as auto_push_open_2_pv
,count(distinct case when location_push_open_pv is not null and location_push_open_pv>0 then t2.cl_id end) as location_push_open_uv --地狱push拉起设备数
,sum(NVL(t2.location_push_open_pv,0)) as location_push_open_pv --地狱push拉起人次
,count(distinct case when location_push_open_2_pv is not null and location_push_open_2_pv>0 then t5.cl_id end) as location_push_open_2_uv --地狱push拉起有2跳设备数
,sum(NVL(t5.location_push_open_2_pv,0)) as location_push_open_2_pv
,count(distinct case when qa_push_open_pv is not null and qa_push_open_pv>0 then t2.cl_id end) as qa_push_open_uv --邀请回答push拉起设备数
,sum(NVL(t2.qa_push_open_pv,0)) as qa_push_open_pv --邀请回答push拉起人次
,count(distinct case when qa_push_open_2_pv is not null and qa_push_open_2_pv>0 then t5.cl_id end) as qa_push_open_2_uv --邀请回答push拉起有2跳设备数
,sum(NVL(t5.qa_push_open_2_pv,0)) as qa_push_open_2_pv
FROM FROM
( (
select select
...@@ -178,7 +197,7 @@ FROM ...@@ -178,7 +197,7 @@ FROM
count(distinct case when push_type in ('106','107','108') then a.message_id end) as will_push_open_pv, --意愿push count(distinct case when push_type in ('106','107','108') then a.message_id end) as will_push_open_pv, --意愿push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv, --签到push count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv, --签到push
count(distinct case when push_type is not null count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.message_id end) as xitong_push_open_pv, --系统push and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.message_id end) as xitong_push_open_pv, --系统push
count(distinct case when push_type in ('93','94') then a.message_id end) as AI_push_open_pv, count(distinct case when push_type in ('93','94') then a.message_id end) as AI_push_open_pv,
count(distinct case when push_type is not null and push_type not in ('70','71') then a.message_id end) as auto_push_open_pv --自动push count(distinct case when push_type is not null and push_type not in ('70','71') then a.message_id end) as auto_push_open_pv --自动push
FROM FROM
...@@ -248,7 +267,7 @@ FROM ...@@ -248,7 +267,7 @@ FROM
count(distinct case when push_type in ('106','107','108') then a.app_session_id end) as will_push_open_2_pv, --意愿push count(distinct case when push_type in ('106','107','108') then a.app_session_id end) as will_push_open_2_pv, --意愿push
count(distinct case when push_type in ('25') then a.app_session_id end) as sign_push_open_2_pv, --签到push count(distinct case when push_type in ('25') then a.app_session_id end) as sign_push_open_2_pv, --签到push
count(distinct case when push_type is not null count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.app_session_id end) as xitong_push_open_2_pv, --系统push and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.app_session_id end) as xitong_push_open_2_pv, --系统push
count(distinct case when push_type in ('93','94') then a.app_session_id end) as AI_push_open_2_pv, --签到push count(distinct case when push_type in ('93','94') then a.app_session_id end) as AI_push_open_2_pv, --签到push
count(distinct case when push_type is not null and push_type not in ('70','71') then a.app_session_id end) as auto_push_open_2_pv --自动push count(distinct case when push_type is not null and push_type not in ('70','71') then a.app_session_id end) as auto_push_open_2_pv --自动push
FROM FROM
...@@ -359,7 +378,7 @@ full join ...@@ -359,7 +378,7 @@ full join
,count(distinct case when push_type in ('106','107','108') then a.device_id end) as will_receive_uv --意愿push ,count(distinct case when push_type in ('106','107','108') then a.device_id end) as will_receive_uv --意愿push
,count(distinct case when push_type in ('25') then a.device_id end) as sign_receive_uv --签到push ,count(distinct case when push_type in ('25') then a.device_id end) as sign_receive_uv --签到push
,count(distinct case when push_type is not null ,count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.device_id end) as xitong_receive_uv --系统push and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.device_id end) as xitong_receive_uv --系统push
,count(distinct case when push_type in ('93','94') then a.device_id end) as AI_receive_uv ,count(distinct case when push_type in ('93','94') then a.device_id end) as AI_receive_uv
,count(distinct case when push_type is not null and push_type not in ('70','71') then a.device_id end) as auto_receive_uv --自动push ,count(distinct case when push_type is not null and push_type not in ('70','71') then a.device_id end) as auto_receive_uv --自动push
,count(a.device_id) as push_receive_pv ,count(a.device_id) as push_receive_pv
...@@ -372,7 +391,7 @@ full join ...@@ -372,7 +391,7 @@ full join
,count(case when push_type in ('106','107','108') then a.device_id end) as will_receive_pv --意愿push ,count(case when push_type in ('106','107','108') then a.device_id end) as will_receive_pv --意愿push
,count(case when push_type in ('25') then a.device_id end) as sign_receive_pv --签到push ,count(case when push_type in ('25') then a.device_id end) as sign_receive_pv --签到push
,count(case when push_type is not null ,count(case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.device_id end) as xitong_receive_pv --系统push and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.device_id end) as xitong_receive_pv --系统push
,count(case when push_type in ('93','94') then a.device_id end) as AI_receive_pv ,count(case when push_type in ('93','94') then a.device_id end) as AI_receive_pv
,count(case when push_type is not null and push_type not in ('70','71') then a.device_id end) as auto_receive_pv --自动push ,count(case when push_type is not null and push_type not in ('70','71') then a.device_id end) as auto_receive_pv --自动push
...@@ -381,7 +400,7 @@ full join ...@@ -381,7 +400,7 @@ full join
select partition_day as partition_date select partition_day as partition_date
,device_id,msg_id,'ios' as device_os_type ,device_id,msg_id,'ios' as device_os_type
from bl.bl_et_bg_trackingpushlog_inc_d from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>='20200521' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','') where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by partition_day,device_id,msg_id group by partition_day,device_id,msg_id
union all union all
...@@ -391,7 +410,7 @@ full join ...@@ -391,7 +410,7 @@ full join
( (
SELECT partition_date,registration_id,platform,mesid SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表 FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>='20200521' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','') where partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
--and notification_state='true'--用户开启通知 --and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执 and callback_type = '0'--送达回执
and platform='android' and platform='android'
......
pm/daily_push/job/step2.job
View file @ 9fd06823
#step2.job #step2.job
type=command type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_push command=sh /home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file \ No newline at end of file
pm/daily_push/report/daily_push_click.sql
View file @ 9fd06823
...@@ -25,7 +25,7 @@ select day_id as `日期` ...@@ -25,7 +25,7 @@ select day_id as `日期`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_uv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳设备/push开启设备数` ,NVL(CONCAT(ROUND(non_ctr_push_open_2_uv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳设备/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳人次/push开启设备数` ,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳人次/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/non_ctr_push_open_pv*100,2),'%'),0) as `非ctr push二跳率=2跳人次/拉起人次` ,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/non_ctr_push_open_pv*100,2),'%'),0) as `非ctr push二跳率=2跳人次/拉起人次`
from pm.tl_pm_push_d from pm.tl_pm_push_detail_d
where partition_day>='20200708' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','') where partition_day>='20200708' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>='20200626' and day_id>='20200626'
and active_type='老活' and active_type='老活'
......
pm/daily_recommend_strategy/job/step2.job
View file @ 9fd06823
#step2.job #step2.job
type=command type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy command=sh /home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file \ No newline at end of file
pm/daily_recommend_strategy/job/step3.job
View file @ 9fd06823
#step3.job #step3.job
type=command type=command
dependencies=step2 dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_recommend_strategy_d command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file \ No newline at end of file
pm/daily_recommend_strategy/job/step4.job deleted 100644 → 0
View file @ 08277965
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
pm/search_related/job/step2.job
View file @ 9fd06823
#step2.job #step2.job
type=command type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
command=/home/bi/bi-report/lib/shell/hive search_related command=sh /home/bi/bi-report/lib/shell/hive search_related
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment