Skip to content

B
bi-report
Commit 9fd06823 authored by 赵建伟's avatar 赵建伟
Browse files
Options

Merge branch 'weiyimin' into 'master' 

Weiyimin

See merge request !20
parents 08277965 ca7669e6
Hide whitespace changes
Inline Side-by-side
Showing with 54 additions and 28 deletions
pm/daily_content_data/job/step2.job
View file @ 9fd06823
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
command=/home/bi/bi-report/lib/shell/hive daily_content_data
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/hive daily_content_data
\ No newline at end of file
pm/daily_push/etl/create_push.sql
View file @ 9fd06823
......@@ -80,7 +80,19 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d
auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
location_receive_uv BIGINT comment '{"chs_name":"地狱push拉起设备数","description":"","etl":"","value":"","remark":""}',
location_receive_pv BIGINT comment '{"chs_name":"地狱push拉起人次","description":"","etl":"","value":"","remark":""}',
location_push_open_uv BIGINT comment '{"chs_name":"地狱push拉起设备数","description":"","etl":"","value":"","remark":""}',
location_push_open_pv BIGINT comment '{"chs_name":"地狱push拉起人次","description":"","etl":"","value":"","remark":""}',
location_push_open_2_uv BIGINT comment '{"chs_name":"地狱push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
location_push_open_2_pv BIGINT comment '{"chs_name":"地狱push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
qa_receive_uv BIGINT comment '{"chs_name":"邀请问答push拉起设备数","description":"","etl":"","value":"","remark":""}',
qa_receive_pv BIGINT comment '{"chs_name":"邀请问答push拉起人次","description":"","etl":"","value":"","remark":""}',
qa_push_open_uv BIGINT comment '{"chs_name":"邀请问答push拉起设备数","description":"","etl":"","value":"","remark":""}',
qa_push_open_pv BIGINT comment '{"chs_name":"邀请问答push拉起人次","description":"","etl":"","value":"","remark":""}',
qa_push_open_2_uv BIGINT comment '{"chs_name":"邀请问答push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
qa_push_open_2_pv BIGINT comment '{"chs_name":"邀请问答push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
)comment 'push日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/daily_push/etl/daily_push.sql
View file @ 9fd06823
......@@ -8,8 +8,7 @@ SET mapred.reduce.tasks=20;
SET role admin;
--push点击数据
INSERT OVERWRITE TABLE pm.tl_pm_push_d PARTITION (PARTITION_DAY = ${partition_day})
INSERT OVERWRITE TABLE pm.tl_pm_push_detail_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT NVL(click.partition_date,received.partition_date) as day_id
,NVL(click.device_os_type,received.device_os_type) as device_os_type
......@@ -57,25 +56,37 @@ SELECT NVL(click.partition_date,received.partition_date) as day_id
,ctr_push_open_uv
,ctr_push_open_pv
,ctr_push_open_2_uv
,ctr_push_open_2_uv
,ctr_push_open_2_pv
,non_ctr_receive_uv
,non_ctr_receive_pv
,non_ctr_push_open_uv
,non_ctr_push_open_pv
,non_ctr_push_open_2_uv
,non_ctr_push_open_2_uv
,non_ctr_push_open_2_pv
,AI_receive_uv
,AI_receive_pv
,AI_push_open_pv
,AI_push_open_uv
,AI_push_open_2_uv
,AI_push_open_2_uv
,AI_push_open_2_pv
,auto_receive_uv
,auto_receive_pv
,auto_push_open_uv
,auto_push_open_pv
,auto_push_open_2_uv
,auto_push_open_2_pv
,location_receive_uv
,location_receive_pv
,location_push_open_uv
,location_push_open_pv
,location_push_open_2_uv
,location_push_open_2_pv
,qa_receive_uv
,qa_receive_pv
,qa_push_open_uv
,qa_push_open_pv
,qa_push_open_2_uv
,qa_push_open_2_pv
FROM
(
SELECT t1.partition_date
......@@ -123,6 +134,14 @@ FROM
,sum(NVL(t2.auto_push_open_pv,0)) as auto_push_open_pv --签到push拉起人次
,count(distinct case when auto_push_open_2_pv is not null and auto_push_open_2_pv>0 then t5.cl_id end) as auto_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.auto_push_open_2_pv,0)) as auto_push_open_2_pv
,count(distinct case when location_push_open_pv is not null and location_push_open_pv>0 then t2.cl_id end) as location_push_open_uv --地狱push拉起设备数
,sum(NVL(t2.location_push_open_pv,0)) as location_push_open_pv --地狱push拉起人次
,count(distinct case when location_push_open_2_pv is not null and location_push_open_2_pv>0 then t5.cl_id end) as location_push_open_2_uv --地狱push拉起有2跳设备数
,sum(NVL(t5.location_push_open_2_pv,0)) as location_push_open_2_pv
,count(distinct case when qa_push_open_pv is not null and qa_push_open_pv>0 then t2.cl_id end) as qa_push_open_uv --邀请回答push拉起设备数
,sum(NVL(t2.qa_push_open_pv,0)) as qa_push_open_pv --邀请回答push拉起人次
,count(distinct case when qa_push_open_2_pv is not null and qa_push_open_2_pv>0 then t5.cl_id end) as qa_push_open_2_uv --邀请回答push拉起有2跳设备数
,sum(NVL(t5.qa_push_open_2_pv,0)) as qa_push_open_2_pv
FROM
(
select
......@@ -178,7 +197,7 @@ FROM
count(distinct case when push_type in ('106','107','108') then a.message_id end) as will_push_open_pv, --意愿push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv, --签到push
count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.message_id end) as xitong_push_open_pv, --系统push
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.message_id end) as xitong_push_open_pv, --系统push
count(distinct case when push_type in ('93','94') then a.message_id end) as AI_push_open_pv,
count(distinct case when push_type is not null and push_type not in ('70','71') then a.message_id end) as auto_push_open_pv --自动push
FROM
......@@ -248,7 +267,7 @@ FROM
count(distinct case when push_type in ('106','107','108') then a.app_session_id end) as will_push_open_2_pv, --意愿push
count(distinct case when push_type in ('25') then a.app_session_id end) as sign_push_open_2_pv, --签到push
count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.app_session_id end) as xitong_push_open_2_pv, --系统push
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.app_session_id end) as xitong_push_open_2_pv, --系统push
count(distinct case when push_type in ('93','94') then a.app_session_id end) as AI_push_open_2_pv, --签到push
count(distinct case when push_type is not null and push_type not in ('70','71') then a.app_session_id end) as auto_push_open_2_pv --自动push
FROM
......@@ -359,7 +378,7 @@ full join
,count(distinct case when push_type in ('106','107','108') then a.device_id end) as will_receive_uv --意愿push
,count(distinct case when push_type in ('25') then a.device_id end) as sign_receive_uv --签到push
,count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.device_id end) as xitong_receive_uv --系统push
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.device_id end) as xitong_receive_uv --系统push
,count(distinct case when push_type in ('93','94') then a.device_id end) as AI_receive_uv
,count(distinct case when push_type is not null and push_type not in ('70','71') then a.device_id end) as auto_receive_uv --自动push
,count(a.device_id) as push_receive_pv
......@@ -372,7 +391,7 @@ full join
,count(case when push_type in ('106','107','108') then a.device_id end) as will_receive_pv --意愿push
,count(case when push_type in ('25') then a.device_id end) as sign_receive_pv --签到push
,count(case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.device_id end) as xitong_receive_pv --系统push
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25','93','94') then a.device_id end) as xitong_receive_pv --系统push
,count(case when push_type in ('93','94') then a.device_id end) as AI_receive_pv
,count(case when push_type is not null and push_type not in ('70','71') then a.device_id end) as auto_receive_pv --自动push
......@@ -381,7 +400,7 @@ full join
select partition_day as partition_date
,device_id,msg_id,'ios' as device_os_type
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>='20200521' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by partition_day,device_id,msg_id
union all
......@@ -391,7 +410,7 @@ full join
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>='20200521' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
where partition_date= regexp_replace(DATE_SUB(current_date,1) ,'-','')
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
......
pm/daily_push/job/step2.job
View file @ 9fd06823
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file
pm/daily_push/report/daily_push_click.sql
View file @ 9fd06823
......@@ -25,7 +25,7 @@ select day_id as `日期`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_uv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳设备/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳人次/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/non_ctr_push_open_pv*100,2),'%'),0) as `非ctr push二跳率=2跳人次/拉起人次`
from pm.tl_pm_push_d
from pm.tl_pm_push_detail_d
where partition_day>='20200708' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>='20200626'
and active_type='老活'
......
pm/daily_recommend_strategy/job/step2.job
View file @ 9fd06823
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file
pm/daily_recommend_strategy/job/step3.job
View file @ 9fd06823
#step3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_recommend_strategy_d
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
pm/daily_recommend_strategy/job/step4.job deleted 100644 → 0
View file @ 08277965
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
pm/search_related/job/step2.job
View file @ 9fd06823
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
command=/home/bi/bi-report/lib/shell/hive search_related
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/hive search_related
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment