Skip to content

B
bi-report
Commit a6997013 authored by 赵建伟's avatar 赵建伟
Browse files
Options

Merge branch 'weiyimin' into 'master' 

Weiyimin

See merge request !52
parents 86532a11 846c3e27
Show whitespace changes
Inline Side-by-side
Showing with 2127 additions and 63 deletions
pm/clue_daily/etl/clue_daily.sql
View file @ a6997013
...@@ -13,15 +13,21 @@ ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar; ...@@ -13,15 +13,21 @@ ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade'; CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption'; CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_userclue_d PARTITION (PARTITION_DAY = ${partition_day}) INSERT OVERWRITE TABLE pm.tl_pm_userclue_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT T1.partition_date as day_id SELECT T1.partition_date as day_id
,T1.device_os_type ,T1.device_os_type
,T1.active_type ,T1.active_type
,T1.grey_type ,T1.grey_type
,T1.channel
,dau ,dau
,user_num ,user_num
,total_valid_dev ,phone_user_num
,total_valid_num ,nvl(total_clue_dev,0)
,nvl(total_clue_num,0)
,nvl(total_valid_dev,0)
,nvl(total_valid_num,0)
,msg_dev ,msg_dev
,msg_num ,msg_num
,valid_msg_dev ,valid_msg_dev
...@@ -87,8 +93,13 @@ FROM ...@@ -87,8 +93,13 @@ FROM
,device_os_type ,device_os_type
,active_type ,active_type
,grey_type ,grey_type
,channel
,count(distinct mas.device_id) as dau ,count(distinct mas.device_id) as dau
,count(distinct a.user_id) as user_num ,count(distinct a.user_id) as user_num
,count(distinct b.user_id) as phone_user_num
,nvl(count(distinct case when t13.total_clue_num>0 then t13.user_id end),0) total_clue_dev
,nvl(sum(t13.total_clue_num),0) total_clue_num
,nvl(count(distinct case when t13_v.total_valid_num>0 then t13_v.user_id end),0) total_valid_dev ,nvl(count(distinct case when t13_v.total_valid_num>0 then t13_v.user_id end),0) total_valid_dev
,nvl(sum(t13_v.total_valid_num),0) total_valid_num ,nvl(sum(t13_v.total_valid_num),0) total_valid_num
...@@ -142,23 +153,47 @@ FROM ...@@ -142,23 +153,47 @@ FROM
FROM FROM
( (
SELECT partition_date,device_id,device_os_type,b.active_type,b.grey_type SELECT partition_date,device_id,device_os_type,b.active_type,b.grey_type,b.channel
FROM FROM
( (
SELECT SELECT
partition_date partition_date
,device_id,device_os_type ,m.device_id,device_os_type
,array(case when active_type in ('1','2') then '新增设备' ,array(case when active_type in ('1','2') then '新增设备'
when active_type ='4' then '老活跃设备' end,'合计') as active_type when active_type ='4' then '老活跃设备' end,'合计') as active_type
,array(case when substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type ,array(case when substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
FROM online.ml_device_day_active_status ,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','') where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<=regexp_replace(date_sub(current_date,1),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4') AND active_type in ('1','2','4')
and device_id is not null and m.device_id IS NOT NULL
)a )a
LATERAL VIEW explode(a.grey_type) b AS grey_type LATERAL VIEW explode(a.grey_type) b AS grey_type
LATERAL VIEW explode(a.active_type) b AS active_type LATERAL VIEW explode(a.active_type) b AS active_type
LATERAL VIEW explode(a.channel) b AS channel
)mas )mas
left join left join
(--找出user_id当天活跃的第一个设备id (--找出user_id当天活跃的第一个设备id
...@@ -170,6 +205,14 @@ FROM ...@@ -170,6 +205,14 @@ FROM
)a )a
on mas.partition_date=a.partition_date and mas.device_id=a.device_id on mas.partition_date=a.partition_date and mas.device_id=a.device_id
left join left join
(
SELECT distinct user_id
FROM online.tl_hdfs_userextra_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND auth_type = '1'
)b
on a.user_id=b.user_id
left join
(--私信线索总数 (--私信线索总数
SELECT msg.partition_date,user_id SELECT msg.partition_date,user_id
,count(msg.merchant_id) as msg_num ,count(msg.merchant_id) as msg_num
...@@ -268,7 +311,7 @@ FROM ...@@ -268,7 +311,7 @@ FROM
)t3 )t3
ON a.partition_date = t3.partition_date and a.user_id=t3.user_id ON a.partition_date = t3.partition_date and a.user_id=t3.user_id
LEFT JOIN LEFT JOIN
( ---拨打电话线索 ( ---有效电话线索
SELECT partition_date,user_id SELECT partition_date,user_id
,count(1) as valid_call_num ,count(1) as valid_call_num
FROM FROM
...@@ -1095,6 +1138,39 @@ FROM ...@@ -1095,6 +1138,39 @@ FROM
)t12_v )t12_v
ON a.partition_date = t12_v.partition_date and a.user_id=t12_v.user_id ON a.partition_date = t12_v.partition_date and a.user_id=t12_v.user_id
left join left join
(
SELECT partition_date,user_id,sum(clue_num) as total_clue_num
FROM
(
SELECT msg.partition_date,user_id,count(1) as clue_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
GROUP BY msg.partition_date,user_id
union all
--电话线索总数
SELECT partition_date,user_id,count(1) AS clue_num
FROM
(
SELECT user_id,merchant_id,MIN(REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '')) as partition_date,regexp_replace(SUBSTR(created_time,1,7),'-','') AS CALL_MONTH
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,7),'-','')
)a
GROUP BY partition_date,user_id
)a
group by partition_date,user_id
)t13
ON a.partition_date = t13.partition_date and a.user_id=t13.user_id
left join
( (
SELECT partition_date,user_id,sum(valid_num) as total_valid_num SELECT partition_date,user_id,sum(valid_num) as total_valid_num
from from
...@@ -1178,12 +1254,12 @@ FROM ...@@ -1178,12 +1254,12 @@ FROM
group by partition_date,user_id group by partition_date,user_id
)t13_v )t13_v
ON a.partition_date = t13_v.partition_date and a.user_id=t13_v.user_id ON a.partition_date = t13_v.partition_date and a.user_id=t13_v.user_id
group by mas.partition_date,device_os_type,active_type,grey_type group by mas.partition_date,device_os_type,active_type,grey_type,channel
)T1 )T1
LEFT JOIN LEFT JOIN
( (
SELECT t3_d.partition_date,a.device_os_type,a.grey_type SELECT t3_d.partition_date,a.device_os_type,a.grey_type
,a.active_type ,a.active_type,a.channel
,nvl(count(distinct case when t3_d.call_num>0 then t3_d.user_id end),0) as call_dev_d ,nvl(count(distinct case when t3_d.call_num>0 then t3_d.user_id end),0) as call_dev_d
,nvl(sum(t3_d.call_num),0) as call_num_d ,nvl(sum(t3_d.call_num),0) as call_num_d
...@@ -1209,19 +1285,26 @@ LEFT JOIN ...@@ -1209,19 +1285,26 @@ LEFT JOIN
,nvl(sum(t9_v_d.valid_call_num),0) as valid_call_num_qa_d ,nvl(sum(t9_v_d.valid_call_num),0) as valid_call_num_qa_d
FROM FROM
( (
SELECT device_id,device_os_type,user_id,v.grey_type,v.active_type SELECT device_id,device_os_type,user_id,v.grey_type,v.active_type,v.channel
FROM FROM
( (
SELECT device_id,device_os_type,user_id SELECT c.device_id,device_os_type,user_id
,array(case when substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type ,array(case when substr(convup(setencryption(c.device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
,array('老活跃设备','合计') as active_type ,array('老活跃设备','合计') as active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM FROM
( (
SELECT device_id,device_os_type,a.user_id,a.partition_date SELECT device_id,device_os_type,a.user_id,a.partition_date,first_channel_source_type
,row_number() over(partition by device_id order by a.partition_date desc) as rn ,row_number() over(partition by device_id order by a.partition_date desc) as rn
FROM FROM
( (
SELECT device_id,v.device_os_type,v.user_id,partition_date SELECT device_id,v.device_os_type,v.user_id,partition_date
,if(size(channel_source_type_list) > 0, channel_source_type_list [ 0 ], '') AS first_channel_source_type
FROM online.ml_device_updates FROM online.ml_device_updates
LATERAL VIEW EXPLODE(user_list) v as user_id LATERAL VIEW EXPLODE(user_list) v as user_id
LATERAL VIEW EXPLODE(device_os_type) v as device_os_type LATERAL VIEW EXPLODE(device_os_type) v as device_os_type
...@@ -1237,10 +1320,34 @@ LEFT JOIN ...@@ -1237,10 +1320,34 @@ LEFT JOIN
)b )b
on a.partition_date=b.partition_date and a.user_id=b.user_id on a.partition_date=b.partition_date and a.user_id=b.user_id
)c )c
LEFT JOIN
(
SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
)tmp
on c.first_channel_source_type=tmp.code and c.partition_date=tmp.partition_day
LEFT JOIN
(
SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-','')
)a
ON c.device_id = a.device_id
LEFT JOIN
(
SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id
)b
ON c.device_id = b.device_id AND c.partition_date = b.day_id
where rn=1 where rn=1
)d )d
LATERAL VIEW EXPLODE(d.grey_type)v as grey_type LATERAL VIEW EXPLODE(d.grey_type)v as grey_type
LATERAL VIEW EXPLODE(d.active_type)v as active_type LATERAL VIEW EXPLODE(d.active_type)v as active_type
LATERAL VIEW explode(d.channel) v AS channel
)a )a
JOIN JOIN
( ---非当天拨打电话线索 ( ---非当天拨打电话线索
...@@ -1798,9 +1905,12 @@ LEFT JOIN ...@@ -1798,9 +1905,12 @@ LEFT JOIN
GROUP BY user_id,partition_date GROUP BY user_id,partition_date
)t12_v_d )t12_v_d
ON t3_d.partition_date = t12_v_d.partition_date and t3_d.user_id=t12_v_d.user_id ON t3_d.partition_date = t12_v_d.partition_date and t3_d.user_id=t12_v_d.user_id
group by t3_d.partition_date,a.device_os_type,a.grey_type,a.active_type group by t3_d.partition_date,a.device_os_type,a.grey_type,a.active_type,a.channel
)T2 )T2
on T1.partition_date=T2.partition_date on T1.partition_date=T2.partition_date
AND T1.device_os_type=T2.device_os_type AND T1.device_os_type=T2.device_os_type
AND T1.active_type=T2.active_type AND T1.active_type=T2.active_type
AND T1.grey_type=T2.grey_type AND T1.grey_type=T2.grey_type
AND T1.channel=T2.channel
pm/clue_daily/etl/create_clue_daily.sql
View file @ a6997013
...@@ -20,10 +20,14 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_userclue_d ...@@ -20,10 +20,14 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_userclue_d
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}', device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}', active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}', grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau bigint comment '{"chs_name":"dau(不去除积分墙)","description":"","etl":"","value":"","remark":""}', dau bigint comment '{"chs_name":"dau(不去除积分墙)","description":"","etl":"","value":"","remark":""}',
user_num bigint comment '{"chs_name":"注册用户","description":"","etl":"","value":"","remark":""}',
phone_user_num bigint comment '{"chs_name":"手机号注册用户","description":"","etl":"","value":"","remark":""}',
total_clue_dev bigint comment '{"chs_name":"总咨询设备","description":"","etl":"","value":"","remark":""}',
total_clue_num bigint comment '{"chs_name":"总线索","description":"","etl":"","value":"","remark":""}',
total_valid_dev bigint comment '{"chs_name":"总有效咨询设备","description":"","etl":"","value":"","remark":""}', total_valid_dev bigint comment '{"chs_name":"总有效咨询设备","description":"","etl":"","value":"","remark":""}',
total_valid_num bigint comment '{"chs_name":"总有效线索","description":"","etl":"","value":"","remark":""}', total_valid_num bigint comment '{"chs_name":"总有效线索","description":"","etl":"","value":"","remark":""}',
msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
msg_dev bigint comment '{"chs_name":"私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}', msg_dev bigint comment '{"chs_name":"私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}', msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
valid_msg_dev bigint comment '{"chs_name":"有效私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}', valid_msg_dev bigint comment '{"chs_name":"有效私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
......
pm/clue_daily/job/step1_14.job 0 → 100644
View file @ a6997013
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file
pm/clue_daily/job/step2.job
View file @ a6997013
#step2.job #step2.job
type=command type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
command=sh /home/bi/bi-report/lib/shell/hive clue_daily command=sh /home/bi/bi-report/lib/shell/hive clue_daily
\ No newline at end of file
pm/clue_daily/report/clue_daily.sql
View file @ a6997013
...@@ -2,22 +2,28 @@ SELECT day_id `日期` ...@@ -2,22 +2,28 @@ SELECT day_id `日期`
,device_os_type `系统` ,device_os_type `系统`
,active_type `新老` ,active_type `新老`
,grey_type `是否灰度` ,grey_type `是否灰度`
,channel `渠道`
,dau as `dau（不去除积分墙）` ,dau as `dau（不去除积分墙）`
,user_num as `注册的用户数` ,user_num as `注册的用户数`
,phone_user_num as `手机号注册用户数`
,total_clue_dev as `总咨询设备（去重）`
,total_clue_num as `总线索`
,total_valid_dev+valid_call_dev_d as `总有效咨询设备（去重）` ,total_valid_dev+valid_call_dev_d as `总有效咨询设备（去重）`
,total_valid_num+valid_call_num_d as `总有效线索` ,total_valid_num+valid_call_num_d as `总有效线索`
,concat(round((total_valid_dev+valid_call_dev_d)/dau,2)*100,'%') as `总有效线索设备(去重)/dau` ,nvl(concat(round((total_valid_dev+valid_call_dev_d)/dau*100,2),'%'),0) as `总有效线索设备(去重)/dau`
,concat(round((total_valid_num+valid_call_num_d)/dau,2)*100,'%') as `总有效线索/dau` ,nvl(concat(round((total_valid_num+valid_call_num_d)/dau*100,2),'%'),0) as `总有效线索/dau`
,nvl(concat(round((total_valid_dev+valid_call_dev_d)/total_clue_dev*100,2),'%'),0) as `总有效线索设备(去重)/总咨询设备(去重)`
,nvl(concat(round((total_valid_num+valid_call_num_d)/total_clue_num*100,2),'%'),0) as `总有效线索/总线索`
,msg_dev as `私信咨询设备数-总计` ,msg_dev as `私信咨询设备数-总计`
,msg_num as `私信咨询人次-总计` ,msg_num as `私信咨询人次-总计`
,valid_msg_dev as `有效私信咨询设备数-总计` ,valid_msg_dev as `有效私信咨询设备数-总计`
,valid_msg_num as `有效私信咨询人次-总计` ,valid_msg_num as `有效私信咨询人次-总计`
,concat(round(valid_msg_num/msg_num,2)*100,'%') as `有效私信咨询人次/私信咨询人次` ,nvl(concat(round(valid_msg_num/msg_num*100,2),'%'),0) as `有效私信咨询人次/私信咨询人次`
,clue_dev as `授权电话线索设备数-总计` ,clue_dev as `授权电话线索设备数-总计`
,clue_num as `授权电话线索人次-总计` ,clue_num as `授权电话线索人次-总计`
,call_dev+call_dev_d as `拨打电话线索设备数-总计` ,call_dev+call_dev_d as `拨打电话线索设备数-总计`
,call_num+call_num_d as `拨打电话线索人次-总计` ,call_num+call_num_d as `拨打电话线索人次-总计`
,concat(round((call_num+call_num_d)/clue_num,2)*100,'%') as `拨打电话人次/授权电话人次` ,nvl(concat(round((call_num+call_num_d)/clue_num*100,2),'%'),0) as `拨打电话人次/授权电话人次`
,valid_call_dev+valid_call_dev_d as `有效电话线索设备数-总计` ,valid_call_dev+valid_call_dev_d as `有效电话线索设备数-总计`
,valid_call_num+valid_call_num_d as `有效电话线索人次-总计` ,valid_call_num+valid_call_num_d as `有效电话线索人次-总计`
,call_dev as `当天拨打电话线索设备数-总计` ,call_dev as `当天拨打电话线索设备数-总计`
...@@ -27,18 +33,18 @@ SELECT day_id `日期` ...@@ -27,18 +33,18 @@ SELECT day_id `日期`
,msg_dev_3 `私信咨询设备数-3级以上商户` ,msg_dev_3 `私信咨询设备数-3级以上商户`
,msg_num_3 `私信咨询人次-3级以上商户` ,msg_num_3 `私信咨询人次-3级以上商户`
,concat(round(msg_num_3/msg_num,2)*100,'%') as `私信咨询人次-3级以上/私信咨询人次` ,nvl(concat(round(msg_num_3/msg_num*100,2),'%'),0) as `私信咨询人次-3级以上/私信咨询人次`
,valid_msg_dev_3 `有效私信咨询设备数-3级以上商户` ,valid_msg_dev_3 `有效私信咨询设备数-3级以上商户`
,valid_msg_num_3 as `有效私信咨询人次-3级以上商户` ,valid_msg_num_3 as `有效私信咨询人次-3级以上商户`
,clue_dev_3 as `授权电话线索设备数-3级以上商户` ,clue_dev_3 as `授权电话线索设备数-3级以上商户`
,clue_num_3 as `授权电话线索人次-3级以上商户` ,clue_num_3 as `授权电话线索人次-3级以上商户`
,concat(round(clue_num_3/clue_num,2)*100,'%') as `授权电话线索-3级以上/授权电话线索` ,nvl(concat(round(clue_num_3/clue_num*100,2),'%'),0) as `授权电话线索-3级以上/授权电话线索`
,call_dev_3+call_dev_3_d as `拨打电话线索设备数-3级以上商户` ,call_dev_3+call_dev_3_d as `拨打电话线索设备数-3级以上商户`
,call_num_3+valid_call_num_3 as `拨打电话线索人次-3级以上商户` ,call_num_3+valid_call_num_3 as `拨打电话线索人次-3级以上商户`
,concat(round((call_num_3+valid_call_num_3)/clue_num_3,2)*100,'%') as `拨打电话线索人次-3级以上/授权电话线索人次-3级以上` ,nvl(concat(round((call_num_3+valid_call_num_3)/clue_num_3*100,2),'%'),0) as `拨打电话线索人次-3级以上/授权电话线索人次-3级以上`
,valid_call_dev_3+valid_call_dev_3_d as `有效电话线索设备数-3级以上商户` ,valid_call_dev_3+valid_call_dev_3_d as `有效电话线索设备数-3级以上商户`
,valid_call_num_3+valid_call_num_3_d as `有效电话线索人次-3级以上商户` ,valid_call_num_3+valid_call_num_3_d as `有效电话线索人次-3级以上商户`
,concat(round((valid_call_num_3+valid_call_num_3_d)/clue_num_3,2)*100,'%') as `有效电话线索人次-3级以上/授权电话线索人次-3级以上` ,nvl(concat(round((valid_call_num_3+valid_call_num_3_d)/clue_num_3*100,2),'%'),0) as `有效电话线索人次-3级以上/授权电话线索人次-3级以上`
,call_dev_3 as `当天拨打电话线索设备数-3级以上商户` ,call_dev_3 as `当天拨打电话线索设备数-3级以上商户`
,call_num_3 as `当天拨打电话线索人次-3级以上商户` ,call_num_3 as `当天拨打电话线索人次-3级以上商户`
,valid_call_dev_3 as `当天有效电话线索设备数-3级以上商户` ,valid_call_dev_3 as `当天有效电话线索设备数-3级以上商户`
......
pm/daily_operation/en-cn.properties 0 → 100644
View file @ a6997013
daily_operation=运营日报
\ No newline at end of file
pm/daily_operation/etl/create_daily_operation.sql 0 → 100644
View file @ a6997013
--***************************************************************
--*脚本名称:
--*功能: 运营日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_operation_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau bigint comment '{"chs_name":"dau(去除积分墙)","description":"","etl":"","value":"","remark":""}',
login_dev_num bigint comment '{"chs_name":"新增注册设备数","description":"","etl":"","value":"","remark":""}',
login_user_num bigint comment '{"chs_name":"新增注册账号","description":"","etl":"","value":"","remark":""}',
login_phone_user_num bigint comment '{"chs_name":"新增注册手机号用户数","description":"","etl":"","value":"","remark":""}',
retention_1 string comment '{"chs_name":"次日留存率","description":"","etl":"","value":"","remark":""}',
retention_7 string comment '{"chs_name":"第7日留存率","description":"","etl":"","value":"","remark":""}',
retention_30 string comment '{"chs_name":"第30日留存率","description":"","etl":"","value":"","remark":""}',
per_open_times bigint comment '{"chs_name":"单设备打开频次","description":"","etl":"","value":"","remark":""}',
home_feeds_ctr string comment '{"chs_name":"信息流内容ctr","description":"","etl":"","value":"","remark":""}',
per_use_time double comment '{"chs_name":"单设备app停留时长(M)","description":"","etl":"","value":"","remark":""}',
neirong_page_stay double comment '{"chs_name":"单设备内容时长(M)","description":"","etl":"","value":"","remark":""}',
neirong_in_app_use_time string comment '{"chs_name":"单设备内容时长/app停留时长","description":"","etl":"","value":"","remark":""}',
service_privacy_uv bigint comment '{"chs_name":"到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
service_click_uv bigint comment '{"chs_name":"同意隐私授权设备设备数","description":"","etl":"","value":"","remark":""}',
service_privacy_uv_in_dau string comment '{"chs_name":"到达隐私授权界面设备数/DAU","description":"","etl":"","value":"","remark":""}',
service_click_uv_in_dau string comment '{"chs_name":"同意隐私授权设备数/到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
open_push_uv bigint comment '{"chs_name":"开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
open_push_uv_in_dau string comment '{"chs_name":"开启推送授权设备数/DAU","description":"","etl":"","value":"","remark":""}',
push_receive_uv_in_open_push_uv string comment '{"chs_name":"次日接收推送设备数/当天开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
click_uv bigint comment '{"chs_name":"点击推送的设备数","description":"","etl":"","value":"","remark":""}',
non_click_uv bigint comment '{"chs_name":"未点击推送的设备数","description":"","etl":"","value":"","remark":""}',
click_uv_in_dau string comment '{"chs_name":"点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
non_click_uv_in_dau string comment '{"chs_name":"未点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
op_push_open_uv bigint comment '{"chs_name":"点击运营推送设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv bigint comment '{"chs_name":"点击系统推送设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv bigint comment '{"chs_name":"点击个性化推送设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv bigint comment '{"chs_name":"点击签到推送设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_uv_in_receive string comment '{"chs_name":"点击运营推送设备/接收到运营推送设备","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv_in_receive string comment '{"chs_name":"点击系统推送设备/接收到系统推送设备","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv_in_receive string comment '{"chs_name":"点击个性化推送设备/接收到个性化推送设备","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv_in_receive string comment '{"chs_name":"点击签到推送设备/接收到签到推送设备","description":"","etl":"","value":"","remark":""}',
question_num bigint comment '{"chs_name":"主动提问数","description":"","etl":"","value":"","remark":""}',
answer_num bigint comment '{"chs_name":"主动回答数","description":"","etl":"","value":"","remark":""}',
topic_num bigint comment '{"chs_name":"主动发帖数","description":"","etl":"","value":"","remark":""}',
reply_num bigint comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
question_num_in_dau string comment '{"chs_name":"主动提问设备数/DAU`","description":"","etl":"","value":"","remark":""}',
answer_num_in_dau string comment '{"chs_name":"主动回答设备数/DAU","description":"","etl":"","value":"","remark":""}',
topic_num_in_dau string comment '{"chs_name":"主动发帖设备数/DAU","description":"","etl":"","value":"","remark":""}',
reply_num_in_dau string comment '{"chs_name":"主动评论设备数/DAU","description":"","etl":"","value":"","remark":""}',
neirongUV bigint comment '{"chs_name":"内容UV","description":"","etl":"","value":"","remark":""}',
neirongPV bigint comment '{"chs_name":"内容PV","description":"","etl":"","value":"","remark":""}',
neirongUV_in_dau string comment '{"chs_name":"内容UV/DAU","description":"","etl":"","value":"","remark":""}',
neirongPV_in_UV double comment '{"chs_name":"内容PV/内容UV","description":"","etl":"","value":"","remark":""}',
result_more_UV bigint comment '{"chs_name":"搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
result_meigou_UV bigint comment '{"chs_name":"搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
result_con_UV bigint comment '{"chs_name":"搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
result_more_UV_in_dau string comment '{"chs_name":"搜索结果综合页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_meigou_UV_in_dau string comment '{"chs_name":"搜索结果美购页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_con_UV_in_dau string comment '{"chs_name":"搜索结果商业页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_more_PV bigint comment '{"chs_name":"搜索结果综合页PV","description":"","etl":"","value":"","remark":""}',
result_meigou_PV bigint comment '{"chs_name":"搜索结果美购页PV","description":"","etl":"","value":"","remark":""}',
result_con_PV bigint comment '{"chs_name":"搜索结果商业页PV","description":"","etl":"","value":"","remark":""}',
result_more_PV_in_UV double comment '{"chs_name":"搜索结果综合页PV/搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
result_meigou_PV_in_UV double comment '{"chs_name":"搜索结果美购页PV/搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
result_con_PV_in_UV double comment '{"chs_name":"搜索结果商业页PV/搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
result_more_PV_in_dau string comment '{"chs_name":"搜索结果综合页PV/DAU","description":"","etl":"","value":"","remark":""}',
result_meigou_PV_in_dau string comment '{"chs_name":"搜索结果美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
result_con_PV_in_dau string comment '{"chs_name":"搜索结果商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_meigou_PV bigint comment '{"chs_name":"来自内容页的美购页PV","description":"","etl":"","value":"","remark":""}',
neirong_con_PV bigint comment '{"chs_name":"来自内容页的商业页PV","description":"","etl":"","value":"","remark":""}',
neirong_meigou_PV_in_dau string comment '{"chs_name":"来自内容页的美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_con_PV_in_dau string comment '{"chs_name":"来自内容页的商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_click_pv bigint comment '{"chs_name":"内容页的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
wel_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的商业二跳/内容pv","description":"","etl":"","value":"","remark":""}',
neirong_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的内容二跳/内容pv","description":"","etl":"","value":"","remark":""}'
)comment '运营日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_operation/etl/daily_operation.sql 0 → 100644
View file @ a6997013
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--运营日报
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_operation_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_date as day_id
,t1.device_os_type as device_os_type
,t1.device_type as active_type
,t1.channel as channel
,t1.active_num as `DAU`
,t14.login_dev_num as login_dev_num
,t14.login_user_num as login_user_num
,t14.login_phone_user_num as login_phone_user_num
,concat(round(t1.retention_num1/t1.active_num*100,1),'%') as retention_1
,concat(round(t1.retention_num7/t1.active_num*100,1),'%') as retention_7
,concat(round(t1.retention_num30/t1.active_num*100,1),'%') as retention_30
,coalesce(t3.per_open_times,0) as per_open_times
,concat(round(coalesce(click_card_pv,0)/coalesce(exp_card_pv,0)*100,3),'%') as home_feeds_ctr
,coalesce(t3.per_use_time,0) as per_use_time
,round(coalesce(t4.neirong_page_stay,0)/coalesce(neirongUV,0)/60,3) as neirong_page_stay
,concat(round(round(coalesce(t4.neirong_page_stay,0)/coalesce(t1.active_num,0)/60,3)/coalesce(t3.per_use_time,0)*100,3),'%') as neirong_in_app_use_time
,coalesce(t4.service_privacy_uv,0) as service_privacy_uv
,coalesce(t4.service_click_uv,0) as service_click_uv
,concat(round(coalesce(t4.service_privacy_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as service_privacy_uv_in_dau
,concat(round(coalesce(t4.service_click_uv,0)/coalesce(t4.service_privacy_uv,0)*100,3),'%') as service_click_uv_in_dau
,coalesce(t11.open_push_uv,0) as open_push_uv
,concat(round(coalesce(t11.open_push_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `open_push_uv_in_dau`
,concat(round(coalesce(t11.push_receive_uv,0)/coalesce(t11.open_push_uv,0)*100,3),'%') as `push_receive_uv_in_open_push_uv`
,coalesce(t10.click_uv,0) as `click_uv`
,coalesce(t1.active_num-t10.click_uv,0) as `non_click_uv`
,concat(round(coalesce(t10.click_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `click_uv_in_dau`
,concat(round(coalesce(t1.active_num-t10.click_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `non_click_uv_in_dau`
,coalesce(t13.op_push_open_uv,0) as `op_push_open_uv`
,coalesce(t13.auto_push_open_uv,0) as `auto_push_open_uv`
,coalesce(t13.personal_push_open_uv,0) as `personal_push_open_uv`
,coalesce(t13.sign_push_open_uv,0) as `sign_push_open_uv`
,concat(round(coalesce(t13.op_push_open_uv,0)/coalesce(t12.op_receive_uv,0)*100,3),'%') as `op_push_open_uv_in_receive`
,concat(round(coalesce(t13.auto_push_open_uv,0)/coalesce(t12.auto_receive_uv,0)*100,3),'%') as `auto_push_open_uv_in_receive`
,concat(round(coalesce(t13.personal_push_open_uv,0)/coalesce(t12.personal_receive_uv,0)*100,3),'%') as `personal_push_open_uv_in_receive`
,concat(round(coalesce(t13.sign_push_open_uv,0)/coalesce(t12.sign_receive_uv,0)*100,3),'%') as `sign_push_open_uv_in_receive`
,coalesce(question_num,0) as `question_num`
,coalesce(answer_num,0) as `answer_num`
,coalesce(topic_num,0) as `topic_num`
,coalesce(reply_num,0) as `reply_num`
,concat(round(coalesce(question_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `question_num_in_dau`
,concat(round(coalesce(answer_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `answer_num_in_dau`
,concat(round(coalesce(topic_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `topic_num_in_dau`
,concat(round(coalesce(reply_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `reply_num_in_dau`
,coalesce(t4.neirongUV,0) as `neirongUV`
,coalesce(t4.neirongPV,0) as `neirongPV`
,concat(round(coalesce(t4.neirongUV,0)/t1.active_num*100,3),'%') as `neirongUV_in_dau`
,round(coalesce(t4.neirongPV,0)/coalesce(t4.neirongUV,0),2) as `neirongPV_in_UV`
--,round(coalesce(t4.neirongPV,0)/t2.active_num,2) as `内容PV/DAU`
,coalesce(result_more_UV,0) as `result_more_UV`
,coalesce(result_meigou_UV,0) as `result_meigou_UV`
,coalesce(result_con_UV,0) as `result_con_UV`
,concat(round(coalesce(result_more_UV,0)/t1.active_num*100,3),'%') as `result_more_UV_in_dau`
,concat(round(coalesce(result_meigou_UV,0)/t1.active_num*100,3),'%') as `result_meigou_UV_in_dau`
,concat(round(coalesce(result_con_UV,0)/t1.active_num*100,3),'%') as `result_con_UV_in_dau`
,coalesce(result_more_PV,0) as `result_more_PV`
,coalesce(result_meigou_PV,0) as `result_meigou_PV`
,coalesce(result_con_PV,0) as `result_con_PV`
,round(coalesce(result_more_PV,0)/coalesce(result_more_UV,0),2) as `result_more_PV_in_UV`
,round(coalesce(result_meigou_PV,0)/coalesce(result_meigou_UV,0),2) as `result_meigou_PV_in_UV`
,round(coalesce(result_con_PV,0)/coalesce(result_con_UV,0),2) as `result_con_PV_in_UV`
,concat(round(coalesce(result_more_PV,0)/t1.active_num*100,2),'%') as `result_more_PV_in_dau`
,concat(round(coalesce(result_meigou_PV,0)/t1.active_num*100,2),'%') as `result_meigou_PV_in_dau`
,concat(round(coalesce(result_con_PV,0)/t1.active_num*100,2),'%') as `result_con_PV_in_dau`
,coalesce(neirong_meigou_PV,0) as `neirong_meigou_PV`
,coalesce(neirong_con_PV,0) as `neirong_con_PV`
,concat(round(coalesce(neirong_meigou_PV,0)/t1.active_num*100,2),'%') as `neirong_meigou_PV_in_dau`
,concat(round(coalesce(neirong_con_PV,0)/t1.active_num*100,2),'%') as `neirong_con_PV_in_dau`
,coalesce(neirong_click_pv,0) as `neirong_click_pv`
,concat(round(coalesce(wel_second_pv,0)/neirongPV*100,2),'%') as `wel_second_pv_in_neirongPV`
,concat(round(coalesce(neirong_second_pv,0)/neirongPV*100,2),'%') as `neirong_second_pv_in_neirongPV`
FROM
(
--日活活跃、留存
SELECT
regexp_replace(substr(t1.partition_date,1,10),'-','') as partition_date
,device_os_type,t1.device_type,channel
,int(count(DISTINCT t1.device_id)) as active_num
,int(count(DISTINCT case WHEN date_add(t1.partition_date,1) = t2.partition_date then t2.device_id end)) as retention_num1
,int(count(DISTINCT case WHEN date_add(t1.partition_date,6) = t2.partition_date then t2.device_id end)) as retention_num7
,int(count(DISTINCT case WHEN date_add(t1.partition_date,29) = t2.partition_date then t2.device_id end)) as retention_num30
FROM
(
SELECT mas.partition_date,mas.device_id,device_type,device_os_type,t2.channel
FROM
(
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
)t1
LEFT JOIN
(
SELECT device_id
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
FROM online.ml_device_day_active_status
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)t2
on t1.device_id = t2.device_id
GROUP BY regexp_replace(substr(t1.partition_date,1,10),'-',''),device_os_type,t1.device_type,channel
)t1
LEFT JOIN
(
--App停留时长、打开频次
SELECT
partition_date,device_os_type,device_type,t.channel
,round(SUM(case when use_duration > 0 and use_duration < 86400 then use_duration else null end)/count(DISTINCT case when use_duration > 0 and use_duration < 86400 then device_id else null end)/60,1) as per_use_time
,round(SUM(open_times)/count(DISTINCT device_id),1) as per_open_times
FROM
(
SELECT b.device_id,use_duration,b.partition_date,open_times,device_os_type,device_type,channel
FROM
(
select device_id,use_duration,partition_date,open_times
from online.ml_device_updates
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)b
JOIN
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
when active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
AND first_channel_source_type not like 'promotion\_jf\_%'
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
)a
on a.device_id= b.device_id
and a.partition_date = b.partition_date
)a
LATERAL VIEW explode(a.channel) t AS channel
GROUP BY partition_date,device_os_type,device_type,t.channel
)t3
ON t1.partition_date = t3.partition_date
AND t1.device_os_type = t3.device_os_type
AND t1.device_type = t3.device_type
AND t1.channel = t3.channel
LEFT JOIN
(
--内容页、搜索各页PV、UV + 搜索数
SELECT partition_date,cl_type,device_type,channel
,count(case when action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirongPV
,count(DISTINCT case when action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then t1.cl_id else null end) as neirongUV
,count(case when action = 'page_view' AND page_name = 'search_result_more' then 1 else null end) as result_more_PV
,count(DISTINCT case when action = 'page_view' AND page_name = 'search_result_more' then t1.cl_id else null end) as result_more_UV
,count(case when action = 'page_view' AND page_name = 'search_result_welfare' then 1 else null end) as result_meigou_PV
,count(DISTINCT case when action = 'page_view' AND page_name = 'search_result_welfare' then t1.cl_id else null end) as result_meigou_UV
,count(case when action = 'page_view' AND page_name in ('search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization') then 1 else null end) as result_con_PV
,count(DISTINCT case when action = 'page_view' AND page_name in ('search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization') then t1.cl_id else null end) as result_con_UV
,count(case when action = 'page_view' AND page_name = 'welfare_detail' AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirong_meigou_PV
,count(case when action = 'page_view' AND page_name in ('welfare_detail','expert_detail','organization_detail') AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirong_con_PV
,count(CASE WHEN action = 'click_neirong' THEN 1 ELSE NULL END) as neirong_click_pv
,count(CASE WHEN action = 'wel_second' THEN 1 ELSE NULL END) as wel_second_pv
,count(CASE WHEN action = 'neirong_second' THEN 1 ELSE NULL END) as neirong_second_pv
,sum(CASE WHEN action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND (page_stay>=0 AND page_stay<1000) then page_stay else null end) as neirong_page_stay
,count(DISTINCT CASE WHEN action = 'popup_view' AND page_name = 'service_privacy' THEN t1.cl_id END) as service_privacy_uv
,count(DISTINCT CASE WHEN action = 'popup_click' AND page_name = 'service_privacy' THEN t1.cl_id END) as service_click_uv
FROM
(
SELECT
a.cl_id as cl_id
,a.partition_date
,page_name,cl_type,device_type,channel,referrer,action,page_stay
FROM
(
SELECT page_name,cl_id,partition_date,cl_type,referrer,action,page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and action = 'page_view'
AND ((page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail'
,'search_result_more','search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization'))
or (page_name = 'welfare_detail' AND params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail'))
or (page_name in ('welfare_detail','expert_detail','organization_detail') AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')))
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'click_neirong' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
and (action ='on_click_card' and params['card_content_type'] in ('qa','diary','user_post','answer')
or action='post_detail_browse_card')
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'wel_second' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
and params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND ((action='on_click_button' and params['button_name'] = 'more_recommendations')
or (action='on_click_button' and params['button_name']='video_interview')
or (action='on_click_button' and params['button_name']='referral')
or (action ='on_click_card' and params['card_content_type']='service'))
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'neirong_second' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') --内容详情页-搜索首页-进行搜索
and (action in ('on_click_navbar_search','do_search')
or (action ='on_click_card' and params['card_type']='highlight_word')
or (action='on_click_card' and params['card_content_type'] in ('qa','diary','user_post','answer')))
UNION ALL
SELECT 'service_privacy' as page_name,cl_id,partition_date,cl_type,'no' as referrer,'popup_view' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','') --隐私弹窗弹出事件
and action = 'popup_view'
AND params['popup_name'] ='service_privacy'
UNION ALL
SELECT 'service_privacy' as page_name,cl_id,partition_date,cl_type,'no' as referrer,'popup_click' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','') --隐私弹窗弹出事件
and action = 'on_click_button'
AND params['popup_name'] ='service_privacy'
AND params['type']='service_privacy'
AND params['button_name']='agree'
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel
FROM
(
SELECT
partition_date,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
when active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
LEFT JOIN
( -- 2.去掉疑似机构刷量的PV和UV
SELECT device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
GROUP BY t1.partition_date,cl_type,device_type,channel
)t4
ON t1.partition_date = t4.partition_date
AND t1.device_os_type = t4.cl_type
AND t1.device_type = t4.device_type
AND t1.channel = t4.channel
LEFT JOIN
( --来自信息流推荐内容的曝光卡片量
SELECT partition_date,device_os_type,active_type,t4.channel
,SUM(exp_card_pv) as exp_card_pv
FROM
(
SELECT mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id,count(DISTINCT t1.card_id) as exp_card_pv
FROM
(
SELECT partition_date
,device_os_type,m.device_id
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
JOIN
(
SELECT partition_day as partition_date,device_id as cl_id,card_id,app_session_id
FROM ml.mid_ml_c_et_pe_preciseexposure_dimen_d --新仓库精准曝光
WHERE partition_day >= regexp_replace(date_sub(current_date,30),'-','')and partition_day <= regexp_replace(date_sub(current_date,1),'-','')
AND page_code='home'
AND is_exposure='1'
AND tab_code='精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND card_content_type in ('user_post','diary','qa','answer')
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id
)t3
LATERAL VIEW explode(t3.channel) t4 AS channel
group by partition_date,device_os_type,active_type,t4.channel
)t5
ON t1.partition_date = t5.partition_date
AND t1.device_os_type = t5.device_os_type
AND t1.device_type = t5.active_type
AND t1.channel = t5.channel
LEFT JOIN
( --来自信息流推荐内容的卡片点击量
SELECT partition_date,device_os_type,active_type,t4.channel
,SUM(click_card_pv) as click_card_pv
FROM
(
SELECT mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id,count(DISTINCT t1.card_id) as click_card_pv
FROM
(
SELECT partition_date
,device_os_type,m.device_id
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day =regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') and partition_date <=regexp_replace(date_sub(current_date,1),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
JOIN
(
SELECT partition_day as partition_date,device_id as cl_id,card_id,app_session_id
FROM ml.ml_c_et_ck_click_dimen_d --新仓库点击表
WHERE partition_day >= regexp_replace(date_sub(current_date,30),'-','')and partition_day <= regexp_replace(date_sub(current_date,1),'-','')
AND action='on_click_card'
AND page_code ='home'
AND tab_code = '精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND card_content_type in ('diary','user_post','qa','answer')
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id
)t3
LATERAL VIEW explode(t3.channel) t4 AS channel
group by partition_date,device_os_type,active_type,t4.channel
)t6
ON t1.partition_date = t6.partition_date
AND t1.device_os_type = t6.device_os_type
AND t1.device_type = t6.active_type
AND t1.channel = t6.channel
LEFT JOIN
(
--纯用户-内容发布成功数pv(去积分墙渠道、去医生账号、马甲号、模特用户)--真实用户
SELECT a.create_date,device_os_type,device_type,channel
,SUM(CASE WHEN type = 'question' THEN num END) as question_num
,SUM(CASE WHEN type = 'answer' THEN num END) as answer_num
,SUM(CASE WHEN type in ('tractate','diary','topic') THEN num END) as topic_num
,count(DISTINCT CASE WHEN type = 'question' THEN dev.device_id END) as question_dev_num
,count(DISTINCT CASE WHEN type = 'answer' THEN dev.device_id END) as answer_dev_num
,count(DISTINCT CASE WHEN type in ('tractate','diary','topic') THEN dev.device_id END) as topic_dev_num
FROM
( --新增帖子（用户发的）
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'tractate' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_api_tractate_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and is_online = 'true'
AND platform = '1' --更美用户发的(去除hera后台，爬虫爬取的)
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'tractate'
UNION ALL
--新增帖子（打卡发帖，去除新增用户中kyc用户的打卡发帖）
SELECT a.user_id,create_date,'tractate' as type,count(DISTINCT a.id) as num
FROM
(
SELECT user_id,id,regexp_replace(substr(create_time,1,10),'-','') as create_date
FROM online.tl_hdfs_api_tractate_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and is_online = 'true'
AND platform = '7' --打卡发帖
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
)a
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
LEFT JOIN
(
SELECT
partition_date,device_id
FROM online.ml_device_day_active_status --中kyc灰度的设备
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2')
AND ((partition_date <= '20200511' AND substr(convup(setencryption(device_id,'sha-1'),16,10),-1) = '0')
or (partition_date >= '20200512' AND substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0))
)c
ON b.partition_date = c.partition_date
AND b.device_id = c.device_id
WHERE c.device_id is NULL
GROUP BY a.user_id,create_date,'tractate'
UNION ALL
--新增日记本
SELECT a.user_id,a.create_date,'diary' as type,count(DISTINCT a.id) as num
FROM
(
SELECT user_id,id,regexp_replace(substr(created_time,1,10),'-','') as create_date
FROM online.tl_hdfs_diary_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
--AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(created_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(created_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
)a
JOIN
( --取非空日记
SELECT diary_id
FROM online.tl_hdfs_problem_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_spam = 'false'
)b
ON a.id = b.diary_id
GROUP BY a.user_id,a.create_date,'diary'
UNION ALL
--新增日记帖
SELECT user_id,regexp_replace(substr(created_time,1,10),'-','') as create_date,'topic' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_problem_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND diary_id is not null
and regexp_replace(substr(created_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(created_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(created_time,1,10),'-',''),'topic'
UNION ALL
--新增问题
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'question' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_question_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND platform = '99' --更美用户发的(去除hera后台，爬虫爬取的，kyc自动回复的)
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'question'
UNION ALL
--新增回答
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'answer' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_answer_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND platform = '99' --更美用户发的(去除hera后台，爬虫爬取的，kyc自动回复的)
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'answer'
)a
JOIN --限制用户是在app进行的发帖
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date<= regexp_replace(date_sub(current_date,1),'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '发帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON a.user_id=c.user_id
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)d
ON a.user_id = d.user_id
AND a.create_date = d.partition_date
JOIN
( -- 1.去掉机刷、积分墙渠道来的设备
SELECT partition_date,device_id,device_os_type,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
LATERAL VIEW explode(dev.channel) t AS channel
)dev
on dev.device_id = d.device_id
AND dev.partition_date = d.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY a.create_date,device_os_type,device_type,channel
)t8
ON t1.partition_date = t8.create_date
AND t1.device_os_type = t8.device_os_type
AND t1.device_type = t8.device_type
AND t1.channel = t8.channel
LEFT JOIN
(
--真实评论数(日记/用户帖/问题/回答)
--有评论过日记帖的设备,排除疑似广告，排除医生账号，排除登陆过以上账号的设备和马甲号，限制用户当天有过评论行为
SELECT
a.reply_date,device_os_type,device_type,channel,SUM(reply_num) as reply_num,count(DISTINCT dev.device_id) as reply_dev_num
FROM
(
SELECT
user_id,reply_date,SUM(reply_num) as reply_num
FROM
(
--有评论过日记帖的设备,排除疑似广告
SELECT t1.user_id,reply_date,sum(reply_num) as reply_num
FROM
(
SELECT user_id,regexp_replace(substr(reply_date,1,10),'-','') as reply_date,problem_id,count(reply_date) as reply_num
FROM online.tl_hdfs_topicreply_view
WHERE partition_date =regexp_replace(date_sub(current_date,1),'-','')
AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(reply_date,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(reply_date,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
group by user_id,regexp_replace(substr(reply_date,1,10),'-',''),problem_id
)t1
JOIN
(
SELECT id,diary_id
FROM online.tl_hdfs_problem_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
group by id,diary_id
)t2
on t2.id=t1.problem_id
group by t1.user_id,reply_date
UNION ALL
--有评论过回答的设备,排除疑似广告
SELECT
t2.user_id as user_id,t2.reply_date as reply_date,sum(reply_num) as reply_num
FROM
(
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as reply_date,count(create_time) as reply_num,answer_id
FROM online.tl_hdfs_answer_reply_view
WHERE partition_date =regexp_replace(date_sub(current_date,1),'-','')
and (is_fake is NULL or is_fake = 'false')
AND answer_id is not NULL
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
group by user_id,answer_id,regexp_replace(substr(create_time,1,10),'-','')
)t2
GROUP BY t2.user_id,t2.reply_date
UNION ALL
--有评论过用户帖的设备
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as reply_date,count(create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(date_sub(current_date,1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP by user_id,regexp_replace(substr(create_time,1,10),'-','')
)reply_user
GROUP BY user_id,reply_date
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date<= regexp_replace(date_sub(current_date,1),'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.reply_date = b.partition_date
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON a.user_id=c.user_id
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >=regexp_replace(date_sub(current_date,30),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)d
ON a.user_id = d.user_id
AND a.reply_date = d.partition_date
JOIN
( -- 1.去掉机刷、积分墙渠道来的设备
SELECT partition_date,device_id,device_os_type,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
LATERAL VIEW explode(dev.channel) t AS channel
)dev
on dev.device_id = d.device_id
AND dev.partition_date = d.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY a.reply_date,device_os_type,device_type,channel
)t9
ON t1.partition_date = t9.reply_date
AND t1.device_os_type = t9.device_os_type
AND t1.device_type = t9.device_type
AND t1.channel = t9.channel
LEFT JOIN
(-- 推送点击 pv、uv
SELECT a.partition_date as partition_date,cl_type,device_type,channel
,count(a.cl_id) as click_pv
,count(DISTINCT a.cl_id) as click_uv
FROM
(
SELECT cl_id,partition_date,cl_type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND action = 'notification_open' --推送点击
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
GROUP BY a.partition_date,cl_type,device_type,channel
)t10
ON t1.partition_date = t10.partition_date
AND t1.device_os_type = t10.cl_type
AND t1.device_type = t10.device_type
AND t1.channel = t10.channel
LEFT JOIN
(-- 开启推送设备数
SELECT regexp_replace(substr(a.partition_date,1,10),'-','') as partition_date,device_os_type,device_type,channel
,count(DISTINCT a.cl_id) as open_push_uv
,count(distinct b.device_id) as push_receive_uv
FROM
(
SELECT concat_ws('-',substr(a.partition_date,1,4),substr(a.partition_date,5,2),substr(a.partition_date,7,2)) as partition_date
,a.cl_id --开启push设备
FROM
(
SELECT partition_date,cl_id,max(time_str) as max_time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
and action='is_open_push'
group by partition_date,cl_id
)a
left join
(
SELECT partition_date,cl_id,time_str,COALESCE(params['type'],params['tyep']) as type
FROM online.bl_hdfs_maidian_updates
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
and action='is_open_push'
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.time_str=a.max_time_str
where type=1
group by a.partition_date,a.cl_id
)a
left join
(
select concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)) as partition_date
,device_id
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
group by partition_day,device_id
union all
SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,device_id
FROM
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_day>=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
group by partition_date,registration_id,platform,mesid
)msg
JOIN
(
SELECT device_id,registration_id
from online.tl_hdfs_api_jpushuserdevicebinding_view--极光id和设备id绑定关系全量表
where partition_date=regexp_replace(date_sub(current_date(),1),'-','')
and (registration_id is not null or registration_id <>'')
group by device_id,registration_id
)reg
on reg.registration_id=msg.registration_id
group by partition_date,device_id
)b
on a.cl_id=b.device_id and date_add(a.partition_date,1)=b.partition_date
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day =regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)c
ON a.cl_id = c.device_id
AND a.partition_date = c.partition_date
GROUP BY a.partition_date,device_os_type,device_type,channel
)t11
ON t1.partition_date = t11.partition_date
AND t1.device_os_type = t11.device_os_type
AND t1.device_type = t11.device_type
AND t1.channel = t11.channel
LEFT JOIN
(-- 接收到推送设备数--只有iOS设备能拿到数据，并且iOS设备的系统版本在10及以上
SELECT a.partition_date
,device_os_type
,case when d.device_id is not null then '新增设备' else '老活跃设备' end as device_type
,channel
,count(distinct a.device_id) as push_receive_uv
,count(distinct case when push_type in ('70','71') then a.device_id end) as op_receive_uv --运营push push_type in ('70','71')
,count(distinct case when push_type in ('25') then a.device_id end) as sign_receive_uv --运营push push_type in ('70','71')
,count(distinct case when push_type in ('101','102','103','104','105','106','107','108','111','112','113') then a.device_id end) as personal_receive_uv --个性化push
,count(distinct case when push_type is not null and push_type not in ('25','70','71','101','102','103','104','105','106','107','108','111','112','113') then a.device_id end) as auto_receive_uv --自动push
FROM
(
select partition_day as partition_date
,device_id,msg_id,'ios' as device_os_type
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
group by partition_day,device_id,msg_id
union all
SELECT partition_date,device_id,mesid as msg_id,'android' as device_os_type
FROM
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
group by partition_date,registration_id,platform,mesid
)msg
JOIN
(
SELECT device_id,registration_id
from online.tl_hdfs_api_jpushuserdevicebinding_view--极光id和设备id绑定关系全量表
where partition_date=regexp_replace(date_sub(current_date(),1),'-','')
and (registration_id is not null or registration_id <>'')
group by device_id,registration_id
)reg
on reg.registration_id=msg.registration_id
)a
JOIN
(
select device_id,t.channel
from
(
select m.device_id
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
from
(
select m.device_id,m.first_channel_source_type
from online.ml_device_history_detail m
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-',''))tmp
on m.first_channel_source_type=tmp.code
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id
)m
lateral view explode(channel) t as channel
)b
on a.device_id=b.device_id
JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)c
on a.msg_id=c.msg_id
left join
(
select
partition_date
,device_id
from online.ml_device_day_active_status
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','3')
)d
on a.partition_date=d.partition_date and a.device_id=d.device_id
group by a.partition_date
,device_os_type
,case when d.device_id is not null then '新增设备' else '老活跃设备' end
,channel
)t12
ON t1.partition_date = t12.partition_date
AND t1.device_os_type = t12.device_os_type
AND t1.device_type = t12.device_type
AND t1.channel = t12.channel
LEFT JOIN
(-- 点击推送设备数
SELECT a.partition_date as partition_date,device_os_type,device_type,channel
,count(DISTINCT a.cl_id) as receive_push_uv
,count(DISTINCT CASE WHEN op_push_open_pv is not null and op_push_open_pv>0 then a.cl_id end) as op_push_open_uv
,count(DISTINCT CASE WHEN auto_push_open_pv is not null and auto_push_open_pv>0 then a.cl_id end) as auto_push_open_uv
,count(DISTINCT CASE WHEN personal_push_open_pv is not null and personal_push_open_pv>0 then a.cl_id end) as personal_push_open_uv
,count(DISTINCT CASE WHEN sign_push_open_pv is not null and sign_push_open_pv>0 then a.cl_id end) as sign_push_open_uv
FROM
(
SELECT a.partition_date,a.cl_id,
count(distinct a.message_id) as push_open_pv,
count(distinct case when push_type in ('70','71') then a.message_id end) as op_push_open_pv, --运营push push_type in ('70','71')
count(distinct case when push_type is not null and push_type not in ('25','70','71','101','102','103','104','105','106','107','108','111','112','113') then a.message_id end) as auto_push_open_pv, --系统push即自动push
count(distinct case when push_type in ('101','102','103','104','105','106','107','108','111','112','113') then a.message_id end) as personal_push_open_pv, --个性化push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv --签到push
FROM
( --推送点击
SELECT partition_date,cl_id,time_str,params['message_id'] as message_id,time_stamp
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND action='notification_open'
GROUP BY partition_date,cl_id,time_str,params['message_id'],time_stamp
)a
LEFT JOIN
(
SELECT msg_id,push_type,time_stamp
FROM online.tl_hdfs_push2_new_view --增量表
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') AND partition_date <=regexp_replace(date_sub(current_date,1),'-','')
GROUP BY msg_id,push_type,time_stamp
)b
on b.msg_id=a.message_id
group by a.partition_date,a.cl_id
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
GROUP BY a.partition_date,device_os_type,device_type,channel
)t13
ON t1.partition_date = t13.partition_date
AND t1.device_os_type = t13.device_os_type
AND t1.device_type = t13.device_type
AND t1.channel = t13.channel
LEFT JOIN
(--新增注册设备数/注册用户数
SELECT t1.partition_date,device_os_type,device_type,channel
,count(DISTINCT t1.device_id) as login_dev_num
,count(DISTINCT t1.user_id) as login_user_num
,count(DISTINCT CASE WHEN t3.user_id is not NULL THEN t2.user_id END) as login_phone_user_num
FROM
(
SELECT partition_date,device_id,a.user_id as user_id
FROM online.ml_device_updates
LATERAL VIEW explode(user_list)a as user_id
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)t1
JOIN
(
SELECT user_id,regexp_replace(substr(date_joined,1,10),'-','') as date_joined
FROM online.ml_user_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND regexp_replace(substr(date_joined,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
AND regexp_replace(substr(date_joined,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
)t2
ON t2.user_id=t1.user_id and t2.date_joined=t1.partition_date
LEFT JOIN
(
SELECT user_id,auth_type,phone
FROM online.tl_hdfs_userextra_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND auth_type = '1'
)t3
ON t2.user_id = t3.user_id
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON t2.user_id=c.user_id
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)t4
ON t1.device_id = t4.device_id
AND t1.partition_date = t4.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY t1.partition_date,device_os_type,device_type,channel
)t14
ON t1.partition_date = t14.partition_date
AND t1.device_os_type = t14.device_os_type
AND t1.device_type = t14.device_type
AND t1.channel = t14.channel
pm/daily_operation/job/step1_1.job 0 → 100644
View file @ a6997013
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_operation/job/step1_10.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_diary_view
\ No newline at end of file
pm/daily_operation/job/step1_11.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_problem_view
\ No newline at end of file
pm/daily_operation/job/step1_12.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_question_view
\ No newline at end of file
pm/daily_operation/job/step1_13.job 0 → 100644
View file @ a6997013
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/daily_operation/job/step1_14.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_view
\ No newline at end of file
pm/daily_operation/job/step1_15.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file
pm/daily_operation/job/step1_16.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
pm/daily_operation/job/step1_17.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file
pm/daily_operation/job/step1_18.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file
pm/daily_operation/job/step1_19.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/daily_operation/job/step1_2.job 0 → 100644
View file @ a6997013
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/daily_operation/job/step1_20.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_topicreply_view
\ No newline at end of file
pm/daily_operation/job/step1_21.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_reply_view
\ No newline at end of file
pm/daily_operation/job/step1_22.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file
pm/daily_operation/job/step1_23.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive bl bl_et_bg_trackingpushlog_inc_d
\ No newline at end of file
pm/daily_operation/job/step1_24.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_auroracallback_view
\ No newline at end of file
pm/daily_operation/job/step1_25.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_jpushuserdevicebinding_view
\ No newline at end of file
pm/daily_operation/job/step1_26.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file
pm/daily_operation/job/step1_27.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file
pm/daily_operation/job/step1_3.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/daily_operation/job/step1_4.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file
pm/daily_operation/job/step1_5.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive dim DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file
pm/daily_operation/job/step1_6.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_channel_d
\ No newline at end of file
pm/daily_operation/job/step1_7.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml mid_ml_c_et_pe_preciseexposure_dimen_d
\ No newline at end of file
pm/daily_operation/job/step1_8.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_ck_click_dimen_d
\ No newline at end of file
pm/daily_operation/job/step1_9.job 0 → 100644
View file @ a6997013
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file
pm/daily_operation/job/step2.job 0 → 100644
View file @ a6997013
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17,step1_18,step1_19,step1_20,step1_21,step1_22,step1_23,step1_24,step1_25,step1_26,step1_27
command=sh /home/bi/bi-report/lib/shell/hive daily_operation
\ No newline at end of file
pm/daily_operation/job/step3.job 0 → 100644
View file @ a6997013
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_operation/liudi@igengmei.com,wangxin@igengmei.com,xuepengfei@igengmei.com,zhaoyang@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/daily_operation/report/daily_operation.sql 0 → 100644
View file @ a6997013
SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(day_id,'yyyymmdd'),'yyyy-mm-dd') `日期`
,device_os_type `系统`
,active_type `新老`
,channel `渠道`
,dau as `dau（去除积分墙）`
,nvl(login_dev_num,0) as `新增注册设备数`
,nvl(login_user_num,0) as `新增注册账号`
,nvl(login_phone_user_num,0) as `新增注册手机号用户数`
,nvl(retention_1,0) as `次日留存率`
,nvl(retention_7,0) as `第7日留存率`
,nvl(retention_30,0) as `第30日留存率`
,nvl(per_open_times,0) as `单设备打开频次`
,nvl(home_feeds_ctr,0) as `信息流内容ctr`
,nvl(per_use_time,0) as `单设备app停留时长(M)`
,nvl(neirong_page_stay,0) as `单设备内容时长(M)`
,nvl(neirong_in_app_use_time,0) as `单设备内容时长/app停留时长`
,nvl(service_privacy_uv,0) as `到达隐私授权界面设备数`
,nvl(service_click_uv,0) as `同意隐私授权设备设备数`
,nvl(service_privacy_uv_in_dau,0) as `到达隐私授权界面设备数/DAU`
,nvl(service_click_uv_in_dau,0) as `同意隐私授权设备数/到达隐私授权界面设备数`
,nvl(open_push_uv,0) as `开启推送授权设备数`
,nvl(open_push_uv_in_dau,0) as `开启推送授权设备数/DAU`
,nvl(push_receive_uv_in_open_push_uv,0) as `次日接收推送设备数/当天开启推送授权设备数`
,nvl(click_uv,0) as `点击推送的设备数`
,nvl(non_click_uv,0) as `未点击推送的设备数`
,nvl(click_uv_in_dau,0) as `点击推送设备数/DAU`
,nvl(non_click_uv_in_dau,0) `未点击推送设备数/DAU`
,nvl(op_push_open_uv,0) `点击运营推送设备数`
,nvl(auto_push_open_uv,0) as `点击系统推送设备数`
,nvl(personal_push_open_uv,0) `点击个性化推送设备数`
,nvl(sign_push_open_uv,0) as `点击签到推送设备数`
,nvl(op_push_open_uv_in_receive,0) as `点击运营推送设备/接收到运营推送设备`
,nvl(auto_push_open_uv_in_receive,0) as `点击系统推送设备/接收到系统推送设备`
,nvl(personal_push_open_uv_in_receive,0) as `点击个性化推送设备/接收到个性化推送设备`
,nvl(sign_push_open_uv_in_receive,0) as `点击签到推送设备/接收到签到推送设备`
,nvl(question_num,0) as `主动提问数`
,nvl(answer_num,0) as `主动回答数`
,nvl(topic_num,0) as `主动发帖数`
,nvl(reply_num,0) as `真实评论数`
,nvl(question_num_in_dau,0) as `主动提问设备数/DAU`
,nvl(answer_num_in_dau,0) as `主动回答设备数/DAU`
,nvl(topic_num_in_dau,0) as `主动发帖设备数/DAU`
,nvl(reply_num_in_dau,0) as `主动评论设备数/DAU`
,nvl(neirongUV,0) as `内容UV`
,nvl(neirongPV,0) `内容PV`
,nvl(neirongUV_in_dau,0) `内容UV/DAU`
,nvl(neirongPV_in_UV,0) `内容PV/内容UV`
,nvl(result_more_UV,0) as `搜索结果综合页UV`
,nvl(result_meigou_UV,0) as `搜索结果美购页UV`
,nvl(result_con_UV,0) as `搜索结果商业页UV`
,nvl(result_more_UV_in_dau,0) as `搜索结果综合页UV/DAU`
,nvl(result_meigou_UV_in_dau,0) as `搜索结果美购页UV/DAU`
,nvl(result_con_UV_in_dau,0) as `搜索结果商业页UV/DAU`
,nvl(result_more_PV,0) as `搜索结果综合页PV`
,nvl(result_meigou_PV,0) as `搜索结果美购页PV`
,nvl(result_con_PV,0) as `搜索结果商业页PV`
,nvl(result_more_PV_in_UV,0) as `搜索结果综合页PV/搜索结果综合页UV`
,nvl(result_meigou_PV_in_UV,0) as `搜索结果美购页PV/搜索结果美购页UV`
,nvl(result_con_PV_in_UV,0) `搜索结果商业页PV/搜索结果商业页UV`
,nvl(result_more_PV_in_dau,0) `搜索结果综合页PV/DAU`
,nvl(result_meigou_PV_in_dau,0) `搜索结果美购页PV/DAU`
,nvl(result_con_PV_in_dau,0) as `搜索结果商业页PV/DAU`
,nvl(neirong_meigou_PV,0) as `来自内容页的美购页PV`
,nvl(neirong_con_PV,0) as `来自内容页的商业页PV`
,nvl(neirong_meigou_PV_in_dau,0) as `来自内容页的美购页PV/DAU`
,nvl(neirong_con_PV_in_dau,0) as `来自内容页的商业页PV/DAU`
,nvl(neirong_click_pv,0) as `内容页的推荐内容点击pv`
,nvl(wel_second_pv_in_neirongPV,0) as `来自内容页的商业二跳/内容pv`
,nvl(neirong_second_pv_in_neirongPV,0) as `来自内容页的内容二跳/内容pv`
FROM pm.tl_pm_operation_d
where ((partition_day ='20200824' and day_id<='20200726')
or (partition_day<regexp_replace(date_sub(current_date,1),'-','') and partition_day>'20200824' and day_id=regexp_replace(date_sub(FROM_UNIXTIME(UNIX_TIMESTAMP(partition_day,'yyyymmdd'),'yyyy-mm-dd'),29),'-',''))
or (partition_day=regexp_replace(date_sub(current_date,1),'-','') and day_id>=regexp_replace(date_sub(current_date,30),'-','')))
order by `日期`,`系统`,`新老`,`渠道`
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ a6997013
...@@ -60,7 +60,8 @@ FROM ...@@ -60,7 +60,8 @@ FROM
( (
SELECT partition_date, SELECT partition_date,
cl_id, cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type, case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end as card_content_type,
CASE when transaction_type in ('fmctr') then array('fmctr','合计') CASE when transaction_type in ('fmctr') then array('fmctr','合计')
WHEN transaction_type like '%ctr' THEN array('ctr预估','合计') WHEN transaction_type like '%ctr' THEN array('ctr预估','合计')
WHEN transaction_type like '%cvr' THEN array('cvr预估','合计') WHEN transaction_type like '%cvr' THEN array('cvr预估','合计')
...@@ -78,9 +79,10 @@ FROM ...@@ -78,9 +79,10 @@ FROM
AND tab_name = '精选' AND tab_name = '精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed') AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or transaction_type like '%ctr' or transaction_type like '%cvr') or transaction_type like '%ctr' or transaction_type like '%cvr')
AND card_content_type in ('qa','diary','user_post','answer') AND card_content_type in ('qa','diary','user_post','answer','special_pool')
group by partition_date, group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end, case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end,
cl_id, cl_id,
CASE when transaction_type in ('fmctr') then array('fmctr','合计') CASE when transaction_type in ('fmctr') then array('fmctr','合计')
WHEN transaction_type like '%ctr' THEN array('ctr预估','合计') WHEN transaction_type like '%ctr' THEN array('ctr预估','合计')
...@@ -108,7 +110,8 @@ FROM ...@@ -108,7 +110,8 @@ FROM
( (
SELECT partition_date, SELECT partition_date,
cl_id, cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type, case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end as card_content_type,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计') CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计') WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计') WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
...@@ -125,10 +128,11 @@ FROM ...@@ -125,10 +128,11 @@ FROM
AND params['tab_name'] = '精选' AND params['tab_name'] = '精选'
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed') AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr') or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
AND params['card_content_type'] in ('qa','diary','user_post','answer') AND params['card_content_type'] in ('qa','diary','user_post','answer','special_pool')
GROUP BY partition_date, GROUP BY partition_date,
cl_id, cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end, case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计') CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计') WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计') WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
...@@ -149,21 +153,35 @@ FROM ...@@ -149,21 +153,35 @@ FROM
and t2.recommend_type=t3.recommend_type and t2.recommend_type=t3.recommend_type
LEFT JOIN LEFT JOIN
(--页面浏览时长 (--页面浏览时长
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,page_stay
from
(
SELECT partition_date,cl_id,business_id, SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
sum(page_stay) as page_stay sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='page_view' AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND referrer='home' AND referrer='home'
AND page_stay>=0 AND page_stay<1000 AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id, GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t4 )t4
on t4.partition_date=t3.partition_date on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id and t4.cl_id=t3.cl_id
...@@ -171,22 +189,36 @@ FROM ...@@ -171,22 +189,36 @@ FROM
and t4.page_name=t3.card_content_type and t4.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--搜索框和点击行为 (--搜索框和点击行为
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,navbar_pv
from
(
SELECT partition_date,cl_id,business_id, SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as navbar_pv count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('on_click_navbar_search','do_search') AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or AND (referrer='home' or
(params['referrer_link'] like '%[%' and (params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home')) json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id, group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t5 )t5
on t5.partition_date=t3.partition_date on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id and t5.cl_id=t3.cl_id
...@@ -194,23 +226,37 @@ FROM ...@@ -194,23 +226,37 @@ FROM
and t5.page_name=t3.card_content_type and t5.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--点击高亮词 (--点击高亮词
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,highlight_pv
from
(
SELECT partition_date,cl_id,business_id, SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as highlight_pv count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card' AND action='on_click_card'
and params['card_type']='highlight_word' and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or AND (referrer='home' or
(params['referrer_link'] like '%[%' and (params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home')) json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id, group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t6 )t6
on t6.partition_date=t3.partition_date on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id and t6.cl_id=t3.cl_id
...@@ -218,13 +264,14 @@ FROM ...@@ -218,13 +264,14 @@ FROM
and t6.page_name=t3.card_content_type and t6.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--关联的美购卡片 (--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as self_wel_pv
FROM FROM
( (
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id, SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as pv count(1) as pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
...@@ -242,9 +289,18 @@ FROM ...@@ -242,9 +289,18 @@ FROM
group by partition_date,cl_id,business_id,app_session_id,params['card_id'], group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a )a
group by partition_date,cl_id,business_id,page_name left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t7 )t7
on t7.partition_date=t3.partition_date on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id and t7.cl_id=t3.cl_id
...@@ -252,29 +308,39 @@ FROM ...@@ -252,29 +308,39 @@ FROM
and t7.page_name=t3.card_content_type and t7.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购) (--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as recom_wel_pv
FROM FROM
( (
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id, SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as service_pv count(1) as service_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (action='on_click_card'and params['card_content_type']='service' AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold' and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail') or action='on_click_button' and params['button_name']='unfold' and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail')
or action='on_click_button' and params['button_name'] = 'more_recommendations') or action='on_click_button' and params['button_name'] = 'more_recommendations')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or AND (referrer='home' or
(params['referrer_link'] like '%[%' and (params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home')) json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'], group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a )a
group by partition_date,cl_id,business_id,page_name left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t8 )t8
on t8.partition_date=t3.partition_date on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id and t8.cl_id=t3.cl_id
...@@ -282,76 +348,114 @@ FROM ...@@ -282,76 +348,114 @@ FROM
and t8.page_name=t3.card_content_type and t8.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--推荐的内容卡片 (--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as recom_content_pv
FROM FROM
( (
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id, SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as service_pv count(1) as service_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card' AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer') and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or AND (referrer='home' or
(params['referrer_link'] like '%[%' and (params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home')) json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'], group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a )a
group by partition_date,cl_id,business_id,page_name left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t9 )t9
on t9.partition_date=t3.partition_date on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type and t9.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--转诊点击，视频面诊点击 (--视频面诊点击
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,video_pv
from
(
SELECT partition_date,cl_id,business_id, SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as video_pv count(1) as video_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button' AND action='on_click_button'
and params['button_name']='video_interview' and params['button_name']='video_interview'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or AND (referrer='home' or
(params['referrer_link'] like '%[%' and (params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home')) json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id, group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t10 )t10
on t10.partition_date=t3.partition_date on t10.partition_date=t3.partition_date
and t10.cl_id=t3.cl_id and t10.cl_id=t3.cl_id
and t10.business_id=t3.card_id and t10.business_id=t3.card_id
and t10.page_name=t3.card_content_type and t10.page_name=t3.card_content_type
LEFT JOIN LEFT JOIN
(--推荐的内容卡片 (--转诊按钮点击
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,referral_pv
from
(
SELECT partition_date,cl_id,business_id, SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name, when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as referral_pv count(1) as referral_pv
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button' AND action='on_click_button'
and params['button_name']='referral' and params['button_name']='referral'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or AND (referrer='home' or
(params['referrer_link'] like '%[%' and (params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home')) json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id, group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary' case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post' when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t11 )t11
on t11.partition_date=t3.partition_date on t11.partition_date=t3.partition_date
and t11.cl_id=t3.cl_id and t11.cl_id=t3.cl_id
......
pm/daily_recommend_strategy/report/home_recommend_strategy.sql
View file @ a6997013
...@@ -19,5 +19,5 @@ SELECT ...@@ -19,5 +19,5 @@ SELECT
-- '未配置' as `来自I的推荐专题点击pv`, -- '未配置' as `来自I的推荐专题点击pv`,
transfer_card as `来自I的转诊点击pv`, transfer_card as `来自I的转诊点击pv`,
video_consultation as `来自I的视频面诊点击pv`FROM pm.tl_pm_recommend_strategy_d video_consultation as `来自I的视频面诊点击pv`FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200817' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','') WHERE partition_day>='20200824' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`; order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment