Skip to content

B
bi-report
Commit 889dd8a8 authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent 2646d22e
Show whitespace changes
Inline Side-by-side
Showing with 2073 additions and 63 deletions
pm/clue_daily/etl/clue_daily.sql
View file @ 889dd8a8
......@@ -13,15 +13,21 @@ ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_userclue_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT T1.partition_date as day_id
,T1.device_os_type
,T1.active_type
,T1.grey_type
,T1.channel
,dau
,user_num
,total_valid_dev
,total_valid_num
,phone_user_num
,nvl(total_clue_dev,0)
,nvl(total_clue_num,0)
,nvl(total_valid_dev,0)
,nvl(total_valid_num,0)
,msg_dev
,msg_num
,valid_msg_dev
......@@ -87,8 +93,13 @@ FROM
,device_os_type
,active_type
,grey_type
,channel
,count(distinct mas.device_id) as dau
,count(distinct a.user_id) as user_num
,count(distinct b.user_id) as phone_user_num
,nvl(count(distinct case when t13.total_clue_num>0 then t13.user_id end),0) total_clue_dev
,nvl(sum(t13.total_clue_num),0) total_clue_num
,nvl(count(distinct case when t13_v.total_valid_num>0 then t13_v.user_id end),0) total_valid_dev
,nvl(sum(t13_v.total_valid_num),0) total_valid_num
......@@ -142,23 +153,47 @@ FROM
FROM
(
SELECT partition_date,device_id,device_os_type,b.active_type,b.grey_type
SELECT partition_date,device_id,device_os_type,b.active_type,b.grey_type,b.channel
FROM
(
SELECT
partition_date
,device_id,device_os_type
,m.device_id,device_os_type
,array(case when active_type in ('1','2') then '新增设备'
when active_type ='4' then '老活跃设备' end,'合计') as active_type
,array(case when substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
FROM online.ml_device_day_active_status
,array(case when substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and device_id is not null
and m.device_id IS NOT NULL
)a
LATERAL VIEW explode(a.grey_type) b AS grey_type
LATERAL VIEW explode(a.active_type) b AS active_type
LATERAL VIEW explode(a.channel) b AS channel
)mas
left join
(--找出user_id当天活跃的第一个设备id
......@@ -170,6 +205,14 @@ FROM
)a
on mas.partition_date=a.partition_date and mas.device_id=a.device_id
left join
(
SELECT distinct user_id
FROM online.tl_hdfs_userextra_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND auth_type = '1'
)b
on a.user_id=b.user_id
left join
(--私信线索总数
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as msg_num
......@@ -268,7 +311,7 @@ FROM
)t3
ON a.partition_date = t3.partition_date and a.user_id=t3.user_id
LEFT JOIN
( ---拨打电话线索
( ---有效电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
......@@ -1095,6 +1138,39 @@ FROM
)t12_v
ON a.partition_date = t12_v.partition_date and a.user_id=t12_v.user_id
left join
(
SELECT partition_date,user_id,sum(clue_num) as total_clue_num
FROM
(
SELECT msg.partition_date,user_id,count(1) as clue_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
GROUP BY msg.partition_date,user_id
union all
--电话线索总数
SELECT partition_date,user_id,count(1) AS clue_num
FROM
(
SELECT user_id,merchant_id,MIN(REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '')) as partition_date,regexp_replace(SUBSTR(created_time,1,7),'-','') AS CALL_MONTH
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,7),'-','')
)a
GROUP BY partition_date,user_id
)a
group by partition_date,user_id
)t13
ON a.partition_date = t13.partition_date and a.user_id=t13.user_id
left join
(
SELECT partition_date,user_id,sum(valid_num) as total_valid_num
from
......@@ -1178,12 +1254,12 @@ FROM
group by partition_date,user_id
)t13_v
ON a.partition_date = t13_v.partition_date and a.user_id=t13_v.user_id
group by mas.partition_date,device_os_type,active_type,grey_type
group by mas.partition_date,device_os_type,active_type,grey_type,channel
)T1
LEFT JOIN
(
SELECT t3_d.partition_date,a.device_os_type,a.grey_type
,a.active_type
,a.active_type,a.channel
,nvl(count(distinct case when t3_d.call_num>0 then t3_d.user_id end),0) as call_dev_d
,nvl(sum(t3_d.call_num),0) as call_num_d
......@@ -1209,19 +1285,26 @@ LEFT JOIN
,nvl(sum(t9_v_d.valid_call_num),0) as valid_call_num_qa_d
FROM
(
SELECT device_id,device_os_type,user_id,v.grey_type,v.active_type
SELECT device_id,device_os_type,user_id,v.grey_type,v.active_type,v.channel
FROM
(
SELECT device_id,device_os_type,user_id
,array(case when substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
SELECT c.device_id,device_os_type,user_id
,array(case when substr(convup(setencryption(c.device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
,array('老活跃设备','合计') as active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM
(
SELECT device_id,device_os_type,a.user_id,a.partition_date
SELECT device_id,device_os_type,a.user_id,a.partition_date,first_channel_source_type
,row_number() over(partition by device_id order by a.partition_date desc) as rn
FROM
(
SELECT device_id,v.device_os_type,v.user_id,partition_date
,if(size(channel_source_type_list) > 0, channel_source_type_list [ 0 ], '') AS first_channel_source_type
FROM online.ml_device_updates
LATERAL VIEW EXPLODE(user_list) v as user_id
LATERAL VIEW EXPLODE(device_os_type) v as device_os_type
......@@ -1237,10 +1320,34 @@ LEFT JOIN
)b
on a.partition_date=b.partition_date and a.user_id=b.user_id
)c
LEFT JOIN
(
SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
)tmp
on c.first_channel_source_type=tmp.code and c.partition_date=tmp.partition_day
LEFT JOIN
(
SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-','')
)a
ON c.device_id = a.device_id
LEFT JOIN
(
SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id
)b
ON c.device_id = b.device_id AND c.partition_date = b.day_id
where rn=1
)d
LATERAL VIEW EXPLODE(d.grey_type)v as grey_type
LATERAL VIEW EXPLODE(d.active_type)v as active_type
LATERAL VIEW explode(d.channel) v AS channel
)a
JOIN
( ---非当天拨打电话线索
......@@ -1798,9 +1905,12 @@ LEFT JOIN
GROUP BY user_id,partition_date
)t12_v_d
ON t3_d.partition_date = t12_v_d.partition_date and t3_d.user_id=t12_v_d.user_id
group by t3_d.partition_date,a.device_os_type,a.grey_type,a.active_type
group by t3_d.partition_date,a.device_os_type,a.grey_type,a.active_type,a.channel
)T2
on T1.partition_date=T2.partition_date
AND T1.device_os_type=T2.device_os_type
AND T1.active_type=T2.active_type
AND T1.grey_type=T2.grey_type
AND T1.channel=T2.channel
pm/clue_daily/etl/create_clue_daily.sql
View file @ 889dd8a8
......@@ -20,10 +20,14 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_userclue_d
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau bigint comment '{"chs_name":"dau(不去除积分墙)","description":"","etl":"","value":"","remark":""}',
user_num bigint comment '{"chs_name":"注册用户","description":"","etl":"","value":"","remark":""}',
phone_user_num bigint comment '{"chs_name":"手机号注册用户","description":"","etl":"","value":"","remark":""}',
total_clue_dev bigint comment '{"chs_name":"总咨询设备","description":"","etl":"","value":"","remark":""}',
total_clue_num bigint comment '{"chs_name":"总线索","description":"","etl":"","value":"","remark":""}',
total_valid_dev bigint comment '{"chs_name":"总有效咨询设备","description":"","etl":"","value":"","remark":""}',
total_valid_num bigint comment '{"chs_name":"总有效线索","description":"","etl":"","value":"","remark":""}',
msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
msg_dev bigint comment '{"chs_name":"私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
valid_msg_dev bigint comment '{"chs_name":"有效私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
......
pm/clue_daily/job/step1_14.job 0 → 100644
View file @ 889dd8a8
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file
pm/clue_daily/job/step2.job
View file @ 889dd8a8
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
command=sh /home/bi/bi-report/lib/shell/hive clue_daily
\ No newline at end of file
pm/clue_daily/report/clue_daily.sql
View file @ 889dd8a8
......@@ -2,22 +2,28 @@ SELECT day_id `日期`
,device_os_type `系统`
,active_type `新老`
,grey_type `是否灰度`
,channel `渠道`
,dau as `dau（不去除积分墙）`
,user_num as `注册的用户数`
,phone_user_num as `手机号注册用户数`
,total_clue_dev as `总咨询设备（去重）`
,total_clue_num as `总线索`
,total_valid_dev+valid_call_dev_d as `总有效咨询设备（去重）`
,total_valid_num+valid_call_num_d as `总有效线索`
,concat(round((total_valid_dev+valid_call_dev_d)/dau,2)*100,'%') as `总有效线索设备(去重)/dau`
,concat(round((total_valid_num+valid_call_num_d)/dau,2)*100,'%') as `总有效线索/dau`
,nvl(concat(round((total_valid_dev+valid_call_dev_d)/dau*100,2),'%'),0) as `总有效线索设备(去重)/dau`
,nvl(concat(round((total_valid_num+valid_call_num_d)/dau*100,2),'%'),0) as `总有效线索/dau`
,nvl(concat(round((total_valid_dev+valid_call_dev_d)/total_clue_dev*100,2),'%'),0) as `总有效线索设备(去重)/总咨询设备(去重)`
,nvl(concat(round((total_valid_num+valid_call_num_d)/total_clue_num*100,2),'%'),0) as `总有效线索/总线索`
,msg_dev as `私信咨询设备数-总计`
,msg_num as `私信咨询人次-总计`
,valid_msg_dev as `有效私信咨询设备数-总计`
,valid_msg_num as `有效私信咨询人次-总计`
,concat(round(valid_msg_num/msg_num,2)*100,'%') as `有效私信咨询人次/私信咨询人次`
,nvl(concat(round(valid_msg_num/msg_num*100,2),'%'),0) as `有效私信咨询人次/私信咨询人次`
,clue_dev as `授权电话线索设备数-总计`
,clue_num as `授权电话线索人次-总计`
,call_dev+call_dev_d as `拨打电话线索设备数-总计`
,call_num+call_num_d as `拨打电话线索人次-总计`
,concat(round((call_num+call_num_d)/clue_num,2)*100,'%') as `拨打电话人次/授权电话人次`
,nvl(concat(round((call_num+call_num_d)/clue_num*100,2),'%'),0) as `拨打电话人次/授权电话人次`
,valid_call_dev+valid_call_dev_d as `有效电话线索设备数-总计`
,valid_call_num+valid_call_num_d as `有效电话线索人次-总计`
,call_dev as `当天拨打电话线索设备数-总计`
......@@ -27,18 +33,18 @@ SELECT day_id `日期`
,msg_dev_3 `私信咨询设备数-3级以上商户`
,msg_num_3 `私信咨询人次-3级以上商户`
,concat(round(msg_num_3/msg_num,2)*100,'%') as `私信咨询人次-3级以上/私信咨询人次`
,nvl(concat(round(msg_num_3/msg_num*100,2),'%'),0) as `私信咨询人次-3级以上/私信咨询人次`
,valid_msg_dev_3 `有效私信咨询设备数-3级以上商户`
,valid_msg_num_3 as `有效私信咨询人次-3级以上商户`
,clue_dev_3 as `授权电话线索设备数-3级以上商户`
,clue_num_3 as `授权电话线索人次-3级以上商户`
,concat(round(clue_num_3/clue_num,2)*100,'%') as `授权电话线索-3级以上/授权电话线索`
,nvl(concat(round(clue_num_3/clue_num*100,2),'%'),0) as `授权电话线索-3级以上/授权电话线索`
,call_dev_3+call_dev_3_d as `拨打电话线索设备数-3级以上商户`
,call_num_3+valid_call_num_3 as `拨打电话线索人次-3级以上商户`
,concat(round((call_num_3+valid_call_num_3)/clue_num_3,2)*100,'%') as `拨打电话线索人次-3级以上/授权电话线索人次-3级以上`
,nvl(concat(round((call_num_3+valid_call_num_3)/clue_num_3*100,2),'%'),0) as `拨打电话线索人次-3级以上/授权电话线索人次-3级以上`
,valid_call_dev_3+valid_call_dev_3_d as `有效电话线索设备数-3级以上商户`
,valid_call_num_3+valid_call_num_3_d as `有效电话线索人次-3级以上商户`
,concat(round((valid_call_num_3+valid_call_num_3_d)/clue_num_3,2)*100,'%') as `有效电话线索人次-3级以上/授权电话线索人次-3级以上`
,nvl(concat(round((valid_call_num_3+valid_call_num_3_d)/clue_num_3*100,2),'%'),0) as `有效电话线索人次-3级以上/授权电话线索人次-3级以上`
,call_dev_3 as `当天拨打电话线索设备数-3级以上商户`
,call_num_3 as `当天拨打电话线索人次-3级以上商户`
,valid_call_dev_3 as `当天有效电话线索设备数-3级以上商户`
......
pm/daily_operation/en-cn.properties 0 → 100644
View file @ 889dd8a8
clue_daily=线索日报（用户侧）
\ No newline at end of file
pm/daily_operation/etl/create_daily_operation.sql 0 → 100644
View file @ 889dd8a8
--***************************************************************
--*脚本名称:
--*功能: 运营日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_operation_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau bigint comment '{"chs_name":"dau(去除积分墙)","description":"","etl":"","value":"","remark":""}',
login_dev_num bigint comment '{"chs_name":"新增注册设备数","description":"","etl":"","value":"","remark":""}',
login_user_num bigint comment '{"chs_name":"新增注册账号","description":"","etl":"","value":"","remark":""}',
login_phone_user_num bigint comment '{"chs_name":"新增注册手机号用户数","description":"","etl":"","value":"","remark":""}',
retention_1 string comment '{"chs_name":"次日留存率","description":"","etl":"","value":"","remark":""}',
retention_7 string comment '{"chs_name":"第7日留存率","description":"","etl":"","value":"","remark":""}',
retention_30 string comment '{"chs_name":"第30日留存率","description":"","etl":"","value":"","remark":""}',
per_open_times bigint comment '{"chs_name":"单设备打开频次","description":"","etl":"","value":"","remark":""}',
home_feeds_ctr string comment '{"chs_name":"信息流内容ctr","description":"","etl":"","value":"","remark":""}',
per_use_time double comment '{"chs_name":"单设备app停留时长(M)","description":"","etl":"","value":"","remark":""}',
neirong_page_stay double comment '{"chs_name":"单设备内容时长(M)","description":"","etl":"","value":"","remark":""}',
neirong_in_app_use_time string comment '{"chs_name":"单设备内容时长/app停留时长","description":"","etl":"","value":"","remark":""}',
service_privacy_uv bigint comment '{"chs_name":"到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
service_click_uv bigint comment '{"chs_name":"同意隐私授权设备设备数","description":"","etl":"","value":"","remark":""}',
service_privacy_uv_in_dau string comment '{"chs_name":"到达隐私授权界面设备数/DAU","description":"","etl":"","value":"","remark":""}',
service_click_uv_in_dau string comment '{"chs_name":"同意隐私授权设备数/到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
open_push_uv bigint comment '{"chs_name":"开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
open_push_uv_in_dau string comment '{"chs_name":"开启推送授权设备数/DAU","description":"","etl":"","value":"","remark":""}',
push_receive_uv_in_open_push_uv string comment '{"chs_name":"次日接收推送设备数/当天开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
click_uv bigint comment '{"chs_name":"点击推送的设备数","description":"","etl":"","value":"","remark":""}',
non_click_uv bigint comment '{"chs_name":"未点击推送的设备数","description":"","etl":"","value":"","remark":""}',
click_uv_in_dau string comment '{"chs_name":"点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
non_click_uv_in_dau string comment '{"chs_name":"未点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
op_push_open_uv bigint comment '{"chs_name":"点击运营推送设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv bigint comment '{"chs_name":"点击系统推送设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv bigint comment '{"chs_name":"点击个性化推送设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv bigint comment '{"chs_name":"点击签到推送设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_uv_in_receive string comment '{"chs_name":"点击运营推送设备/接收到运营推送设备","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv_in_receive string comment '{"chs_name":"点击系统推送设备/接收到系统推送设备","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv_in_receive string comment '{"chs_name":"点击个性化推送设备/接收到个性化推送设备","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv_in_receive string comment '{"chs_name":"点击签到推送设备/接收到签到推送设备","description":"","etl":"","value":"","remark":""}',
question_num bigint comment '{"chs_name":"主动提问数","description":"","etl":"","value":"","remark":""}',
answer_num bigint comment '{"chs_name":"主动回答数","description":"","etl":"","value":"","remark":""}',
topic_num bigint comment '{"chs_name":"主动发帖数","description":"","etl":"","value":"","remark":""}',
reply_num bigint comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
question_num_in_dau string comment '{"chs_name":"主动提问设备数/DAU`","description":"","etl":"","value":"","remark":""}',
answer_num_in_dau string comment '{"chs_name":"主动回答设备数/DAU","description":"","etl":"","value":"","remark":""}',
topic_num_in_dau string comment '{"chs_name":"主动发帖设备数/DAU","description":"","etl":"","value":"","remark":""}',
reply_num_in_dau string comment '{"chs_name":"主动评论设备数/DAU","description":"","etl":"","value":"","remark":""}',
neirongUV bigint comment '{"chs_name":"内容UV","description":"","etl":"","value":"","remark":""}',
neirongPV bigint comment '{"chs_name":"内容PV","description":"","etl":"","value":"","remark":""}',
neirongUV_in_dau string comment '{"chs_name":"内容UV/DAU","description":"","etl":"","value":"","remark":""}',
neirongPV_in_UV double comment '{"chs_name":"内容PV/内容UV","description":"","etl":"","value":"","remark":""}',
result_more_UV bigint comment '{"chs_name":"搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
result_meigou_UV bigint comment '{"chs_name":"搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
result_con_UV bigint comment '{"chs_name":"搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
result_more_UV_in_dau string comment '{"chs_name":"搜索结果综合页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_meigou_UV_in_dau string comment '{"chs_name":"搜索结果美购页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_con_UV_in_dau string comment '{"chs_name":"搜索结果商业页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_more_PV bigint comment '{"chs_name":"搜索结果综合页PV","description":"","etl":"","value":"","remark":""}',
result_meigou_PV bigint comment '{"chs_name":"搜索结果美购页PV","description":"","etl":"","value":"","remark":""}',
result_con_PV bigint comment '{"chs_name":"搜索结果商业页PV","description":"","etl":"","value":"","remark":""}',
result_more_PV_in_UV double comment '{"chs_name":"搜索结果综合页PV/搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
result_meigou_PV_in_UV double comment '{"chs_name":"搜索结果美购页PV/搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
result_con_PV_in_UV double comment '{"chs_name":"搜索结果商业页PV/搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
result_more_PV_in_dau string comment '{"chs_name":"搜索结果综合页PV/DAU","description":"","etl":"","value":"","remark":""}',
result_meigou_PV_in_dau string comment '{"chs_name":"搜索结果美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
result_con_PV_in_dau string comment '{"chs_name":"搜索结果商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_meigou_PV bigint comment '{"chs_name":"来自内容页的美购页PV","description":"","etl":"","value":"","remark":""}',
neirong_con_PV bigint comment '{"chs_name":"来自内容页的商业页PV","description":"","etl":"","value":"","remark":""}',
neirong_meigou_PV_in_dau string comment '{"chs_name":"来自内容页的美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_con_PV_in_dau string comment '{"chs_name":"来自内容页的商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_click_pv bigint comment '{"chs_name":"内容页的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
wel_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的商业二跳/内容pv","description":"","etl":"","value":"","remark":""}',
neirong_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的内容二跳/内容pv","description":"","etl":"","value":"","remark":""}'
)comment '运营日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_operation/etl/daily_operation.sql 0 → 100644
View file @ 889dd8a8
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--运营日报
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_operation_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_date as day_id
,t1.device_os_type as device_os_type
,t1.device_type as active_type
,t1.channel as channel
,t1.active_num as `DAU`
,t14.login_dev_num as login_dev_num
,t14.login_user_num as login_user_num
,t14.login_phone_user_num as login_phone_user_num
,concat(round(t1.retention_num1/t1.active_num*100,1),'%') as retention_1
,concat(round(t1.retention_num7/t1.active_num*100,1),'%') as retention_7
,concat(round(t1.retention_num30/t1.active_num*100,1),'%') as retention_30
,coalesce(t3.per_open_times,0) as per_open_times
,concat(round(coalesce(click_card_pv,0)/coalesce(exp_card_pv,0)*100,3),'%') as home_feeds_ctr
,coalesce(t3.per_use_time,0) as per_use_time
,round(coalesce(t4.neirong_page_stay,0)/coalesce(neirongUV,0)/60,3) as neirong_page_stay
,concat(round(round(coalesce(t4.neirong_page_stay,0)/coalesce(t1.active_num,0)/60,3)/coalesce(t3.per_use_time,0)*100,3),'%') as neirong_in_app_use_time
,coalesce(t4.service_privacy_uv,0) as service_privacy_uv
,coalesce(t4.service_click_uv,0) as service_click_uv
,concat(round(coalesce(t4.service_privacy_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as service_privacy_uv_in_dau
,concat(round(coalesce(t4.service_click_uv,0)/coalesce(t4.service_privacy_uv,0)*100,3),'%') as service_click_uv_in_dau
,coalesce(t11.open_push_uv,0) as open_push_uv
,concat(round(coalesce(t11.open_push_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `open_push_uv_in_dau`
,concat(round(coalesce(t11.push_receive_uv,0)/coalesce(t11.open_push_uv,0)*100,3),'%') as `push_receive_uv_in_open_push_uv`
,coalesce(t10.click_uv,0) as `click_uv`
,coalesce(t1.active_num-t10.click_uv,0) as `non_click_uv`
,concat(round(coalesce(t10.click_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `click_uv_in_dau`
,concat(round(coalesce(t1.active_num-t10.click_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `non_click_uv_in_dau`
,coalesce(t13.op_push_open_uv,0) as `op_push_open_uv`
,coalesce(t13.auto_push_open_uv,0) as `auto_push_open_uv`
,coalesce(t13.personal_push_open_uv,0) as `personal_push_open_uv`
,coalesce(t13.sign_push_open_uv,0) as `sign_push_open_uv`
,concat(round(coalesce(t13.op_push_open_uv,0)/coalesce(t12.op_receive_uv,0)*100,3),'%') as `op_push_open_uv_in_receive`
,concat(round(coalesce(t13.auto_push_open_uv,0)/coalesce(t12.auto_receive_uv,0)*100,3),'%') as `auto_push_open_uv_in_receive`
,concat(round(coalesce(t13.personal_push_open_uv,0)/coalesce(t12.personal_receive_uv,0)*100,3),'%') as `personal_push_open_uv_in_receive`
,concat(round(coalesce(t13.sign_push_open_uv,0)/coalesce(t12.sign_receive_uv,0)*100,3),'%') as `sign_push_open_uv_in_receive`
,coalesce(question_num,0) as `question_num`
,coalesce(answer_num,0) as `answer_num`
,coalesce(topic_num,0) as `topic_num`
,coalesce(reply_num,0) as `reply_num`
,concat(round(coalesce(question_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `question_num_in_dau`
,concat(round(coalesce(answer_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `answer_num_in_dau`
,concat(round(coalesce(topic_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `topic_num_in_dau`
,concat(round(coalesce(reply_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `reply_num_in_dau`
,coalesce(t4.neirongUV,0) as `neirongUV`
,coalesce(t4.neirongPV,0) as `neirongPV`
,concat(round(coalesce(t4.neirongUV,0)/t1.active_num*100,3),'%') as `neirongUV_in_dau`
,round(coalesce(t4.neirongPV,0)/coalesce(t4.neirongUV,0),2) as `neirongPV_in_UV`
--,round(coalesce(t4.neirongPV,0)/t2.active_num,2) as `内容PV/DAU`
,coalesce(result_more_UV,0) as `result_more_UV`
,coalesce(result_meigou_UV,0) as `result_meigou_UV`
,coalesce(result_con_UV,0) as `result_con_UV`
,concat(round(coalesce(result_more_UV,0)/t1.active_num*100,3),'%') as `result_more_UV_in_dau`
,concat(round(coalesce(result_meigou_UV,0)/t1.active_num*100,3),'%') as `result_meigou_UV_in_dau`
,concat(round(coalesce(result_con_UV,0)/t1.active_num*100,3),'%') as `result_con_UV_in_dau`
,coalesce(result_more_PV,0) as `result_more_PV`
,coalesce(result_meigou_PV,0) as `result_meigou_PV`
,coalesce(result_con_PV,0) as `result_con_PV`
,round(coalesce(result_more_PV,0)/coalesce(result_more_UV,0),2) as `result_more_PV_in_UV`
,round(coalesce(result_meigou_PV,0)/coalesce(result_meigou_UV,0),2) as `result_meigou_PV_in_UV`
,round(coalesce(result_con_PV,0)/coalesce(result_con_UV,0),2) as `result_con_PV_in_UV`
,concat(round(coalesce(result_more_PV,0)/t1.active_num*100,2),'%') as `result_more_PV_in_dau`
,concat(round(coalesce(result_meigou_PV,0)/t1.active_num*100,2),'%') as `result_meigou_PV_in_dau`
,concat(round(coalesce(result_con_PV,0)/t1.active_num*100,2),'%') as `result_con_PV_in_dau`
,coalesce(neirong_meigou_PV,0) as `neirong_meigou_PV`
,coalesce(neirong_con_PV,0) as `neirong_con_PV`
,concat(round(coalesce(neirong_meigou_PV,0)/t1.active_num*100,2),'%') as `neirong_meigou_PV_in_dau`
,concat(round(coalesce(neirong_con_PV,0)/t1.active_num*100,2),'%') as `neirong_con_PV_in_dau`
,coalesce(neirong_click_pv,0) as `neirong_click_pv`
,concat(round(coalesce(wel_second_pv,0)/neirongPV*100,2),'%') as `wel_second_pv_in_neirongPV`
,concat(round(coalesce(neirong_second_pv,0)/neirongPV*100,2),'%') as `neirong_second_pv_in_neirongPV`
FROM
(
--日活活跃、留存
SELECT
regexp_replace(substr(t1.partition_date,1,10),'-','') as partition_date
,device_os_type,t1.device_type,channel
,int(count(DISTINCT t1.device_id)) as active_num
,int(count(DISTINCT case WHEN date_add(t1.partition_date,1) = t2.partition_date then t2.device_id end)) as retention_num1
,int(count(DISTINCT case WHEN date_add(t1.partition_date,6) = t2.partition_date then t2.device_id end)) as retention_num7
,int(count(DISTINCT case WHEN date_add(t1.partition_date,29) = t2.partition_date then t2.device_id end)) as retention_num30
FROM
(
SELECT mas.partition_date,mas.device_id,device_type,device_os_type,t2.channel
FROM
(
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
)t1
LEFT JOIN
(
SELECT device_id
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
FROM online.ml_device_day_active_status
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}'
)t2
on t1.device_id = t2.device_id
GROUP BY regexp_replace(substr(t1.partition_date,1,10),'-',''),device_os_type,t1.device_type,channel
)t1
LEFT JOIN
(
--App停留时长、打开频次
SELECT
partition_date,device_os_type,device_type,t.channel
,round(SUM(case when use_duration > 0 and use_duration < 86400 then use_duration else null end)/count(DISTINCT case when use_duration > 0 and use_duration < 86400 then device_id else null end)/60,1) as per_use_time
,round(SUM(open_times)/count(DISTINCT device_id),1) as per_open_times
FROM
(
SELECT b.device_id,use_duration,b.partition_date,open_times,device_os_type,device_type,channel
FROM
(
select device_id,use_duration,partition_date,open_times
from online.ml_device_updates
where partition_date >= '${start_date}'
and partition_date < '${end_date}'
)b
JOIN
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
when active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
AND first_channel_source_type not like 'promotion\_jf\_%'
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
)a
on a.device_id= b.device_id
and a.partition_date = b.partition_date
)a
LATERAL VIEW explode(a.channel) t AS channel
GROUP BY partition_date,device_os_type,device_type,t.channel
)t3
ON t1.partition_date = t3.partition_date
AND t1.device_os_type = t3.device_os_type
AND t1.device_type = t3.device_type
AND t1.channel = t3.channel
LEFT JOIN
(
--内容页、搜索各页PV、UV + 搜索数
SELECT partition_date,cl_type,device_type,channel
,count(case when action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirongPV
,count(DISTINCT case when action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then t1.cl_id else null end) as neirongUV
,count(case when action = 'page_view' AND page_name = 'search_result_more' then 1 else null end) as result_more_PV
,count(DISTINCT case when action = 'page_view' AND page_name = 'search_result_more' then t1.cl_id else null end) as result_more_UV
,count(case when action = 'page_view' AND page_name = 'search_result_welfare' then 1 else null end) as result_meigou_PV
,count(DISTINCT case when action = 'page_view' AND page_name = 'search_result_welfare' then t1.cl_id else null end) as result_meigou_UV
,count(case when action = 'page_view' AND page_name in ('search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization') then 1 else null end) as result_con_PV
,count(DISTINCT case when action = 'page_view' AND page_name in ('search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization') then t1.cl_id else null end) as result_con_UV
,count(case when action = 'page_view' AND page_name = 'welfare_detail' AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirong_meigou_PV
,count(case when action = 'page_view' AND page_name in ('welfare_detail','expert_detail','organization_detail') AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirong_con_PV
,count(CASE WHEN action = 'click_neirong' THEN 1 ELSE NULL END) as neirong_click_pv
,count(CASE WHEN action = 'wel_second' THEN 1 ELSE NULL END) as wel_second_pv
,count(CASE WHEN action = 'neirong_second' THEN 1 ELSE NULL END) as neirong_second_pv
,sum(CASE WHEN action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND (page_stay>=0 AND page_stay<1000) then page_stay else null end) as neirong_page_stay
,count(DISTINCT CASE WHEN action = 'popup_view' AND page_name = 'service_privacy' THEN t1.cl_id END) as service_privacy_uv
,count(DISTINCT CASE WHEN action = 'popup_click' AND page_name = 'service_privacy' THEN t1.cl_id END) as service_click_uv
FROM
(
SELECT
a.cl_id as cl_id
,a.partition_date
,page_name,cl_type,device_type,channel,referrer,action,page_stay
FROM
(
SELECT page_name,cl_id,partition_date,cl_type,referrer,action,page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}'
and action = 'page_view'
AND ((page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail'
,'search_result_more','search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization'))
or (page_name = 'welfare_detail' AND params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail'))
or (page_name in ('welfare_detail','expert_detail','organization_detail') AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')))
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'click_neirong' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}'
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
and (action ='on_click_card' and params['card_content_type'] in ('qa','diary','user_post','answer')
or action='post_detail_browse_card')
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'wel_second' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}'
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
and params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND ((action='on_click_button' and params['button_name'] = 'more_recommendations')
or (action='on_click_button' and params['button_name']='video_interview')
or (action='on_click_button' and params['button_name']='referral')
or (action ='on_click_card' and params['card_content_type']='service'))
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'neirong_second' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}'
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') --内容详情页-搜索首页-进行搜索
and (action in ('on_click_navbar_search','do_search')
or (action ='on_click_card' and params['card_type']='highlight_word')
or (action='on_click_card' and params['card_content_type'] in ('qa','diary','user_post','answer')))
UNION ALL
SELECT 'service_privacy' as page_name,cl_id,partition_date,cl_type,'no' as referrer,'popup_view' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}' --隐私弹窗弹出事件
and action = 'popup_view'
AND params['popup_name'] ='service_privacy'
UNION ALL
SELECT 'service_privacy' as page_name,cl_id,partition_date,cl_type,'no' as referrer,'popup_click' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}'
and partition_date < '${end_date}' --隐私弹窗弹出事件
and action = 'on_click_button'
AND params['popup_name'] ='service_privacy'
AND params['type']='service_privacy'
AND params['button_name']='agree'
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel
FROM
(
SELECT
partition_date,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
when active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
LEFT JOIN
( -- 2.去掉疑似机构刷量的PV和UV
SELECT device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day='${partition_date}'
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
GROUP BY t1.partition_date,cl_type,device_type,channel
)t4
ON t1.partition_date = t4.partition_date
AND t1.device_os_type = t4.cl_type
AND t1.device_type = t4.device_type
AND t1.channel = t4.channel
LEFT JOIN
( --来自信息流推荐内容的曝光卡片量
SELECT partition_date,device_os_type,active_type,t4.channel
,SUM(exp_card_pv) as exp_card_pv
FROM
(
SELECT mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id,count(DISTINCT t1.card_id) as exp_card_pv
FROM
(
SELECT partition_date
,device_os_type,m.device_id
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
WHERE partition_date >= '${start_date}' and partition_date < '${end_date}'
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
JOIN
(
SELECT partition_day as partition_date,device_id as cl_id,card_id,app_session_id
FROM ml.mid_ml_c_et_pe_preciseexposure_dimen_d --新仓库精准曝光
WHERE partition_day >= '${start_date}'and partition_day < '${end_date}'
AND page_code='home'
AND is_exposure='1'
AND tab_code='精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND card_content_type in ('user_post','diary','qa','answer')
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day='${partition_date}'
)spam_pv
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id
)t3
LATERAL VIEW explode(t3.channel) t4 AS channel
group by partition_date,device_os_type,active_type,t4.channel
)t5
ON t1.partition_date = t5.partition_date
AND t1.device_os_type = t5.device_os_type
AND t1.device_type = t5.active_type
AND t1.channel = t5.channel
LEFT JOIN
( --来自信息流推荐内容的卡片点击量
SELECT partition_date,device_os_type,active_type,t4.channel
,SUM(click_card_pv) as click_card_pv
FROM
(
SELECT mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id,count(DISTINCT t1.card_id) as click_card_pv
FROM
(
SELECT partition_date
,device_os_type,m.device_id
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
WHERE partition_date >= '${start_date}' and partition_date < '${end_date}'
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
JOIN
(
SELECT partition_day as partition_date,device_id as cl_id,card_id,app_session_id
FROM ml.ml_c_et_ck_click_dimen_d --新仓库点击表
WHERE partition_day >= '${start_date}'and partition_day < '${end_date}'
AND action='on_click_card'
AND page_code ='home'
AND tab_code = '精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND card_content_type in ('diary','user_post','qa','answer')
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day='${partition_date}'
)spam_pv
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id
)t3
LATERAL VIEW explode(t3.channel) t4 AS channel
group by partition_date,device_os_type,active_type,t4.channel
)t6
ON t1.partition_date = t6.partition_date
AND t1.device_os_type = t6.device_os_type
AND t1.device_type = t6.active_type
AND t1.channel = t6.channel
LEFT JOIN
(
--纯用户-内容发布成功数pv(去积分墙渠道、去医生账号、马甲号、模特用户)--真实用户
SELECT a.create_date,device_os_type,device_type,channel
,SUM(CASE WHEN type = 'question' THEN num END) as question_num
,SUM(CASE WHEN type = 'answer' THEN num END) as answer_num
,SUM(CASE WHEN type in ('tractate','diary','topic') THEN num END) as topic_num
,count(DISTINCT CASE WHEN type = 'question' THEN dev.device_id END) as question_dev_num
,count(DISTINCT CASE WHEN type = 'answer' THEN dev.device_id END) as answer_dev_num
,count(DISTINCT CASE WHEN type in ('tractate','diary','topic') THEN dev.device_id END) as topic_dev_num
FROM
( --新增帖子（用户发的）
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'tractate' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_api_tractate_view
WHERE partition_date = '${partition_date}'
and is_online = 'true'
AND platform = '1' --更美用户发的(去除hera后台，爬虫爬取的)
and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'tractate'
UNION ALL
--新增帖子（打卡发帖，去除新增用户中kyc用户的打卡发帖）
SELECT a.user_id,create_date,'tractate' as type,count(DISTINCT a.id) as num
FROM
(
SELECT user_id,id,regexp_replace(substr(create_time,1,10),'-','') as create_date
FROM online.tl_hdfs_api_tractate_view
WHERE partition_date = '${partition_date}'
and is_online = 'true'
AND platform = '7' --打卡发帖
and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
)a
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= '${start_date}'
and t.partition_date < '${end_date}'
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
LEFT JOIN
(
SELECT
partition_date,device_id
FROM online.ml_device_day_active_status --中kyc灰度的设备
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2')
AND ((partition_date <= '20200511' AND substr(convup(setencryption(device_id,'sha-1'),16,10),-1) = '0')
or (partition_date >= '20200512' AND substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0))
)c
ON b.partition_date = c.partition_date
AND b.device_id = c.device_id
WHERE c.device_id is NULL
GROUP BY a.user_id,create_date,'tractate'
UNION ALL
--新增日记本
SELECT a.user_id,a.create_date,'diary' as type,count(DISTINCT a.id) as num
FROM
(
SELECT user_id,id,regexp_replace(substr(created_time,1,10),'-','') as create_date
FROM online.tl_hdfs_diary_view
WHERE partition_date = '${partition_date}'
--and is_online = 'true'
--AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(created_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(created_time,1,10),'-','') < '${end_date}'
)a
JOIN
( --取非空日记
SELECT diary_id
FROM online.tl_hdfs_problem_view
WHERE partition_date = '${partition_date}'
AND is_spam = 'false'
)b
ON a.id = b.diary_id
GROUP BY a.user_id,a.create_date,'diary'
UNION ALL
--新增日记帖
SELECT user_id,regexp_replace(substr(created_time,1,10),'-','') as create_date,'topic' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_problem_view
WHERE partition_date = '${partition_date}'
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND diary_id is not null
and regexp_replace(substr(created_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(created_time,1,10),'-','') < '${end_date}'
GROUP BY user_id,regexp_replace(substr(created_time,1,10),'-',''),'topic'
UNION ALL
--新增问题
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'question' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_question_view
WHERE partition_date = '${partition_date}'
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND platform = '99' --更美用户发的(去除hera后台，爬虫爬取的，kyc自动回复的)
and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'question'
UNION ALL
--新增回答
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'answer' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_answer_view
WHERE partition_date = '${partition_date}'
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND platform = '99' --更美用户发的(去除hera后台，爬虫爬取的，kyc自动回复的)
and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'answer'
)a
JOIN --限制用户是在app进行的发帖
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>='${start_date}'
AND partition_date< '${end_date}'
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '发帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = '${partition_date}'
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = '${partition_date}'
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = '${partition_date}'
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = '${partition_date}'
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON a.user_id=c.user_id
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= '${start_date}'
and t.partition_date < '${end_date}'
)d
ON a.user_id = d.user_id
AND a.create_date = d.partition_date
JOIN
( -- 1.去掉机刷、积分墙渠道来的设备
SELECT partition_date,device_id,device_os_type,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
LATERAL VIEW explode(dev.channel) t AS channel
)dev
on dev.device_id = d.device_id
AND dev.partition_date = d.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY a.create_date,device_os_type,device_type,channel
)t8
ON t1.partition_date = t8.create_date
AND t1.device_os_type = t8.device_os_type
AND t1.device_type = t8.device_type
AND t1.channel = t8.channel
LEFT JOIN
(
--真实评论数(日记/用户帖/问题/回答)
--有评论过日记帖的设备,排除疑似广告，排除医生账号，排除登陆过以上账号的设备和马甲号，限制用户当天有过评论行为
SELECT
a.reply_date,device_os_type,device_type,channel,SUM(reply_num) as reply_num,count(DISTINCT dev.device_id) as reply_dev_num
FROM
(
SELECT
user_id,reply_date,SUM(reply_num) as reply_num
FROM
(
--有评论过日记帖的设备,排除疑似广告
SELECT t1.user_id,reply_date,sum(reply_num) as reply_num
FROM
(
SELECT user_id,regexp_replace(substr(reply_date,1,10),'-','') as reply_date,problem_id,count(reply_date) as reply_num
FROM online.tl_hdfs_topicreply_view
WHERE partition_date = '${partition_date}'
AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(reply_date,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(reply_date,1,10),'-','') < '${end_date}'
group by user_id,regexp_replace(substr(reply_date,1,10),'-',''),problem_id
)t1
JOIN
(
SELECT id,diary_id
FROM online.tl_hdfs_problem_view
WHERE partition_date = '${partition_date}'
group by id,diary_id
)t2
on t2.id=t1.problem_id
group by t1.user_id,reply_date
UNION ALL
--有评论过回答的设备,排除疑似广告
SELECT
t2.user_id as user_id,t2.reply_date as reply_date,sum(reply_num) as reply_num
FROM
(
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as reply_date,count(create_time) as reply_num,answer_id
FROM online.tl_hdfs_answer_reply_view
WHERE partition_date ='${partition_date}'
and (is_fake is NULL or is_fake = 'false')
AND answer_id is not NULL
and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
group by user_id,answer_id,regexp_replace(substr(create_time,1,10),'-','')
)t2
GROUP BY t2.user_id,t2.reply_date
UNION ALL
--有评论过用户帖的设备
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as reply_date,count(create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date ='${partition_date}'
and regexp_replace(substr(create_time,1,10),'-','') >= '${start_date}'
and regexp_replace(substr(create_time,1,10),'-','') < '${end_date}'
GROUP by user_id,regexp_replace(substr(create_time,1,10),'-','')
)reply_user
GROUP BY user_id,reply_date
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>='${start_date}'
AND partition_date< '${end_date}'
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.reply_date = b.partition_date
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = '${partition_date}'
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = '${partition_date}'
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = '${partition_date}'
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = '${partition_date}'
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON a.user_id=c.user_id
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= '${start_date}'
and t.partition_date < '${end_date}'
)d
ON a.user_id = d.user_id
AND a.reply_date = d.partition_date
JOIN
( -- 1.去掉机刷、积分墙渠道来的设备
SELECT partition_date,device_id,device_os_type,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
LATERAL VIEW explode(dev.channel) t AS channel
)dev
on dev.device_id = d.device_id
AND dev.partition_date = d.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY a.reply_date,device_os_type,device_type,channel
)t9
ON t1.partition_date = t9.reply_date
AND t1.device_os_type = t9.device_os_type
AND t1.device_type = t9.device_type
AND t1.channel = t9.channel
LEFT JOIN
(-- 推送点击 pv、uv
SELECT a.partition_date as partition_date,cl_type,device_type,channel
,count(a.cl_id) as click_pv
,count(DISTINCT a.cl_id) as click_uv
FROM
(
SELECT cl_id,partition_date,cl_type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>='${start_date}'
AND partition_date < '${end_date}'
AND action = 'notification_open' --推送点击
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
GROUP BY a.partition_date,cl_type,device_type,channel
)t10
ON t1.partition_date = t10.partition_date
AND t1.device_os_type = t10.cl_type
AND t1.device_type = t10.device_type
AND t1.channel = t10.channel
LEFT JOIN
(-- 开启推送设备数
SELECT regexp_replace(substr(a.partition_date,1,10),'-','') as partition_date,device_os_type,device_type,channel
,count(DISTINCT a.cl_id) as open_push_uv
,count(distinct b.device_id) as push_receive_uv
FROM
(
SELECT concat_ws('-',substr(a.partition_date,1,4),substr(a.partition_date,5,2),substr(a.partition_date,7,2)) as partition_date
,a.cl_id --开启push设备
FROM
(
SELECT partition_date,cl_id,max(time_str) as max_time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>='${start_date}' AND partition_date<'${end_date}'
and action='is_open_push'
group by partition_date,cl_id
)a
left join
(
SELECT partition_date,cl_id,time_str,COALESCE(params['type'],params['tyep']) as type
FROM online.bl_hdfs_maidian_updates
where partition_date>='${start_date}' AND partition_date<'${end_date}'
and action='is_open_push'
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.time_str=a.max_time_str
where type=1
group by a.partition_date,a.cl_id
)a
left join
(
select concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)) as partition_date
,device_id
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>='20200521' and partition_day<'${end_date}'
group by partition_day,device_id
union all
SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,device_id
FROM
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>='20200521' AND partition_date<'${end_date}'
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
group by partition_date,registration_id,platform,mesid
)msg
JOIN
(
SELECT device_id,registration_id
from online.tl_hdfs_api_jpushuserdevicebinding_view--极光id和设备id绑定关系全量表
where partition_date=regexp_replace(date_sub(current_date(),1),'-','')
and (registration_id is not null or registration_id <>'')
group by device_id,registration_id
)reg
on reg.registration_id=msg.registration_id
group by partition_date,device_id
)b
on a.cl_id=b.device_id and date_add(a.partition_date,1)=b.partition_date
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)c
ON a.cl_id = c.device_id
AND a.partition_date = c.partition_date
GROUP BY a.partition_date,device_os_type,device_type,channel
)t11
ON t1.partition_date = t11.partition_date
AND t1.device_os_type = t11.device_os_type
AND t1.device_type = t11.device_type
AND t1.channel = t11.channel
LEFT JOIN
(-- 接收到推送设备数--只有iOS设备能拿到数据，并且iOS设备的系统版本在10及以上
SELECT a.partition_date
,device_os_type
,case when d.device_id is not null then '新增设备' else '老活跃设备' end as device_type
,channel
,count(distinct a.device_id) as push_receive_uv
,count(distinct case when push_type in ('70','71') then a.device_id end) as op_receive_uv --运营push push_type in ('70','71')
,count(distinct case when push_type in ('25') then a.device_id end) as sign_receive_uv --运营push push_type in ('70','71')
,count(distinct case when push_type in ('101','102','103','104','105','106','107','108','111','112','113') then a.device_id end) as personal_receive_uv --个性化push
,count(distinct case when push_type is not null and push_type not in ('25','70','71','101','102','103','104','105','106','107','108','111','112','113') then a.device_id end) as auto_receive_uv --自动push
FROM
(
select partition_day as partition_date
,device_id,msg_id,'ios' as device_os_type
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>='20200521' and partition_day<'${end_date}'
group by partition_day,device_id,msg_id
union all
SELECT partition_date,device_id,mesid as msg_id,'android' as device_os_type
FROM
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>='20200521' AND partition_date<'${end_date}'
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
group by partition_date,registration_id,platform,mesid
)msg
JOIN
(
SELECT device_id,registration_id
from online.tl_hdfs_api_jpushuserdevicebinding_view--极光id和设备id绑定关系全量表
where partition_date=regexp_replace(date_sub(current_date(),1),'-','')
and (registration_id is not null or registration_id <>'')
group by device_id,registration_id
)reg
on reg.registration_id=msg.registration_id
)a
JOIN
(
select device_id,t.channel
from
(
select m.device_id
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
from
(
select m.device_id,m.first_channel_source_type
from online.ml_device_history_detail m
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day ='${partition_date}')tmp
on m.first_channel_source_type=tmp.code
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id
)m
lateral view explode(channel) t as channel
)b
on a.device_id=b.device_id
JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='${start_date}' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)c
on a.msg_id=c.msg_id
left join
(
select
partition_date
,device_id
from online.ml_device_day_active_status
where partition_date>='${start_date}' AND partition_date<'${end_date}'
AND active_type in ('1','2','3')
)d
on a.partition_date=d.partition_date and a.device_id=d.device_id
group by a.partition_date
,device_os_type
,case when d.device_id is not null then '新增设备' else '老活跃设备' end
,channel
)t12
ON t1.partition_date = t12.partition_date
AND t1.device_os_type = t12.device_os_type
AND t1.device_type = t12.device_type
AND t1.channel = t12.channel
LEFT JOIN
(-- 点击推送设备数
SELECT a.partition_date as partition_date,device_os_type,device_type,channel
,count(DISTINCT a.cl_id) as receive_push_uv
,count(DISTINCT CASE WHEN op_push_open_pv is not null and op_push_open_pv>0 then a.cl_id end) as op_push_open_uv
,count(DISTINCT CASE WHEN auto_push_open_pv is not null and auto_push_open_pv>0 then a.cl_id end) as auto_push_open_uv
,count(DISTINCT CASE WHEN personal_push_open_pv is not null and personal_push_open_pv>0 then a.cl_id end) as personal_push_open_uv
,count(DISTINCT CASE WHEN sign_push_open_pv is not null and sign_push_open_pv>0 then a.cl_id end) as sign_push_open_uv
FROM
(
SELECT a.partition_date,a.cl_id,
count(distinct a.message_id) as push_open_pv,
count(distinct case when push_type in ('70','71') then a.message_id end) as op_push_open_pv, --运营push push_type in ('70','71')
count(distinct case when push_type is not null and push_type not in ('25','70','71','101','102','103','104','105','106','107','108','111','112','113') then a.message_id end) as auto_push_open_pv, --系统push即自动push
count(distinct case when push_type in ('101','102','103','104','105','106','107','108','111','112','113') then a.message_id end) as personal_push_open_pv, --个性化push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv --签到push
FROM
( --推送点击
SELECT partition_date,cl_id,time_str,params['message_id'] as message_id,time_stamp
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '${start_date}' AND partition_date < '${end_date}'
AND action='notification_open'
GROUP BY partition_date,cl_id,time_str,params['message_id'],time_stamp
)a
LEFT JOIN
(
SELECT msg_id,push_type,time_stamp
FROM online.tl_hdfs_push2_new_view --增量表
WHERE partition_date >= '${start_date}' AND partition_date < '${end_date}'
GROUP BY msg_id,push_type,time_stamp
)b
on b.msg_id=a.message_id
group by a.partition_date,a.cl_id
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
GROUP BY a.partition_date,device_os_type,device_type,channel
)t13
ON t1.partition_date = t13.partition_date
AND t1.device_os_type = t13.device_os_type
AND t1.device_type = t13.device_type
AND t1.channel = t13.channel
LEFT JOIN
(--新增注册设备数/注册用户数
SELECT t1.partition_date,device_os_type,device_type,channel
,count(DISTINCT t1.device_id) as login_dev_num
,count(DISTINCT t1.user_id) as login_user_num
,count(DISTINCT CASE WHEN t3.user_id is not NULL THEN t2.user_id END) as login_phone_user_num
FROM
(
SELECT partition_date,device_id,a.user_id as user_id
FROM online.ml_device_updates
LATERAL VIEW explode(user_list)a as user_id
WHERE partition_date >= '${start_date}'
AND partition_date < '${end_date}'
)t1
JOIN
(
SELECT user_id,regexp_replace(substr(date_joined,1,10),'-','') as date_joined
FROM online.ml_user_history_detail
WHERE partition_date = '${partition_date}'
AND regexp_replace(substr(date_joined,1,10),'-','') >= '${start_date}'
AND regexp_replace(substr(date_joined,1,10),'-','') < '${end_date}'
)t2
ON t2.user_id=t1.user_id and t2.date_joined=t1.partition_date
LEFT JOIN
(
SELECT user_id,auth_type,phone
FROM online.tl_hdfs_userextra_view
WHERE partition_date = '${partition_date}'
AND auth_type = '1'
)t3
ON t2.user_id = t3.user_id
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = '${partition_date}'
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = '${partition_date}'
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = '${partition_date}'
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = '${partition_date}'
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON t2.user_id=c.user_id
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >='${start_date}' and partition_day<'${end_date}')tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = '${per_partition_date}')a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = '${partition_date}'
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= '${start_date}'
AND partition_date < '${end_date}'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)t4
ON t1.device_id = t4.device_id
AND t1.partition_date = t4.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY t1.partition_date,device_os_type,device_type,channel
)t14
ON t1.partition_date = t14.partition_date
AND t1.device_os_type = t14.device_os_type
AND t1.device_type = t14.device_type
AND t1.channel = t14.channel
order by `日期`,`设备类型`,`活跃类型`,`渠道`
\ No newline at end of file
pm/daily_operation/job/step1_1.job 0 → 100644
View file @ 889dd8a8
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_operation/job/step1_10.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/daily_operation/job/step1_11.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/daily_operation/job/step1_12.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/daily_operation/job/step1_13.job 0 → 100644
View file @ 889dd8a8
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/daily_operation/job/step1_2.job 0 → 100644
View file @ 889dd8a8
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/daily_operation/job/step1_3.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/daily_operation/job/step1_4.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file
pm/daily_operation/job/step1_5.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_message_view
\ No newline at end of file
pm/daily_operation/job/step1_6.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file
pm/daily_operation/job/step1_7.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_doctor_relation_dimen_day
\ No newline at end of file
pm/daily_operation/job/step1_8.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/daily_operation/job/step1_9.job 0 → 100644
View file @ 889dd8a8
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/daily_operation/job/step2.job 0 → 100644
View file @ 889dd8a8
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
command=sh /home/bi/bi-report/lib/shell/hive clue_daily
\ No newline at end of file
pm/daily_operation/job/step3.job 0 → 100644
View file @ 889dd8a8
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/clue_daily/liudi@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/daily_operation/report/daily_operation.sql 0 → 100644
View file @ 889dd8a8
SELECT day_id `日期`
,device_os_type `系统`
,active_type `新老`
,channel `渠道`
,dau as `dau（去除积分墙）`
,login_dev_num as `新增注册设备数`
,login_user_num as `新增注册账号`
,login_phone_user_num as `新增注册手机号用户数`
,retention_1 as `次日留存率`
,retention_7 as `第7日留存率`
,retention_30 as `第30日留存率`
,per_open_times as `单设备打开频次`
,home_feeds_ctr as `信息流内容ctr`
,per_use_time as `单设备app停留时长(M)`
,neirong_page_stay as `单设备内容时长(M)`
,neirong_in_app_use_time as `单设备内容时长/app停留时长`
,service_privacy_uv as `到达隐私授权界面设备数`
,service_click_uv as `同意隐私授权设备设备数`
,service_privacy_uv_in_dau as `到达隐私授权界面设备数/DAU`
,service_click_uv_in_dau as `同意隐私授权设备数/到达隐私授权界面设备数`
,open_push_uv as `开启推送授权设备数`
,open_push_uv_in_dau as `开启推送授权设备数/DAU`
,push_receive_uv_in_open_push_uv as `次日接收推送设备数/当天开启推送授权设备数`
,click_uv as `点击推送的设备数`
,non_click_uv as `未点击推送的设备数`
,click_uv_in_dau as `点击推送设备数/DAU`
,non_click_uv_in_dau `未点击推送设备数/DAU`
,op_push_open_uv `点击运营推送设备数`
,auto_push_open_uv as `点击系统推送设备数`
,personal_push_open_uv `点击个性化推送设备数`
,sign_push_open_uv as `点击签到推送设备数`
,op_push_open_uv_in_receive as `点击运营推送设备/接收到运营推送设备`
,auto_push_open_uv_in_receive as `点击系统推送设备/接收到系统推送设备`
,personal_push_open_uv_in_receive as `点击个性化推送设备/接收到个性化推送设备`
,sign_push_open_uv_in_receive as `点击签到推送设备/接收到签到推送设备`
,question_num as `主动提问数`
,answer_num as `主动回答数`
,topic_num as `主动发帖数`
,reply_num as `真实评论数`
,question_num_in_dau as `主动提问设备数/DAU`
,answer_num_in_dau as `主动回答设备数/DAU`
,topic_num_in_dau as `主动发帖设备数/DAU`
,reply_num_in_dau as `主动评论设备数/DAU`
,neirongUV as `内容UV`
,neirongPV `内容PV`
,neirongUV_in_dau `内容UV/DAU`
,neirongPV_in_UV `内容PV/内容UV`
,result_more_UV as `搜索结果综合页UV`
,result_meigou_UV as `搜索结果美购页UV`
,result_con_UV as `搜索结果商业页UV`
,result_more_UV_in_dau as `搜索结果综合页UV/DAU`
,result_meigou_UV_in_dau as `搜索结果美购页UV/DAU`
,result_con_UV_in_dau as `搜索结果商业页UV/DAU`
,result_more_PV as `搜索结果综合页PV`
,result_meigou_PV as `搜索结果美购页PV`
,result_con_PV as `搜索结果商业页PV`
,result_more_PV_in_UV as `搜索结果综合页PV/搜索结果综合页UV`
,result_meigou_PV_in_UV as `搜索结果美购页PV/搜索结果美购页UV`
,result_con_PV_in_UV `搜索结果商业页PV/搜索结果商业页UV`
,result_more_PV_in_dau `搜索结果综合页PV/DAU`
,result_meigou_PV_in_dau `搜索结果美购页PV/DAU`
,result_con_PV_in_dau as `搜索结果商业页PV/DAU`
,neirong_meigou_PV as `来自内容页的美购页PV`
,neirong_con_PV as `来自内容页的商业页PV`
,neirong_meigou_PV_in_dau as `来自内容页的美购页PV/DAU`
,neirong_con_PV_in_dau as `来自内容页的商业页PV/DAU`
,neirong_click_pv as `内容页的推荐内容点击pv`
,wel_second_pv_in_neirongPV as `来自内容页的商业二跳/内容pv`
,neirong_second_pv_in_neirongPV as `来自内容页的内容二跳/内容pv`
FROM pm.tl_pm_operation_d
where partition_day ='20200824' and day_id<='20200726'
or partition_day=regexp_replace(date_sub(current_date,1),'-','') and
order by `日期`,`系统`,`新老`,`渠道`
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ 889dd8a8
......@@ -60,7 +60,8 @@ FROM
(
SELECT partition_date,
cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type,
case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end as card_content_type,
CASE when transaction_type in ('fmctr') then array('fmctr','合计')
WHEN transaction_type like '%ctr' THEN array('ctr预估','合计')
WHEN transaction_type like '%cvr' THEN array('cvr预估','合计')
......@@ -78,9 +79,10 @@ FROM
AND tab_name = '精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND card_content_type in ('qa','diary','user_post','answer')
AND card_content_type in ('qa','diary','user_post','answer','special_pool')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end,
cl_id,
CASE when transaction_type in ('fmctr') then array('fmctr','合计')
WHEN transaction_type like '%ctr' THEN array('ctr预估','合计')
......@@ -108,7 +110,8 @@ FROM
(
SELECT partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type,
case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end as card_content_type,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
......@@ -125,10 +128,11 @@ FROM
AND params['tab_name'] = '精选'
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
AND params['card_content_type'] in ('qa','diary','user_post','answer','special_pool')
GROUP BY partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end,
case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
......@@ -149,21 +153,35 @@ FROM
and t2.recommend_type=t3.recommend_type
LEFT JOIN
(--页面浏览时长
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,page_stay
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND referrer='home'
AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t4
on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id
......@@ -171,22 +189,36 @@ FROM
and t4.page_name=t3.card_content_type
LEFT JOIN
(--搜索框和点击行为
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,navbar_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t5
on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id
......@@ -194,23 +226,37 @@ FROM
and t5.page_name=t3.card_content_type
LEFT JOIN
(--点击高亮词
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,highlight_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t6
on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id
......@@ -218,13 +264,14 @@ FROM
and t6.page_name=t3.card_content_type
LEFT JOIN
(--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv
SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as self_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
......@@ -242,9 +289,18 @@ FROM
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
group by partition_date,cl_id,business_id,page_name
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t7
on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id
......@@ -252,29 +308,39 @@ FROM
and t7.page_name=t3.card_content_type
LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv
SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as recom_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold' and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail')
or action='on_click_button' and params['button_name'] = 'more_recommendations')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
group by partition_date,cl_id,business_id,page_name
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t8
on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id
......@@ -282,76 +348,114 @@ FROM
and t8.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv
SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as recom_content_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
group by partition_date,cl_id,business_id,page_name
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t9
on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(--转诊点击，视频面诊点击
(--视频面诊点击
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,video_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as video_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button'
and params['button_name']='video_interview'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t10
on t10.partition_date=t3.partition_date
and t10.cl_id=t3.cl_id
and t10.business_id=t3.card_id
and t10.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
(--转诊按钮点击
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,referral_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as referral_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button'
and params['button_name']='referral'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t11
on t11.partition_date=t3.partition_date
and t11.cl_id=t3.cl_id
......
pm/daily_recommend_strategy/report/home_recommend_strategy.sql
View file @ 889dd8a8
......@@ -19,5 +19,5 @@ SELECT
-- '未配置' as `来自I的推荐专题点击pv`,
transfer_card as `来自I的转诊点击pv`,
video_consultation as `来自I的视频面诊点击pv`FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200817' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
WHERE partition_day>='20200824' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment