Skip to content

B
bi-report
Commit d33f53e6 authored by yindanlei's avatar yindanlei
Browse files
Options

Merge branch 'master' of http://git.wanmeizhensuo.com/data/bi-report into yindanlei

parents b7acdb69 17aa35bf
Hide whitespace changes
Inline Side-by-side
Showing with 7278 additions and 683 deletions
.idea/misc.xml
View file @ d33f53e6
......@@ -16,5 +16,8 @@
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectPlainTextFileTypeManager">
<file url="file://$PROJECT_DIR$/pm/clue_daily/report/ai_qa.sql" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_13" default="false" project-jdk-name="Python 2.7 (shell-tools)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
pm/clue_daily/en-cn.properties 0 → 100644
View file @ d33f53e6
clue_daily=线索日报（用户侧）
\ No newline at end of file
pm/clue_daily/etl/clue_daily.sql 0 → 100644
View file @ d33f53e6
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--线索日报
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_userclue_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT T1.partition_date as day_id
,T1.device_os_type
,T1.active_type
,T1.grey_type
,T1.channel
,dau
,user_num
,phone_user_num
,nvl(total_clue_dev,0)
,nvl(total_clue_num,0)
,nvl(total_valid_dev,0)
,nvl(total_valid_num,0)
,msg_dev
,msg_num
,valid_msg_dev
,valid_msg_num
,clue_dev
,clue_num
,nvl(call_dev,0)
,nvl(call_num,0)
,nvl(valid_call_dev,0)
,nvl(valid_call_num,0)
,nvl(call_dev_d,0)
,nvl(call_num_d,0)
,nvl(valid_call_dev_d,0)
,nvl(valid_call_num_d,0)
,msg_dev_3
,msg_num_3
,valid_msg_dev_3
,valid_msg_num_3
,clue_dev_3
,clue_num_3
,nvl(call_dev_3,0)
,nvl(call_num_3,0)
,nvl(valid_call_dev_3,0)
,nvl(valid_call_num_3,0)
,nvl(call_dev_3_d,0)
,nvl(call_num_3_d,0)
,nvl(valid_call_dev_3_d,0)
,nvl(valid_call_num_3_d,0)
,msg_dev_kyc
,msg_num_kyc
,valid_msg_dev_kyc
,valid_msg_num_kyc
,clue_dev_kyc
,clue_num_kyc
,nvl(call_dev_kyc,0)
,nvl(call_num_kyc,0)
,nvl(valid_call_dev_kyc,0)
,nvl(valid_call_num_kyc,0)
,nvl(call_dev_kyc_d,0)
,nvl(call_num_kyc_d,0)
,nvl(valid_call_dev_kyc_d,0)
,nvl(valid_call_num_kyc_d,0)
,msg_dev_qa
,msg_num_qa
,valid_msg_dev_qa
,valid_msg_num_qa
,clue_dev_qa
,clue_num_qa
,nvl(call_dev_qa,0)
,nvl(call_num_qa,0)
,nvl(valid_call_dev_qa,0)
,nvl(valid_call_num_qa,0)
,nvl(call_dev_qa_d,0)
,nvl(call_num_qa_d,0)
,nvl(valid_call_dev_qa_d,0)
,nvl(valid_call_num_qa_d,0)
FROM
(
SELECT mas.partition_date as partition_date
,device_os_type
,active_type
,grey_type
,channel
,count(distinct mas.device_id) as dau
,count(distinct a.user_id) as user_num
,count(distinct b.user_id) as phone_user_num
,nvl(count(distinct case when t13.total_clue_num>0 then t13.user_id end),0) total_clue_dev
,nvl(sum(t13.total_clue_num),0) total_clue_num
,nvl(count(distinct case when t13_v.total_valid_num>0 then t13_v.user_id end),0) total_valid_dev
,nvl(sum(t13_v.total_valid_num),0) total_valid_num
,nvl(count(distinct case when t1.msg_num>0 then t1.user_id end),0) msg_dev
,nvl(sum(t1.msg_num),0) msg_num
,nvl(count(distinct case when t1_v.valid_msg_num>0 then t1_v.user_id end),0) valid_msg_dev
,nvl(sum(t1_v.valid_msg_num),0) as valid_msg_num
,nvl(count(distinct case when t2.clue_num>0 then t2.user_id end),0) as clue_dev
,nvl(sum(t2.clue_num),0) as clue_num
,nvl(count(distinct case when t10.msg_num>0 then t10.user_id end),0) msg_dev_3
,nvl(sum(t10.msg_num),0) msg_num_3
,nvl(count(distinct case when t10_v.valid_msg_num>0 then t10_v.user_id end),0) valid_msg_dev_3
,nvl(sum(t10_v.valid_msg_num),0) as valid_msg_num_3
,nvl(count(distinct case when t11.clue_num>0 then t11.user_id end),0) as clue_dev_3
,nvl(sum(t11.clue_num),0) as clue_num_3
,nvl(count(distinct case when t4.msg_num>0 then t4.user_id end),0) msg_dev_kyc
,nvl(sum(t4.msg_num),0) msg_num_kyc
,nvl(count(distinct case when t4_v.valid_msg_num>0 then t4_v.user_id end),0) valid_msg_dev_kyc
,nvl(sum(t4_v.valid_msg_num),0) as valid_msg_num_kyc
,nvl(count(distinct case when t5.clue_num>0 then t5.user_id end),0) as clue_dev_kyc
,nvl(sum(t5.clue_num),0) as clue_num_kyc
,nvl(count(distinct case when t7.msg_num>0 then t7.user_id end),0) msg_dev_qa
,nvl(sum(t7.msg_num),0) msg_num_qa
,nvl(count(distinct case when t7_v.valid_msg_num>0 then t7_v.user_id end),0) valid_msg_dev_qa
,nvl(sum(t7_v.valid_msg_num),0) as valid_msg_num_qa
,nvl(count(distinct case when t8.clue_num>0 then t8.user_id end),0) as clue_dev_qa
,nvl(sum(t8.clue_num),0) as clue_num_qa
,nvl(count(distinct case when t3.call_num>0 then t3.user_id end),0) as call_dev
,nvl(sum(t3.call_num),0) as call_num
,nvl(count(distinct case when t3_v.valid_call_num>0 then t3_v.user_id end),0) as valid_call_dev
,nvl(sum(t3_v.valid_call_num),0) as valid_call_num
,nvl(count(distinct case when t12.call_num>0 then t12.user_id end),0) as call_dev_3
,nvl(sum(t12.call_num),0) as call_num_3
,nvl(count(distinct case when t12_v.valid_call_num>0 then t12_v.user_id end),0) as valid_call_dev_3
,nvl(sum(t12_v.valid_call_num),0) as valid_call_num_3
,nvl(count(distinct case when t6.call_num>0 then t6.user_id end),0) as call_dev_kyc
,nvl(sum(t6.call_num),0) as call_num_kyc
,nvl(count(distinct case when t6_v.valid_call_num>0 then t6_v.user_id end),0) as valid_call_dev_kyc
,nvl(sum(t6_v.valid_call_num),0) as valid_call_num_kyc
,nvl(count(distinct case when t9.call_num>0 then t9.user_id end),0) as call_dev_qa
,nvl(sum(t9.call_num),0) as call_num_qa
,nvl(count(distinct case when t9_v.valid_call_num>0 then t9_v.user_id end),0) as valid_call_dev_qa
,nvl(sum(t9_v.valid_call_num),0) as valid_call_num_qa
FROM
(
SELECT partition_date,device_id,device_os_type,b.active_type,b.grey_type,b.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,array(case when active_type in ('1','2') then '新增设备'
when active_type ='4' then '老活跃设备' end,'合计') as active_type
,array(case when substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and m.device_id IS NOT NULL
)a
LATERAL VIEW explode(a.grey_type) b AS grey_type
LATERAL VIEW explode(a.active_type) b AS active_type
LATERAL VIEW explode(a.channel) b AS channel
)mas
left join
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<= regexp_replace(date_sub(current_date,1),'-','')
)a
on mas.partition_date=a.partition_date and mas.device_id=a.device_id
left join
(
SELECT distinct user_id
FROM online.tl_hdfs_userextra_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND auth_type = '1'
)b
on a.user_id=b.user_id
left join
(--私信线索总数
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as msg_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
GROUP BY msg.partition_date,user_id
)t1
ON a.partition_date = t1.partition_date and a.user_id=t1.user_id
left join
(--有效私信线索总数
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as valid_msg_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
GROUP BY msg.partition_date,user_id
)t1_v
ON a.partition_date = t1_v.partition_date and a.user_id=t1_v.user_id
LEFT JOIN
( --电话线索总数
SELECT partition_date,user_id,count(1) AS clue_num
FROM
(
SELECT user_id,merchant_id,MIN(REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '')) as partition_date,regexp_replace(SUBSTR(created_time,1,7),'-','') AS CALL_MONTH
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,7),'-','')
)a
GROUP BY partition_date,user_id
)t2
ON a.partition_date = t2.partition_date and a.user_id=t2.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t3
ON a.partition_date = t3.partition_date and a.user_id=t3.user_id
LEFT JOIN
( ---有效电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t3_v
ON a.partition_date = t3_v.partition_date and a.user_id=t3_v.user_id
left join
(--kyc私信
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as msg_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM
(
SELECT distinct conversation_id
FROM online.tl_hdfs_message_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(to_date(send_time) ,'-','')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and regexp_replace(to_date(send_time) ,'-','') <= regexp_replace(date_sub(current_date,1),'-','')
and content like '%对%感兴趣'
)a
join
(
SELECT user_id,merchant_id,conversation_id,partition_day
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,merchant_id,conversation_id,partition_day
)b
on a.conversation_id=b.conversation_id
group by user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
group by msg.partition_date,user_id
)t4
on a.partition_date=t4.partition_date and a.user_id=t4.user_id
left join
(--kyc有效私信
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as valid_msg_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM
(
SELECT distinct conversation_id
FROM online.tl_hdfs_message_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(to_date(send_time) ,'-','')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and regexp_replace(to_date(send_time) ,'-','') <= regexp_replace(date_sub(current_date,1),'-','')
and content like '%对%感兴趣'
)a
join
(
SELECT user_id,merchant_id,conversation_id,partition_day
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,conversation_id,partition_day
)b
on a.conversation_id=b.conversation_id
group by user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
group by msg.partition_date,user_id
)t4_v
on a.partition_date=t4_v.partition_date and a.user_id=t4_v.user_id
LEFT JOIN
( --kyc电话线索总数
SELECT partition_date,user_id,count(1) as clue_num
FROM
(
SELECT a.user_id,a.merchant_id,substr(a.partition_date,1,6) as month,min(a.partition_date) as partition_date
FROM
(
SELECT user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as partition_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,10),'-','')
)a
join
(
SELECT partition_date,user_id,merchant_id
from
(
SELECT partition_date,cl_id,user_id,hospital_id
from
(
SELECT partition_date,cl_id,user_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'--其实传的是doctor_id
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id,user_id,hospital_id
)a
join
(
SELECT doctor_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY doctor_id,merchant_id
)b
on a.hospital_id=b.doctor_id
)kyc
on a.user_id=kyc.user_id and a.merchant_id=kyc.merchant_id and a.partition_date=kyc.partition_date
GROUP BY a.user_id,a.merchant_id,substr(a.partition_date,1,6)
)a
group by partition_date,user_id
)t5
ON a.partition_date = t5.partition_date and a.user_id=t5.user_id
LEFT JOIN
( --拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as partition_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,user_id,merchant_id
from
(
SELECT partition_date,cl_id,user_id,hospital_id
from
(
SELECT partition_date,cl_id,user_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'--其实传的是doctor_id
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id,user_id,hospital_id
)a
join
(
SELECT doctor_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY doctor_id,merchant_id
)b
on a.hospital_id=b.doctor_id
)kyc
on e.user_id=kyc.user_id and e.merchant_id=kyc.merchant_id and e.partition_date=kyc.partition_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t6
ON a.partition_date = t6.partition_date and a.user_id=t6.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,user_id,merchant_id
from
(
SELECT partition_date,cl_id,user_id,hospital_id
from
(
SELECT partition_date,cl_id,user_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'--其实传的是doctor_id
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id,user_id,hospital_id
)a
join
(
SELECT doctor_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY doctor_id,merchant_id
)b
on a.hospital_id=b.doctor_id
)kyc
on e.user_id=kyc.user_id and e.merchant_id=kyc.merchant_id and e.created_date=kyc.partition_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t6_v
ON a.partition_date = t6_v.partition_date and a.user_id=t6_v.user_id
left join
(--需求自测私信
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as msg_num
FROM
(
SELECT a.user_id,a.merchant_id,substr(partition_day,1,6) as month,min(partition_day) as partition_date
FROM
(
SELECT partition_date,user_id,merchant_id
from
(--点击发送
SELECT a.partition_date,user_id,cl_id
,json_map(b.hospital_id,'string,string')['hospital_id'] hospital_id
FROM
(
SELECT partition_date,user_id,cl_id,json_split(params['hospital_id_list']) as hospital_id_list
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name='ai_plan_consult_list'
and params['button_name'] in ('send')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
group by partition_date,user_id,cl_id,params['hospital_id_list']
)a
LATERAL VIEW EXPLODE(a.hospital_id_list) b as hospital_id
)b
join
(
SELECT hospital_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY hospital_id,merchant_id
)c
on c.hospital_id=b.hospital_id
group by partition_date,user_id,merchant_id
)a
join
(
SELECT user_id,merchant_id,partition_day
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,merchant_id,partition_day
)b
on a.user_id=b.user_id and a.merchant_id=b.merchant_id and a.partition_date=b.partition_day
group by a.user_id,a.merchant_id,substr(partition_day,1,6)
)msg
group by msg.partition_date,user_id
)t7
on a.partition_date=t7.partition_date and a.user_id=t7.user_id
left join
(--需求自测有效私信
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as valid_msg_num
FROM
(
SELECT a.user_id,a.merchant_id,substr(partition_day,1,6) as month,min(partition_day) as partition_date
FROM
(
SELECT partition_date,user_id,merchant_id
from
(--点击发送
SELECT a.partition_date,user_id,cl_id
,json_map(b.hospital_id,'string,string')['hospital_id'] hospital_id
FROM
(
SELECT partition_date,user_id,cl_id,json_split(params['hospital_id_list']) as hospital_id_list
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name='ai_plan_consult_list'
and params['button_name'] in ('send')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
group by partition_date,user_id,cl_id,params['hospital_id_list']
)a
LATERAL VIEW EXPLODE(a.hospital_id_list) b as hospital_id
)b
join
(
SELECT hospital_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY hospital_id,merchant_id
)c
on c.hospital_id=b.hospital_id
group by partition_date,user_id,merchant_id
)a
join
(
SELECT user_id,merchant_id,partition_day
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,partition_day
)b
on a.user_id=b.user_id and a.merchant_id=b.merchant_id and a.partition_date=b.partition_day
group by a.user_id,a.merchant_id,substr(partition_day,1,6)
)msg
group by msg.partition_date,user_id
)t7_v
on a.partition_date=t7_v.partition_date and a.user_id=t7_v.user_id
LEFT JOIN
( --需求自测电话线索总数
SELECT partition_date,user_id,count(1) as clue_num
FROM
(
SELECT a.user_id,a.merchant_id,substr(a.partition_date,1,6) as month,min(a.partition_date) as partition_date
FROM
(
SELECT user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as partition_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,10),'-','')
)a
join
(
SELECT partition_date,cl_id,user_id
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and params['popup_name']='bargain_phone'
and page_name='ai_plan_list'
and params['button_name'] in ('confirm')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
)ai
on a.user_id=ai.user_id and a.partition_date=ai.partition_date
GROUP BY a.user_id,a.merchant_id,substr(a.partition_date,1,6)
)a
group by partition_date,user_id
)t8
ON a.partition_date = t8.partition_date and a.user_id=t8.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,cl_id,user_id
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and params['popup_name']='bargain_phone'
and page_name='ai_plan_list'
and params['button_name'] in ('confirm')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
)ai
on e.user_id=ai.user_id and e.created_date=ai.partition_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t9
ON a.partition_date = t9.partition_date and a.user_id=t9.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,cl_id,user_id
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and params['popup_name']='bargain_phone'
and page_name='ai_plan_list'
and params['button_name'] in ('confirm')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
)ai
on e.user_id=ai.user_id and e.created_date=ai.partition_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t9_v
ON a.partition_date = t9_v.partition_date and a.user_id=t9_v.user_id
left join
(--t3以上商户私信线索总数
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as msg_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
group by user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on msg.merchant_id=b.merchant_id
GROUP BY msg.partition_date,user_id
)t10
ON a.partition_date = t10.partition_date and a.user_id=t10.user_id
left join
(--t3以上商户有效私信线索总数
SELECT msg.partition_date,user_id
,count(msg.merchant_id) as valid_msg_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on msg.merchant_id=b.merchant_id
GROUP BY msg.partition_date,user_id
)t10_v
ON a.partition_date = t10_v.partition_date and a.user_id=t10_v.user_id
left join
(--t3以上商户电话线索总数
SELECT partition_date,user_id,count(1) AS clue_num
FROM
(
SELECT user_id,merchant_id,MIN(REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '')) as partition_date,regexp_replace(SUBSTR(created_time,1,7),'-','') AS CALL_MONTH
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,7),'-','')
)a
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on a.merchant_id=b.merchant_id
GROUP BY partition_date,user_id
)t11
ON a.partition_date = t11.partition_date and a.user_id=t11.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on t1.merchant_id=b.merchant_id
GROUP BY user_id,partition_date
)t12
ON a.partition_date = t12.partition_date and a.user_id=t12.user_id
LEFT JOIN
( ---拨打有效电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on t1.merchant_id=b.merchant_id
GROUP BY user_id,partition_date
)t12_v
ON a.partition_date = t12_v.partition_date and a.user_id=t12_v.user_id
left join
(
SELECT partition_date,user_id,sum(clue_num) as total_clue_num
FROM
(
SELECT msg.partition_date,user_id,count(1) as clue_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
GROUP BY msg.partition_date,user_id
union all
--电话线索总数
SELECT partition_date,user_id,count(1) AS clue_num
FROM
(
SELECT user_id,merchant_id,MIN(REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '')) as partition_date,regexp_replace(SUBSTR(created_time,1,7),'-','') AS CALL_MONTH
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(created_time,1,7),'-','')
)a
GROUP BY partition_date,user_id
)a
group by partition_date,user_id
)t13
ON a.partition_date = t13.partition_date and a.user_id=t13.user_id
left join
(
SELECT partition_date,user_id,sum(valid_num) as total_valid_num
from
( ---拨打有效电话线索
SELECT partition_date,user_id,count(1) as valid_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on t1.merchant_id=b.merchant_id
GROUP BY user_id,partition_date
union all
SELECT msg.partition_date,user_id,count(msg.merchant_id) as valid_num
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS MONTH,min(partition_day) as partition_date
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)msg
GROUP BY msg.partition_date,user_id
)a
group by partition_date,user_id
)t13_v
ON a.partition_date = t13_v.partition_date and a.user_id=t13_v.user_id
group by mas.partition_date,device_os_type,active_type,grey_type,channel
)T1
LEFT JOIN
(
SELECT t3_d.partition_date,a.device_os_type,a.grey_type
,a.active_type,a.channel
,nvl(count(distinct case when t3_d.call_num>0 then t3_d.user_id end),0) as call_dev_d
,nvl(sum(t3_d.call_num),0) as call_num_d
,nvl(count(distinct case when t3_v_d.valid_call_num>0 then t3_v_d.user_id end),0) as valid_call_dev_d
,nvl(sum(t3_v_d.valid_call_num),0) as valid_call_num_d
,nvl(count(distinct case when t12_d.call_num>0 then t12_d.user_id end),0) as call_dev_3_d
,nvl(sum(t12_d.call_num),0) as call_num_3_d
,nvl(count(distinct case when t12_v_d.valid_call_num>0 then t12_v_d.user_id end),0) as valid_call_dev_3_d
,nvl(sum(t12_v_d.valid_call_num),0) as valid_call_num_3_d
,nvl(count(distinct case when t6_d.call_num>0 then t6_d.user_id end),0) as call_dev_kyc_d
,nvl(sum(t6_d.call_num),0) as call_num_kyc_d
,nvl(count(distinct case when t6_v_d.valid_call_num>0 then t6_v_d.user_id end),0) as valid_call_dev_kyc_d
,nvl(sum(t6_v_d.valid_call_num),0) as valid_call_num_kyc_d
,nvl(count(distinct case when t9_d.call_num>0 then t9_d.user_id end),0) as call_dev_qa_d
,nvl(sum(t9_d.call_num),0) as call_num_qa_d
,nvl(count(distinct case when t9_v_d.valid_call_num>0 then t9_v_d.user_id end),0) as valid_call_dev_qa_d
,nvl(sum(t9_v_d.valid_call_num),0) as valid_call_num_qa_d
FROM
(
SELECT device_id,device_os_type,user_id,v.grey_type,v.active_type,v.channel
FROM
(
SELECT c.device_id,device_os_type,user_id
,array(case when substr(convup(setencryption(c.device_id,'sha-1'),16,10),-2,2)%20 = 0 then 'kyc灰度' else '非灰' end,'合计') as grey_type
,array('老活跃设备','合计') as active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM
(
SELECT device_id,device_os_type,a.user_id,a.partition_date,first_channel_source_type
,row_number() over(partition by device_id order by a.partition_date desc) as rn
FROM
(
SELECT device_id,v.device_os_type,v.user_id,partition_date
,if(size(channel_source_type_list) > 0, channel_source_type_list [ 0 ], '') AS first_channel_source_type
FROM online.ml_device_updates
LATERAL VIEW EXPLODE(user_list) v as user_id
LATERAL VIEW EXPLODE(device_os_type) v as device_os_type
where partition_date>='20200801' and partition_date<=regexp_replace(date_sub(current_date(),1),'-','')
)a--由于要取近期该设备用过的user_id，故时间取的较长，避免用公式而将月末数据少算
join
(
SELECT partition_date,user_id
FROM online.ml_user_updates
where partition_date>='20200801' and partition_date<=regexp_replace(date_sub(current_date(),1),'-','')
and phone is not null and phone <>''
group by partition_date,user_id
)b
on a.partition_date=b.partition_date and a.user_id=b.user_id
)c
LEFT JOIN
(
SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
)tmp
on c.first_channel_source_type=tmp.code and c.partition_date=tmp.partition_day
LEFT JOIN
(
SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-','')
)a
ON c.device_id = a.device_id
LEFT JOIN
(
SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id
)b
ON c.device_id = b.device_id AND c.partition_date = b.day_id
where rn=1
)d
LATERAL VIEW EXPLODE(d.grey_type)v as grey_type
LATERAL VIEW EXPLODE(d.active_type)v as active_type
LATERAL VIEW explode(d.channel) v AS channel
)a
JOIN
( ---非当天拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
where a.partition_date<>e.created_date
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t3_d
ON a.user_id=t3_d.user_id
LEFT JOIN
( ---非当天拨打有效电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
where a.partition_date<>e.created_date
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t3_v_d
ON t3_d.partition_date = t3_v_d.partition_date and t3_d.user_id=t3_v_d.user_id
LEFT JOIN
( --拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,user_id,merchant_id
from
(
SELECT partition_date,cl_id,user_id,hospital_id
from
(
SELECT partition_date,cl_id,user_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'--其实传的是doctor_id
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id,user_id,hospital_id
)a
join
(
SELECT doctor_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY doctor_id,merchant_id
)b
on a.hospital_id=b.doctor_id
)kyc
on e.user_id=kyc.user_id and e.merchant_id=kyc.merchant_id and e.created_date=kyc.partition_date
where a.partition_date<>e.created_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t6_d
ON t3_d.partition_date = t6_d.partition_date and t3_d.user_id=t6_d.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,user_id,merchant_id
from
(
SELECT partition_date,cl_id,user_id,hospital_id
from
(
SELECT partition_date,cl_id,user_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'--其实传的是doctor_id
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id,user_id,hospital_id
)a
join
(
SELECT doctor_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY doctor_id,merchant_id
)b
on a.hospital_id=b.doctor_id
)kyc
on e.user_id=kyc.user_id and e.merchant_id=kyc.merchant_id and e.created_date=kyc.partition_date
where a.partition_date<>e.created_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t6_v_d
ON t3_d.partition_date = t6_v_d.partition_date and t3_d.user_id=t6_v_d.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,cl_id,user_id
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and params['popup_name']='bargain_phone'
and page_name='ai_plan_list'
and params['button_name'] in ('confirm')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
)ai
on e.user_id=ai.user_id and e.created_date=ai.partition_date
where a.partition_date<>e.created_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t9_d
ON t3_d.partition_date = t9_d.partition_date and t3_d.user_id=t9_d.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(a.partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day<=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
group by id,user_id,merchant_id,created_time
)e
ON d.lead_task_id = e.id
join
(
SELECT partition_date,cl_id,user_id
FROM online.bl_hdfs_maidian_updates
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and params['popup_name']='bargain_phone'
and page_name='ai_plan_list'
and params['button_name'] in ('confirm')
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
)ai
on e.user_id=ai.user_id and e.created_date=ai.partition_date
where a.partition_date<>e.created_date
GROUP BY e.user_id,e.merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
GROUP BY user_id,partition_date
)t9_v_d
ON t3_d.partition_date = t9_v_d.partition_date and t3_d.user_id=t9_v_d.user_id
LEFT JOIN
( ---拨打电话线索
SELECT partition_date,user_id
,count(1) as call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
where a.partition_date<>e.created_date
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on t1.merchant_id=b.merchant_id
GROUP BY user_id,partition_date
)t12_d
ON t3_d.partition_date = t12_d.partition_date and t3_d.user_id=t12_d.user_id
LEFT JOIN
( ---拨打有效电话线索
SELECT partition_date,user_id
,count(1) as valid_call_num
FROM
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as partition_date,
1 as call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
and start_time>call_time
and start_time<release_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id,REGEXP_REPLACE(SUBSTR(created_time,1,10), '-', '') as created_date
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
where a.partition_date<>e.created_date
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)t1
join
(
SELECT distinct merchant_id
FROM
(
SELECT merchant_id,merchant_level
,row_number() over(partition by merchant_id order by data_month desc) as rn
FROM online.al_business_merchant_grade_day_repo
where partition_date=regexp_replace(date_sub(current_date,1),'-','')
)a
where rn=1 and merchant_level>=3
)b
on t1.merchant_id=b.merchant_id
GROUP BY user_id,partition_date
)t12_v_d
ON t3_d.partition_date = t12_v_d.partition_date and t3_d.user_id=t12_v_d.user_id
group by t3_d.partition_date,a.device_os_type,a.grey_type,a.active_type,a.channel
)T2
on T1.partition_date=T2.partition_date
AND T1.device_os_type=T2.device_os_type
AND T1.active_type=T2.active_type
AND T1.grey_type=T2.grey_type
AND T1.channel=T2.channel
pm/clue_daily/etl/create_clue_daily.sql 0 → 100644
View file @ d33f53e6
--***************************************************************
--*脚本名称:
--*功能: 线索日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_userclue_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau bigint comment '{"chs_name":"dau(不去除积分墙)","description":"","etl":"","value":"","remark":""}',
user_num bigint comment '{"chs_name":"注册用户","description":"","etl":"","value":"","remark":""}',
phone_user_num bigint comment '{"chs_name":"手机号注册用户","description":"","etl":"","value":"","remark":""}',
total_clue_dev bigint comment '{"chs_name":"总咨询设备","description":"","etl":"","value":"","remark":""}',
total_clue_num bigint comment '{"chs_name":"总线索","description":"","etl":"","value":"","remark":""}',
total_valid_dev bigint comment '{"chs_name":"总有效咨询设备","description":"","etl":"","value":"","remark":""}',
total_valid_num bigint comment '{"chs_name":"总有效线索","description":"","etl":"","value":"","remark":""}',
msg_dev bigint comment '{"chs_name":"私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
valid_msg_dev bigint comment '{"chs_name":"有效私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
valid_msg_num bigint comment '{"chs_name":"有效私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
clue_dev bigint comment '{"chs_name":"授权电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
clue_num bigint comment '{"chs_name":"授权电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
call_dev bigint comment '{"chs_name":"当天拨打电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
call_num bigint comment '{"chs_name":"当天拨打电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
valid_call_dev bigint comment '{"chs_name":"当天有效电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
valid_call_num bigint comment '{"chs_name":"当天有效电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
call_dev_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
call_num_d bigint comment '{"chs_name":"非当天拨打电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
valid_call_dev_d bigint comment '{"chs_name":"非当天有效电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
valid_call_num_d bigint comment '{"chs_name":"非当天有效电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
msg_dev_3 bigint comment '{"chs_name":"私信咨询设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
msg_num_3 bigint comment '{"chs_name":"私信咨询人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
valid_msg_dev_3 bigint comment '{"chs_name":"有效私信咨询设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
valid_msg_num_3 bigint comment '{"chs_name":"有效私信咨询人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
clue_dev_3 bigint comment '{"chs_name":"授权电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
clue_num_3 bigint comment '{"chs_name":"授权电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
call_dev_3 bigint comment '{"chs_name":"当天拨打电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
call_num_3 bigint comment '{"chs_name":"当天拨打电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
valid_call_dev_3 bigint comment '{"chs_name":"当天有效电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
valid_call_num_3 bigint comment '{"chs_name":"当天有效电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
call_dev_3_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
call_num_3_d bigint comment '{"chs_name":"非当天拨打电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
valid_call_dev_3_d bigint comment '{"chs_name":"非当天有效电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
valid_call_num_3_d bigint comment '{"chs_name":"非当天有效电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
msg_dev_kyc bigint comment '{"chs_name":"私信咨询设备数-kyc","description":"","etl":"","value":"","remark":""}',
msg_num_kyc bigint comment '{"chs_name":"私信咨询人次-kyc","description":"","etl":"","value":"","remark":""}',
valid_msg_dev_kyc bigint comment '{"chs_name":"有效私信咨询设备数-kyc","description":"","etl":"","value":"","remark":""}',
valid_msg_num_kyc bigint comment '{"chs_name":"有效私信咨询人次-kyc","description":"","etl":"","value":"","remark":""}',
clue_dev_kyc bigint comment '{"chs_name":"授权电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
clue_num_kyc bigint comment '{"chs_name":"授权电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
call_dev_kyc bigint comment '{"chs_name":"当天拨打电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
call_num_kyc bigint comment '{"chs_name":"当天拨打电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
valid_call_dev_kyc bigint comment '{"chs_name":"当天有效电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
valid_call_num_kyc bigint comment '{"chs_name":"当天有效电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
call_dev_kyc_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
call_num_kyc_d bigint comment '{"chs_name":"非当天拨打电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
valid_call_dev_kyc_d bigint comment '{"chs_name":"非当天有效电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
valid_call_num_kyc_d bigint comment '{"chs_name":"非当天有效电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
msg_dev_qa bigint comment '{"chs_name":"私信咨询设备数-需求自测","description":"","etl":"","value":"","remark":""}',
msg_num_qa bigint comment '{"chs_name":"私信咨询人次-需求自测","description":"","etl":"","value":"","remark":""}',
valid_msg_dev_qa bigint comment '{"chs_name":"有效私信咨询设备数-需求自测","description":"","etl":"","value":"","remark":""}',
valid_msg_num_qa bigint comment '{"chs_name":"有效私信咨询人次-需求自测","description":"","etl":"","value":"","remark":""}',
clue_dev_qa bigint comment '{"chs_name":"授权电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
clue_num_qa bigint comment '{"chs_name":"授权电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
call_dev_qa bigint comment '{"chs_name":"当天拨打电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
call_num_qa bigint comment '{"chs_name":"当天拨打电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
valid_call_dev_qa bigint comment '{"chs_name":"当天有效电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
valid_call_num_qa bigint comment '{"chs_name":"当天有效电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
call_dev_qa_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
call_num_qa_d bigint comment '{"chs_name":"非当天拨打电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
valid_call_dev_qa_d bigint comment '{"chs_name":"非当天有效电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
valid_call_num_qa_d bigint comment '{"chs_name":"非当天有效电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}'
)comment '用户侧线索日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/clue_daily/job/step1_1.job 0 → 100644
View file @ d33f53e6
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/clue_daily/job/step1_10.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/clue_daily/job/step1_11.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/clue_daily/job/step1_12.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/clue_daily/job/step1_13.job 0 → 100644
View file @ d33f53e6
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/clue_daily/job/step1_14.job 0 → 100644
View file @ d33f53e6
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file
pm/clue_daily/job/step1_2.job 0 → 100644
View file @ d33f53e6
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/clue_daily/job/step1_3.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/clue_daily/job/step1_4.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file
pm/clue_daily/job/step1_5.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_message_view
\ No newline at end of file
pm/clue_daily/job/step1_6.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file
pm/clue_daily/job/step1_7.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_doctor_relation_dimen_day
\ No newline at end of file
pm/clue_daily/job/step1_8.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/clue_daily/job/step1_9.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/clue_daily/job/step2.job 0 → 100644
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
command=sh /home/bi/bi-report/lib/shell/hive clue_daily
\ No newline at end of file
pm/clue_daily/job/step3.job 0 → 100644
View file @ d33f53e6
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/clue_daily/liudi@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/clue_daily/report/clue_daily.sql 0 → 100644
View file @ d33f53e6
SELECT day_id `日期`
,device_os_type `系统`
,active_type `新老`
,grey_type `是否灰度`
,channel `渠道`
,dau as `dau（不去除积分墙）`
,user_num as `注册的用户数`
,phone_user_num as `手机号注册用户数`
,total_clue_dev as `总咨询设备（去重）`
,total_clue_num as `总线索`
,total_valid_dev+valid_call_dev_d as `总有效咨询设备（去重）`
,total_valid_num+valid_call_num_d as `总有效线索`
,nvl(concat(round((total_valid_dev+valid_call_dev_d)/dau*100,2),'%'),0) as `总有效线索设备(去重)/dau`
,nvl(concat(round((total_valid_num+valid_call_num_d)/dau*100,2),'%'),0) as `总有效线索/dau`
,nvl(concat(round((total_valid_dev+valid_call_dev_d)/total_clue_dev*100,2),'%'),0) as `总有效线索设备(去重)/总咨询设备(去重)`
,nvl(concat(round((total_valid_num+valid_call_num_d)/total_clue_num*100,2),'%'),0) as `总有效线索/总线索`
,msg_dev as `私信咨询设备数-总计`
,msg_num as `私信咨询人次-总计`
,valid_msg_dev as `有效私信咨询设备数-总计`
,valid_msg_num as `有效私信咨询人次-总计`
,nvl(concat(round(valid_msg_num/msg_num*100,2),'%'),0) as `有效私信咨询人次/私信咨询人次`
,clue_dev as `授权电话线索设备数-总计`
,clue_num as `授权电话线索人次-总计`
,call_dev+call_dev_d as `拨打电话线索设备数-总计`
,call_num+call_num_d as `拨打电话线索人次-总计`
,nvl(concat(round((call_num+call_num_d)/clue_num*100,2),'%'),0) as `拨打电话人次/授权电话人次`
,valid_call_dev+valid_call_dev_d as `有效电话线索设备数-总计`
,valid_call_num+valid_call_num_d as `有效电话线索人次-总计`
,call_dev as `当天拨打电话线索设备数-总计`
,call_num as `当天拨打电话线索人次-总计`
,valid_call_dev as `当天有效电话线索设备数-总计`
,valid_call_num as `当天有效电话线索人次-总计`
,msg_dev_3 `私信咨询设备数-3级以上商户`
,msg_num_3 `私信咨询人次-3级以上商户`
,nvl(concat(round(msg_num_3/msg_num*100,2),'%'),0) as `私信咨询人次-3级以上/私信咨询人次`
,valid_msg_dev_3 `有效私信咨询设备数-3级以上商户`
,valid_msg_num_3 as `有效私信咨询人次-3级以上商户`
,clue_dev_3 as `授权电话线索设备数-3级以上商户`
,clue_num_3 as `授权电话线索人次-3级以上商户`
,nvl(concat(round(clue_num_3/clue_num*100,2),'%'),0) as `授权电话线索-3级以上/授权电话线索`
,call_dev_3+call_dev_3_d as `拨打电话线索设备数-3级以上商户`
,call_num_3+valid_call_num_3 as `拨打电话线索人次-3级以上商户`
,nvl(concat(round((call_num_3+valid_call_num_3)/clue_num_3*100,2),'%'),0) as `拨打电话线索人次-3级以上/授权电话线索人次-3级以上`
,valid_call_dev_3+valid_call_dev_3_d as `有效电话线索设备数-3级以上商户`
,valid_call_num_3+valid_call_num_3_d as `有效电话线索人次-3级以上商户`
,nvl(concat(round((valid_call_num_3+valid_call_num_3_d)/clue_num_3*100,2),'%'),0) as `有效电话线索人次-3级以上/授权电话线索人次-3级以上`
,call_dev_3 as `当天拨打电话线索设备数-3级以上商户`
,call_num_3 as `当天拨打电话线索人次-3级以上商户`
,valid_call_dev_3 as `当天有效电话线索设备数-3级以上商户`
,valid_call_num_3 as `当天有效电话线索人次-3级以上商户`
,msg_dev_kyc `私信咨询设备数-kyc`
,msg_num_kyc `私信咨询人次-kyc`
,valid_msg_dev_kyc `有效私信咨询设备数-kyc`
,valid_msg_num_kyc as `有效私信咨询人次-kyc`
,clue_dev_kyc as `授权电话线索设备数-kyc`
,clue_num_kyc as `授权电话线索人次-kyc`
,call_dev_kyc+call_dev_kyc_d as `拨打电话线索设备数-kyc`
,call_num_kyc+call_num_kyc_d as `拨打电话线索人次-kyc`
,valid_call_dev_kyc+valid_call_dev_kyc_d as `有效电话线索设备数-kyc`
,valid_call_num_kyc+valid_call_num_kyc_d as `有效电话线索人次-kyc`
,call_dev_kyc as `当天拨打电话线索设备数-kyc`
,call_num_kyc as `当天拨打电话线索人次-kyc`
,valid_call_dev_kyc as `当天有效电话线索设备数-kyc`
,valid_call_num_kyc as `当天有效电话线索人次-kyc`
,msg_dev_qa `私信咨询设备数-需求自测`
,msg_num_qa `私信咨询人次-需求自测`
,valid_msg_dev_qa `有效私信咨询设备数-需求自测`
,valid_msg_num_qa as `有效私信咨询人次-需求自测`
,clue_dev_qa as `授权电话线索设备数-需求自测`
,clue_num_qa as `授权电话线索人次-需求自测`
,call_dev_qa+call_dev_qa_d as `拨打电话线索设备数-需求自测`
,call_num_qa+call_num_qa_d as `拨打电话线索人次-需求自测`
,valid_call_dev_qa+valid_call_dev_qa_d as `有效电话线索设备数-需求自测`
,valid_call_num_qa+valid_call_num_qa_d as `有效电话线索人次-需求自测`
,call_dev_qa_d as `当天拨打电话线索设备数-需求自测`
,call_num_qa_d as `当天拨打电话线索人次-需求自测`
,valid_call_dev_qa_d as `当天有效电话线索设备数-需求自测`
,valid_call_num_qa_d as `当天有效电话线索人次-需求自测`
FROM pm.tl_pm_userclue_d
where partition_day in ('20200901','202001001','20201101','20201201','20210101',regexp_replace(date_sub(current_date,1),'-',''))
order by `日期`,`系统`,`新老`,`是否灰度`
pm/daily_content_data/etl/daily_content_data.sql
View file @ d33f53e6
......@@ -14,6 +14,8 @@ CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_content_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT
T1.partition_date AS day_id
,T1.device_os_type AS device_os_type
......@@ -614,7 +616,9 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action = 'on_click_card'
AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
and params['transaction_type'] not like '%ctr'
and params['transaction_type'] not like '%cvr'
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL
......@@ -626,7 +630,9 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
and params['transaction_type'] not like '%ctr'
and params['transaction_type'] not like '%cvr'
UNION ALL
......@@ -637,7 +643,8 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action = 'on_click_card'
AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL
......@@ -649,7 +656,8 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
)t2
ON t1.partition_date=t2.partition_date
AND t1.device_id=t2.cl_id
......@@ -1161,3 +1169,5 @@ ON T1.partition_date=T9.partition_date
AND T1.channel=T9.channel
ORDER BY day_id desc,device_os_type,active_type,is_ai_channel;
pm/daily_content_data/job/step3.job
View file @ d33f53e6
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/liudi@igengmei.com,wangxin@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/liudi@igengmei.com,wangxin@igengmei.com,dengguangyu@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
pm/daily_operation/en-cn.properties 0 → 100644
View file @ d33f53e6
daily_operation=运营日报
\ No newline at end of file
pm/daily_operation/etl/create_daily_operation.sql 0 → 100644
View file @ d33f53e6
--***************************************************************
--*脚本名称:
--*功能: 运营日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_operation_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
dau bigint comment '{"chs_name":"dau(去除积分墙)","description":"","etl":"","value":"","remark":""}',
login_dev_num bigint comment '{"chs_name":"新增注册设备数","description":"","etl":"","value":"","remark":""}',
login_user_num bigint comment '{"chs_name":"新增注册账号","description":"","etl":"","value":"","remark":""}',
login_phone_user_num bigint comment '{"chs_name":"新增注册手机号用户数","description":"","etl":"","value":"","remark":""}',
retention_1 string comment '{"chs_name":"次日留存率","description":"","etl":"","value":"","remark":""}',
retention_7 string comment '{"chs_name":"第7日留存率","description":"","etl":"","value":"","remark":""}',
retention_30 string comment '{"chs_name":"第30日留存率","description":"","etl":"","value":"","remark":""}',
per_open_times bigint comment '{"chs_name":"单设备打开频次","description":"","etl":"","value":"","remark":""}',
home_feeds_ctr string comment '{"chs_name":"信息流内容ctr","description":"","etl":"","value":"","remark":""}',
per_use_time double comment '{"chs_name":"单设备app停留时长(M)","description":"","etl":"","value":"","remark":""}',
neirong_page_stay double comment '{"chs_name":"单设备内容时长(M)","description":"","etl":"","value":"","remark":""}',
neirong_in_app_use_time string comment '{"chs_name":"单设备内容时长/app停留时长","description":"","etl":"","value":"","remark":""}',
service_privacy_uv bigint comment '{"chs_name":"到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
service_click_uv bigint comment '{"chs_name":"同意隐私授权设备设备数","description":"","etl":"","value":"","remark":""}',
service_privacy_uv_in_dau string comment '{"chs_name":"到达隐私授权界面设备数/DAU","description":"","etl":"","value":"","remark":""}',
service_click_uv_in_dau string comment '{"chs_name":"同意隐私授权设备数/到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
open_push_uv bigint comment '{"chs_name":"开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
open_push_uv_in_dau string comment '{"chs_name":"开启推送授权设备数/DAU","description":"","etl":"","value":"","remark":""}',
push_receive_uv_in_open_push_uv string comment '{"chs_name":"次日接收推送设备数/当天开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
click_uv bigint comment '{"chs_name":"点击推送的设备数","description":"","etl":"","value":"","remark":""}',
non_click_uv bigint comment '{"chs_name":"未点击推送的设备数","description":"","etl":"","value":"","remark":""}',
click_uv_in_dau string comment '{"chs_name":"点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
non_click_uv_in_dau string comment '{"chs_name":"未点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
op_push_open_uv bigint comment '{"chs_name":"点击运营推送设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv bigint comment '{"chs_name":"点击系统推送设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv bigint comment '{"chs_name":"点击个性化推送设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv bigint comment '{"chs_name":"点击签到推送设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_uv_in_receive string comment '{"chs_name":"点击运营推送设备/接收到运营推送设备","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv_in_receive string comment '{"chs_name":"点击系统推送设备/接收到系统推送设备","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv_in_receive string comment '{"chs_name":"点击个性化推送设备/接收到个性化推送设备","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv_in_receive string comment '{"chs_name":"点击签到推送设备/接收到签到推送设备","description":"","etl":"","value":"","remark":""}',
question_num bigint comment '{"chs_name":"主动提问数","description":"","etl":"","value":"","remark":""}',
answer_num bigint comment '{"chs_name":"主动回答数","description":"","etl":"","value":"","remark":""}',
topic_num bigint comment '{"chs_name":"主动发帖数","description":"","etl":"","value":"","remark":""}',
reply_num bigint comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
question_num_in_dau string comment '{"chs_name":"主动提问设备数/DAU`","description":"","etl":"","value":"","remark":""}',
answer_num_in_dau string comment '{"chs_name":"主动回答设备数/DAU","description":"","etl":"","value":"","remark":""}',
topic_num_in_dau string comment '{"chs_name":"主动发帖设备数/DAU","description":"","etl":"","value":"","remark":""}',
reply_num_in_dau string comment '{"chs_name":"主动评论设备数/DAU","description":"","etl":"","value":"","remark":""}',
neirongUV bigint comment '{"chs_name":"内容UV","description":"","etl":"","value":"","remark":""}',
neirongPV bigint comment '{"chs_name":"内容PV","description":"","etl":"","value":"","remark":""}',
neirongUV_in_dau string comment '{"chs_name":"内容UV/DAU","description":"","etl":"","value":"","remark":""}',
neirongPV_in_UV double comment '{"chs_name":"内容PV/内容UV","description":"","etl":"","value":"","remark":""}',
result_more_UV bigint comment '{"chs_name":"搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
result_meigou_UV bigint comment '{"chs_name":"搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
result_con_UV bigint comment '{"chs_name":"搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
result_more_UV_in_dau string comment '{"chs_name":"搜索结果综合页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_meigou_UV_in_dau string comment '{"chs_name":"搜索结果美购页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_con_UV_in_dau string comment '{"chs_name":"搜索结果商业页UV/DAU","description":"","etl":"","value":"","remark":""}',
result_more_PV bigint comment '{"chs_name":"搜索结果综合页PV","description":"","etl":"","value":"","remark":""}',
result_meigou_PV bigint comment '{"chs_name":"搜索结果美购页PV","description":"","etl":"","value":"","remark":""}',
result_con_PV bigint comment '{"chs_name":"搜索结果商业页PV","description":"","etl":"","value":"","remark":""}',
result_more_PV_in_UV double comment '{"chs_name":"搜索结果综合页PV/搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
result_meigou_PV_in_UV double comment '{"chs_name":"搜索结果美购页PV/搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
result_con_PV_in_UV double comment '{"chs_name":"搜索结果商业页PV/搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
result_more_PV_in_dau string comment '{"chs_name":"搜索结果综合页PV/DAU","description":"","etl":"","value":"","remark":""}',
result_meigou_PV_in_dau string comment '{"chs_name":"搜索结果美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
result_con_PV_in_dau string comment '{"chs_name":"搜索结果商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_meigou_PV bigint comment '{"chs_name":"来自内容页的美购页PV","description":"","etl":"","value":"","remark":""}',
neirong_con_PV bigint comment '{"chs_name":"来自内容页的商业页PV","description":"","etl":"","value":"","remark":""}',
neirong_meigou_PV_in_dau string comment '{"chs_name":"来自内容页的美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_con_PV_in_dau string comment '{"chs_name":"来自内容页的商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
neirong_click_pv bigint comment '{"chs_name":"内容页的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
wel_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的商业二跳/内容pv","description":"","etl":"","value":"","remark":""}',
neirong_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的内容二跳/内容pv","description":"","etl":"","value":"","remark":""}'
)comment '运营日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_operation/etl/daily_operation.sql 0 → 100644
View file @ d33f53e6
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--运营日报
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_channel_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT a.partition_date as day_id
,a.device_id as device_id
,device_os_type,active_type,first_channel_source_type,first_city
FROM
(
SELECT
partition_date,device_os_type
,device_id,first_channel_source_type,first_city
,case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END as active_type
FROM online.ml_device_day_active_status
where partition_date >= '20200101'
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LEFT JOIN
(
SELECT partition_date,cl_id
from online.bl_hdfs_maidian_updates
where partition_date >= '20200101'
AND partition_date < regexp_replace(current_date ,'-','')
AND action = 'popup_view'
AND params['popup_name']='service_privacy'
GROUP BY partition_date,cl_id
)b
ON a.device_id = b.cl_id
AND a.partition_date = b.partition_date
LEFT JOIN
(
SELECT partition_date,cl_id
from online.bl_hdfs_maidian_updates
where partition_date >= '20200101'
AND partition_date < regexp_replace(current_date ,'-','')
AND action = 'page_view'
GROUP BY partition_date,cl_id
)c
ON a.device_id = c.cl_id
AND a.partition_date = c.partition_date
LEFT JOIN
(
SELECT partition_day,device_id
FROM ml.ml_c_et_ck_click_dimen_d
where partition_day >= '20200101'
AND partition_day < regexp_replace(current_date ,'-','')
GROUP BY partition_day,device_id
)d
on a.device_id = d.device_id
AND a.partition_date = d.partition_day
WHERE b.cl_id is NOT NULL
AND c.cl_id is NULL
AND d.device_id is NULL
GROUP BY a.partition_date,a.device_id,device_os_type,active_type,first_channel_source_type,first_city;
INSERT OVERWRITE TABLE pm.tl_pm_operation_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_date as day_id
,t1.device_os_type as device_os_type
,t1.device_type as active_type
,t1.channel as channel
,t1.active_num as `DAU`
,t14.login_dev_num as login_dev_num
,t14.login_user_num as login_user_num
,t14.login_phone_user_num as login_phone_user_num
,concat(round(t1.retention_num1/t1.active_num*100,1),'%') as retention_1
,concat(round(t1.retention_num7/t1.active_num*100,1),'%') as retention_7
,concat(round(t1.retention_num30/t1.active_num*100,1),'%') as retention_30
,coalesce(t3.per_open_times,0) as per_open_times
,concat(round(coalesce(click_card_pv,0)/coalesce(exp_card_pv,0)*100,3),'%') as home_feeds_ctr
,coalesce(t3.per_use_time,0) as per_use_time
,round(coalesce(t4.neirong_page_stay,0)/coalesce(neirongUV,0)/60,3) as neirong_page_stay
,concat(round(round(coalesce(t4.neirong_page_stay,0)/coalesce(t1.active_num,0)/60,3)/coalesce(t3.per_use_time,0)*100,3),'%') as neirong_in_app_use_time
,coalesce(t4.service_privacy_uv,0) as service_privacy_uv
,coalesce(t4.service_click_uv,0) as service_click_uv
,concat(round(coalesce(t4.service_privacy_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as service_privacy_uv_in_dau
,concat(round(coalesce(t4.service_click_uv,0)/coalesce(t4.service_privacy_uv,0)*100,3),'%') as service_click_uv_in_dau
,coalesce(t11.open_push_uv,0) as open_push_uv
,concat(round(coalesce(t11.open_push_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `open_push_uv_in_dau`
,concat(round(coalesce(t11.push_receive_uv,0)/coalesce(t11.open_push_uv,0)*100,3),'%') as `push_receive_uv_in_open_push_uv`
,coalesce(t10.click_uv,0) as `click_uv`
,coalesce(t1.active_num-t10.click_uv,0) as `non_click_uv`
,concat(round(coalesce(t10.click_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `click_uv_in_dau`
,concat(round(coalesce(t1.active_num-t10.click_uv,0)/coalesce(t1.active_num,0)*100,3),'%') as `non_click_uv_in_dau`
,coalesce(t13.op_push_open_uv,0) as `op_push_open_uv`
,coalesce(t13.auto_push_open_uv,0) as `auto_push_open_uv`
,coalesce(t13.personal_push_open_uv,0) as `personal_push_open_uv`
,coalesce(t13.sign_push_open_uv,0) as `sign_push_open_uv`
,concat(round(coalesce(t13.op_push_open_uv,0)/coalesce(t12.op_receive_uv,0)*100,3),'%') as `op_push_open_uv_in_receive`
,concat(round(coalesce(t13.auto_push_open_uv,0)/coalesce(t12.auto_receive_uv,0)*100,3),'%') as `auto_push_open_uv_in_receive`
,concat(round(coalesce(t13.personal_push_open_uv,0)/coalesce(t12.personal_receive_uv,0)*100,3),'%') as `personal_push_open_uv_in_receive`
,concat(round(coalesce(t13.sign_push_open_uv,0)/coalesce(t12.sign_receive_uv,0)*100,3),'%') as `sign_push_open_uv_in_receive`
,coalesce(question_num,0) as `question_num`
,coalesce(answer_num,0) as `answer_num`
,coalesce(topic_num,0) as `topic_num`
,coalesce(reply_num,0) as `reply_num`
,concat(round(coalesce(question_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `question_num_in_dau`
,concat(round(coalesce(answer_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `answer_num_in_dau`
,concat(round(coalesce(topic_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `topic_num_in_dau`
,concat(round(coalesce(reply_dev_num,0)/coalesce(t1.active_num,0)*100,3),'%') as `reply_num_in_dau`
,coalesce(t4.neirongUV,0) as `neirongUV`
,coalesce(t4.neirongPV,0) as `neirongPV`
,concat(round(coalesce(t4.neirongUV,0)/t1.active_num*100,3),'%') as `neirongUV_in_dau`
,round(coalesce(t4.neirongPV,0)/coalesce(t4.neirongUV,0),2) as `neirongPV_in_UV`
--,round(coalesce(t4.neirongPV,0)/t2.active_num,2) as `内容PV/DAU`
,coalesce(result_more_UV,0) as `result_more_UV`
,coalesce(result_meigou_UV,0) as `result_meigou_UV`
,coalesce(result_con_UV,0) as `result_con_UV`
,concat(round(coalesce(result_more_UV,0)/t1.active_num*100,3),'%') as `result_more_UV_in_dau`
,concat(round(coalesce(result_meigou_UV,0)/t1.active_num*100,3),'%') as `result_meigou_UV_in_dau`
,concat(round(coalesce(result_con_UV,0)/t1.active_num*100,3),'%') as `result_con_UV_in_dau`
,coalesce(result_more_PV,0) as `result_more_PV`
,coalesce(result_meigou_PV,0) as `result_meigou_PV`
,coalesce(result_con_PV,0) as `result_con_PV`
,round(coalesce(result_more_PV,0)/coalesce(result_more_UV,0),2) as `result_more_PV_in_UV`
,round(coalesce(result_meigou_PV,0)/coalesce(result_meigou_UV,0),2) as `result_meigou_PV_in_UV`
,round(coalesce(result_con_PV,0)/coalesce(result_con_UV,0),2) as `result_con_PV_in_UV`
,concat(round(coalesce(result_more_PV,0)/t1.active_num*100,2),'%') as `result_more_PV_in_dau`
,concat(round(coalesce(result_meigou_PV,0)/t1.active_num*100,2),'%') as `result_meigou_PV_in_dau`
,concat(round(coalesce(result_con_PV,0)/t1.active_num*100,2),'%') as `result_con_PV_in_dau`
,coalesce(neirong_meigou_PV,0) as `neirong_meigou_PV`
,coalesce(neirong_con_PV,0) as `neirong_con_PV`
,concat(round(coalesce(neirong_meigou_PV,0)/t1.active_num*100,2),'%') as `neirong_meigou_PV_in_dau`
,concat(round(coalesce(neirong_con_PV,0)/t1.active_num*100,2),'%') as `neirong_con_PV_in_dau`
,coalesce(neirong_click_pv,0) as `neirong_click_pv`
,concat(round(coalesce(wel_second_pv,0)/neirongPV*100,2),'%') as `wel_second_pv_in_neirongPV`
,concat(round(coalesce(neirong_second_pv,0)/neirongPV*100,2),'%') as `neirong_second_pv_in_neirongPV`
FROM
(
--日活活跃、留存
SELECT
regexp_replace(substr(t1.partition_date,1,10),'-','') as partition_date
,device_os_type,t1.device_type,channel
,int(count(DISTINCT t1.device_id)) as active_num
,int(count(DISTINCT case WHEN date_add(t1.partition_date,1) = t2.partition_date then t2.device_id end)) as retention_num1
,int(count(DISTINCT case WHEN date_add(t1.partition_date,6) = t2.partition_date then t2.device_id end)) as retention_num7
,int(count(DISTINCT case WHEN date_add(t1.partition_date,29) = t2.partition_date then t2.device_id end)) as retention_num30
FROM
(
SELECT mas.partition_date,mas.device_id,device_type,device_os_type,t2.channel
FROM
(
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
)t1
LEFT JOIN
(
SELECT device_id
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
FROM online.ml_device_day_active_status
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)t2
on t1.device_id = t2.device_id
GROUP BY regexp_replace(substr(t1.partition_date,1,10),'-',''),device_os_type,t1.device_type,channel
)t1
LEFT JOIN
(
--App停留时长、打开频次
SELECT
partition_date,device_os_type,device_type,t.channel
,round(SUM(case when use_duration > 0 and use_duration < 86400 then use_duration else null end)/count(DISTINCT case when use_duration > 0 and use_duration < 86400 then device_id else null end)/60,1) as per_use_time
,round(SUM(open_times)/count(DISTINCT device_id),1) as per_open_times
FROM
(
SELECT b.device_id,use_duration,b.partition_date,open_times,device_os_type,device_type,channel
FROM
(
select device_id,use_duration,partition_date,open_times
from online.ml_device_updates
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)b
JOIN
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
when active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
AND first_channel_source_type not like 'promotion\_jf\_%'
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
)a
on a.device_id= b.device_id
and a.partition_date = b.partition_date
)a
LATERAL VIEW explode(a.channel) t AS channel
GROUP BY partition_date,device_os_type,device_type,t.channel
)t3
ON t1.partition_date = t3.partition_date
AND t1.device_os_type = t3.device_os_type
AND t1.device_type = t3.device_type
AND t1.channel = t3.channel
LEFT JOIN
(
--内容页、搜索各页PV、UV + 搜索数
SELECT partition_date,cl_type,device_type,channel
,count(case when action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirongPV
,count(DISTINCT case when action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then t1.cl_id else null end) as neirongUV
,count(case when action = 'page_view' AND page_name = 'search_result_more' then 1 else null end) as result_more_PV
,count(DISTINCT case when action = 'page_view' AND page_name = 'search_result_more' then t1.cl_id else null end) as result_more_UV
,count(case when action = 'page_view' AND page_name = 'search_result_welfare' then 1 else null end) as result_meigou_PV
,count(DISTINCT case when action = 'page_view' AND page_name = 'search_result_welfare' then t1.cl_id else null end) as result_meigou_UV
,count(case when action = 'page_view' AND page_name in ('search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization') then 1 else null end) as result_con_PV
,count(DISTINCT case when action = 'page_view' AND page_name in ('search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization') then t1.cl_id else null end) as result_con_UV
,count(case when action = 'page_view' AND page_name = 'welfare_detail' AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirong_meigou_PV
,count(case when action = 'page_view' AND page_name in ('welfare_detail','expert_detail','organization_detail') AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') then 1 else null end) as neirong_con_PV
,count(CASE WHEN action = 'click_neirong' THEN 1 ELSE NULL END) as neirong_click_pv
,count(CASE WHEN action = 'wel_second' THEN 1 ELSE NULL END) as wel_second_pv
,count(CASE WHEN action = 'neirong_second' THEN 1 ELSE NULL END) as neirong_second_pv
,sum(CASE WHEN action = 'page_view' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') AND (page_stay>=0 AND page_stay<1000) then page_stay else null end) as neirong_page_stay
,count(DISTINCT CASE WHEN action = 'popup_view' AND page_name = 'service_privacy' THEN t1.cl_id END) as service_privacy_uv
,count(DISTINCT CASE WHEN action = 'popup_click' AND page_name = 'service_privacy' THEN t1.cl_id END) as service_click_uv
FROM
(
SELECT
a.cl_id as cl_id
,a.partition_date
,page_name,cl_type,device_type,channel,referrer,action,page_stay
FROM
(
SELECT page_name,cl_id,partition_date,cl_type,referrer,action,page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and action = 'page_view'
AND ((page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail'
,'search_result_more','search_result_welfare','search_result_hospital','search_result_export','search_result_doctor','search_result_organization'))
or (page_name = 'welfare_detail' AND params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail'))
or (page_name in ('welfare_detail','expert_detail','organization_detail') AND referrer in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')))
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'click_neirong' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
and (action ='on_click_card' and params['card_content_type'] in ('qa','diary','user_post','answer')
or action='post_detail_browse_card')
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'wel_second' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
and params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND ((action='on_click_button' and params['button_name'] = 'more_recommendations')
or (action='on_click_button' and params['button_name']='video_interview')
or (action='on_click_button' and params['button_name']='referral')
or (action ='on_click_card' and params['card_content_type']='service'))
UNION ALL
SELECT page_name,cl_id,partition_date,cl_type,referrer,'neirong_second' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND params['referrer'] in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail') --内容详情页-搜索首页-进行搜索
and (action in ('on_click_navbar_search','do_search')
or (action ='on_click_card' and params['card_type']='highlight_word')
or (action='on_click_card' and params['card_content_type'] in ('qa','diary','user_post','answer')))
UNION ALL
SELECT 'service_privacy' as page_name,cl_id,partition_date,cl_type,'no' as referrer,'popup_view' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','') --隐私弹窗弹出事件
and action = 'popup_view'
AND params['popup_name'] ='service_privacy'
UNION ALL
SELECT 'service_privacy' as page_name,cl_id,partition_date,cl_type,'no' as referrer,'popup_click' as action,'no' as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and partition_date <= regexp_replace(date_sub(current_date,1),'-','') --隐私弹窗弹出事件
and action = 'on_click_button'
AND params['popup_name'] ='service_privacy'
AND params['type']='service_privacy'
AND params['button_name']='agree'
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel
FROM
(
SELECT
partition_date,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
when active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
LEFT JOIN
( -- 2.去掉疑似机构刷量的PV和UV
SELECT device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
GROUP BY t1.partition_date,cl_type,device_type,channel
)t4
ON t1.partition_date = t4.partition_date
AND t1.device_os_type = t4.cl_type
AND t1.device_type = t4.device_type
AND t1.channel = t4.channel
LEFT JOIN
( --来自信息流推荐内容的曝光卡片量
SELECT partition_date,device_os_type,active_type,t4.channel
,SUM(exp_card_pv) as exp_card_pv
FROM
(
SELECT mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id,count(DISTINCT t1.card_id) as exp_card_pv
FROM
(
SELECT partition_date
,device_os_type,m.device_id
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') and partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
JOIN
(
SELECT partition_day as partition_date,device_id as cl_id,card_id,app_session_id
FROM ml.mid_ml_c_et_pe_preciseexposure_dimen_d --新仓库精准曝光
WHERE partition_day >= regexp_replace(date_sub(current_date,30),'-','')and partition_day <= regexp_replace(date_sub(current_date,1),'-','')
AND page_code='home'
AND is_exposure='1'
AND tab_code='精选'
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND card_content_type in ('user_post','diary','qa','answer')
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id
)t3
LATERAL VIEW explode(t3.channel) t4 AS channel
group by partition_date,device_os_type,active_type,t4.channel
)t5
ON t1.partition_date = t5.partition_date
AND t1.device_os_type = t5.device_os_type
AND t1.device_type = t5.active_type
AND t1.channel = t5.channel
LEFT JOIN
( --来自信息流推荐内容的卡片点击量
SELECT partition_date,device_os_type,active_type,t4.channel
,SUM(click_card_pv) as click_card_pv
FROM
(
SELECT mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id,count(DISTINCT t1.card_id) as click_card_pv
FROM
(
SELECT partition_date
,device_os_type,m.device_id
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day =regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') and partition_date <=regexp_replace(date_sub(current_date,1),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
JOIN
(
SELECT partition_day as partition_date,device_id as cl_id,card_id,app_session_id
FROM ml.ml_c_et_ck_click_dimen_d --新仓库点击表
WHERE partition_day >= regexp_replace(date_sub(current_date,30),'-','')and partition_day <= regexp_replace(date_sub(current_date,1),'-','')
AND action='on_click_card'
AND page_code ='home'
AND tab_code = '精选'
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND card_content_type in ('diary','user_post','qa','answer')
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY mas.partition_date,device_os_type,active_type,channel,mas.device_id,app_session_id
)t3
LATERAL VIEW explode(t3.channel) t4 AS channel
group by partition_date,device_os_type,active_type,t4.channel
)t6
ON t1.partition_date = t6.partition_date
AND t1.device_os_type = t6.device_os_type
AND t1.device_type = t6.active_type
AND t1.channel = t6.channel
LEFT JOIN
(
--纯用户-内容发布成功数pv(去积分墙渠道、去医生账号、马甲号、模特用户)--真实用户
SELECT a.create_date,device_os_type,device_type,channel
,SUM(CASE WHEN type = 'question' THEN num END) as question_num
,SUM(CASE WHEN type = 'answer' THEN num END) as answer_num
,SUM(CASE WHEN type in ('tractate','diary','topic') THEN num END) as topic_num
,count(DISTINCT CASE WHEN type = 'question' THEN dev.device_id END) as question_dev_num
,count(DISTINCT CASE WHEN type = 'answer' THEN dev.device_id END) as answer_dev_num
,count(DISTINCT CASE WHEN type in ('tractate','diary','topic') THEN dev.device_id END) as topic_dev_num
FROM
( --新增帖子（用户发的）
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'tractate' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_api_tractate_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and is_online = 'true'
AND platform = '1' --更美用户发的(去除hera后台，爬虫爬取的)
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'tractate'
UNION ALL
--新增帖子（打卡发帖，去除新增用户中kyc用户的打卡发帖）
SELECT a.user_id,create_date,'tractate' as type,count(DISTINCT a.id) as num
FROM
(
SELECT user_id,id,regexp_replace(substr(create_time,1,10),'-','') as create_date
FROM online.tl_hdfs_api_tractate_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and is_online = 'true'
AND platform = '7' --打卡发帖
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
)a
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
LEFT JOIN
(
SELECT
partition_date,device_id
FROM online.ml_device_day_active_status --中kyc灰度的设备
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2')
AND ((partition_date <= '20200511' AND substr(convup(setencryption(device_id,'sha-1'),16,10),-1) = '0')
or (partition_date >= '20200512' AND substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0))
)c
ON b.partition_date = c.partition_date
AND b.device_id = c.device_id
WHERE c.device_id is NULL
GROUP BY a.user_id,create_date,'tractate'
UNION ALL
--新增日记本
SELECT a.user_id,a.create_date,'diary' as type,count(DISTINCT a.id) as num
FROM
(
SELECT user_id,id,regexp_replace(substr(created_time,1,10),'-','') as create_date
FROM online.tl_hdfs_diary_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
--AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(created_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(created_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
)a
JOIN
( --取非空日记
SELECT diary_id
FROM online.tl_hdfs_problem_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_spam = 'false'
)b
ON a.id = b.diary_id
GROUP BY a.user_id,a.create_date,'diary'
UNION ALL
--新增日记帖
SELECT user_id,regexp_replace(substr(created_time,1,10),'-','') as create_date,'topic' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_problem_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND diary_id is not null
and regexp_replace(substr(created_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(created_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(created_time,1,10),'-',''),'topic'
UNION ALL
--新增问题
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'question' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_question_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND platform = '99' --更美用户发的(去除hera后台，爬虫爬取的，kyc自动回复的)
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'question'
UNION ALL
--新增回答
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as create_date,'answer' as type,count(DISTINCT id) as num
FROM online.tl_hdfs_answer_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--and is_online = 'true'
AND is_spam = 'false' --排除疑似广告
AND platform = '99' --更美用户发的(去除hera后台，爬虫爬取的，kyc自动回复的)
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP BY user_id,regexp_replace(substr(create_time,1,10),'-',''),'answer'
)a
JOIN --限制用户是在app进行的发帖
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date<= regexp_replace(date_sub(current_date,1),'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '发帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.create_date = b.partition_date
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON a.user_id=c.user_id
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= regexp_replace(date_sub(current_date,30),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)d
ON a.user_id = d.user_id
AND a.create_date = d.partition_date
JOIN
( -- 1.去掉机刷、积分墙渠道来的设备
SELECT partition_date,device_id,device_os_type,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
LATERAL VIEW explode(dev.channel) t AS channel
)dev
on dev.device_id = d.device_id
AND dev.partition_date = d.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY a.create_date,device_os_type,device_type,channel
)t8
ON t1.partition_date = t8.create_date
AND t1.device_os_type = t8.device_os_type
AND t1.device_type = t8.device_type
AND t1.channel = t8.channel
LEFT JOIN
(
--真实评论数(日记/用户帖/问题/回答)
--有评论过日记帖的设备,排除疑似广告，排除医生账号，排除登陆过以上账号的设备和马甲号，限制用户当天有过评论行为
SELECT
a.reply_date,device_os_type,device_type,channel,SUM(reply_num) as reply_num,count(DISTINCT dev.device_id) as reply_dev_num
FROM
(
SELECT
user_id,reply_date,SUM(reply_num) as reply_num
FROM
(
--有评论过日记帖的设备,排除疑似广告
SELECT t1.user_id,reply_date,sum(reply_num) as reply_num
FROM
(
SELECT user_id,regexp_replace(substr(reply_date,1,10),'-','') as reply_date,problem_id,count(reply_date) as reply_num
FROM online.tl_hdfs_topicreply_view
WHERE partition_date =regexp_replace(date_sub(current_date,1),'-','')
AND is_spam = 'false' --排除疑似广告
and regexp_replace(substr(reply_date,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(reply_date,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
group by user_id,regexp_replace(substr(reply_date,1,10),'-',''),problem_id
)t1
JOIN
(
SELECT id,diary_id
FROM online.tl_hdfs_problem_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
group by id,diary_id
)t2
on t2.id=t1.problem_id
group by t1.user_id,reply_date
UNION ALL
--有评论过回答的设备,排除疑似广告
SELECT
t2.user_id as user_id,t2.reply_date as reply_date,sum(reply_num) as reply_num
FROM
(
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as reply_date,count(create_time) as reply_num,answer_id
FROM online.tl_hdfs_answer_reply_view
WHERE partition_date =regexp_replace(date_sub(current_date,1),'-','')
and (is_fake is NULL or is_fake = 'false')
AND answer_id is not NULL
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
group by user_id,answer_id,regexp_replace(substr(create_time,1,10),'-','')
)t2
GROUP BY t2.user_id,t2.reply_date
UNION ALL
--有评论过用户帖的设备
SELECT user_id,regexp_replace(substr(create_time,1,10),'-','') as reply_date,count(create_time) as reply_num
FROM online.tl_hdfs_api_tractate_reply_view
WHERE partition_date =regexp_replace(date_sub(current_date,1),'-','')
and regexp_replace(substr(create_time,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
and regexp_replace(substr(create_time,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
GROUP by user_id,regexp_replace(substr(create_time,1,10),'-','')
)reply_user
GROUP BY user_id,reply_date
)a
JOIN --限制用户是在app进行的回复
(
SELECT a.partition_date,user_id
FROM
(
SELECT partition_date,user_id,device_id,action
FROM online.bl_hdfs_operation_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date<= regexp_replace(date_sub(current_date,1),'-','')
)a
JOIN
(
SELECT code
FROM dim.dim_community_action_type
WHERE communityuserbehavior_type_name = '回帖'
)type
ON a.action = code
GROUP BY a.partition_date,user_id
)b
ON a.user_id = b.user_id
AND a.reply_date = b.partition_date
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON a.user_id=c.user_id
LEFT JOIN
(
SELECT t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >=regexp_replace(date_sub(current_date,30),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)d
ON a.user_id = d.user_id
AND a.reply_date = d.partition_date
JOIN
( -- 1.去掉机刷、积分墙渠道来的设备
SELECT partition_date,device_id,device_os_type,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
LATERAL VIEW explode(dev.channel) t AS channel
)dev
on dev.device_id = d.device_id
AND dev.partition_date = d.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY a.reply_date,device_os_type,device_type,channel
)t9
ON t1.partition_date = t9.reply_date
AND t1.device_os_type = t9.device_os_type
AND t1.device_type = t9.device_type
AND t1.channel = t9.channel
LEFT JOIN
(-- 推送点击 pv、uv
SELECT a.partition_date as partition_date,cl_type,device_type,channel
,count(a.cl_id) as click_pv
,count(DISTINCT a.cl_id) as click_uv
FROM
(
SELECT cl_id,partition_date,cl_type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND action = 'notification_open' --推送点击
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
GROUP BY a.partition_date,cl_type,device_type,channel
)t10
ON t1.partition_date = t10.partition_date
AND t1.device_os_type = t10.cl_type
AND t1.device_type = t10.device_type
AND t1.channel = t10.channel
LEFT JOIN
(-- 开启推送设备数
SELECT regexp_replace(substr(a.partition_date,1,10),'-','') as partition_date,device_os_type,device_type,channel
,count(DISTINCT a.cl_id) as open_push_uv
,count(distinct b.device_id) as push_receive_uv
FROM
(
SELECT concat_ws('-',substr(a.partition_date,1,4),substr(a.partition_date,5,2),substr(a.partition_date,7,2)) as partition_date
,a.cl_id --开启push设备
FROM
(
SELECT partition_date,cl_id,max(time_str) as max_time_str
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
and action='is_open_push'
group by partition_date,cl_id
)a
left join
(
SELECT partition_date,cl_id,time_str,COALESCE(params['type'],params['tyep']) as type
FROM online.bl_hdfs_maidian_updates
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
and action='is_open_push'
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.time_str=a.max_time_str
where type=1
group by a.partition_date,a.cl_id
)a
left join
(
select concat_ws('-',substr(partition_day,1,4),substr(partition_day,5,2),substr(partition_day,7,2)) as partition_date
,device_id
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
group by partition_day,device_id
union all
SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,device_id
FROM
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') and partition_date<=regexp_replace(date_sub(current_date,1),'-','')
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
group by partition_date,registration_id,platform,mesid
)msg
JOIN
(
SELECT device_id,registration_id
from online.tl_hdfs_api_jpushuserdevicebinding_view--极光id和设备id绑定关系全量表
where partition_date=regexp_replace(date_sub(current_date(),1),'-','')
and (registration_id is not null or registration_id <>'')
group by device_id,registration_id
)reg
on reg.registration_id=msg.registration_id
group by partition_date,device_id
)b
on a.cl_id=b.device_id and date_add(a.partition_date,1)=b.partition_date
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day =regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)c
ON a.cl_id = c.device_id
AND a.partition_date = c.partition_date
GROUP BY a.partition_date,device_os_type,device_type,channel
)t11
ON t1.partition_date = t11.partition_date
AND t1.device_os_type = t11.device_os_type
AND t1.device_type = t11.device_type
AND t1.channel = t11.channel
LEFT JOIN
(-- 接收到推送设备数--只有iOS设备能拿到数据，并且iOS设备的系统版本在10及以上
SELECT a.partition_date
,device_os_type
,case when d.device_id is not null then '新增设备' else '老活跃设备' end as device_type
,channel
,count(distinct a.device_id) as push_receive_uv
,count(distinct case when push_type in ('70','71') then a.device_id end) as op_receive_uv --运营push push_type in ('70','71')
,count(distinct case when push_type in ('25') then a.device_id end) as sign_receive_uv --运营push push_type in ('70','71')
,count(distinct case when push_type in ('101','102','103','104','105','106','107','108','111','112','113') then a.device_id end) as personal_receive_uv --个性化push
,count(distinct case when push_type is not null and push_type not in ('25','70','71','101','102','103','104','105','106','107','108','111','112','113') then a.device_id end) as auto_receive_uv --自动push
FROM
(
select partition_day as partition_date
,device_id,msg_id,'ios' as device_os_type
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day>=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
group by partition_day,device_id,msg_id
union all
SELECT partition_date,device_id,mesid as msg_id,'android' as device_os_type
FROM
(
SELECT partition_date,registration_id,platform,mesid
FROM online.tl_hdfs_auroracallback_view--推送push的回调表
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
--and notification_state='true'--用户开启通知
and callback_type = '0'--送达回执
and platform='android'
group by partition_date,registration_id,platform,mesid
)msg
JOIN
(
SELECT device_id,registration_id
from online.tl_hdfs_api_jpushuserdevicebinding_view--极光id和设备id绑定关系全量表
where partition_date=regexp_replace(date_sub(current_date(),1),'-','')
and (registration_id is not null or registration_id <>'')
group by device_id,registration_id
)reg
on reg.registration_id=msg.registration_id
)a
JOIN
(
select device_id,t.channel
from
(
select m.device_id
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
from
(
select m.device_id,m.first_channel_source_type
from online.ml_device_history_detail m
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day =regexp_replace(date_sub(current_date,1),'-',''))tmp
on m.first_channel_source_type=tmp.code
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id
)m
lateral view explode(channel) t as channel
)b
on a.device_id=b.device_id
JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)c
on a.msg_id=c.msg_id
left join
(
select
partition_date
,device_id
from online.ml_device_day_active_status
where partition_date>=regexp_replace(date_sub(current_date,30),'-','') AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','3')
)d
on a.partition_date=d.partition_date and a.device_id=d.device_id
group by a.partition_date
,device_os_type
,case when d.device_id is not null then '新增设备' else '老活跃设备' end
,channel
)t12
ON t1.partition_date = t12.partition_date
AND t1.device_os_type = t12.device_os_type
AND t1.device_type = t12.device_type
AND t1.channel = t12.channel
LEFT JOIN
(-- 点击推送设备数
SELECT a.partition_date as partition_date,device_os_type,device_type,channel
,count(DISTINCT a.cl_id) as receive_push_uv
,count(DISTINCT CASE WHEN op_push_open_pv is not null and op_push_open_pv>0 then a.cl_id end) as op_push_open_uv
,count(DISTINCT CASE WHEN auto_push_open_pv is not null and auto_push_open_pv>0 then a.cl_id end) as auto_push_open_uv
,count(DISTINCT CASE WHEN personal_push_open_pv is not null and personal_push_open_pv>0 then a.cl_id end) as personal_push_open_uv
,count(DISTINCT CASE WHEN sign_push_open_pv is not null and sign_push_open_pv>0 then a.cl_id end) as sign_push_open_uv
FROM
(
SELECT a.partition_date,a.cl_id,
count(distinct a.message_id) as push_open_pv,
count(distinct case when push_type in ('70','71') then a.message_id end) as op_push_open_pv, --运营push push_type in ('70','71')
count(distinct case when push_type is not null and push_type not in ('25','70','71','101','102','103','104','105','106','107','108','111','112','113') then a.message_id end) as auto_push_open_pv, --系统push即自动push
count(distinct case when push_type in ('101','102','103','104','105','106','107','108','111','112','113') then a.message_id end) as personal_push_open_pv, --个性化push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv --签到push
FROM
( --推送点击
SELECT partition_date,cl_id,time_str,params['message_id'] as message_id,time_stamp
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND action='notification_open'
GROUP BY partition_date,cl_id,time_str,params['message_id'],time_stamp
)a
LEFT JOIN
(
SELECT msg_id,push_type,time_stamp
FROM online.tl_hdfs_push2_new_view --增量表
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','') AND partition_date <=regexp_replace(date_sub(current_date,1),'-','')
GROUP BY msg_id,push_type,time_stamp
)b
on b.msg_id=a.message_id
group by a.partition_date,a.cl_id
)a
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)b
ON a.cl_id = b.device_id
AND a.partition_date = b.partition_date
GROUP BY a.partition_date,device_os_type,device_type,channel
)t13
ON t1.partition_date = t13.partition_date
AND t1.device_os_type = t13.device_os_type
AND t1.device_type = t13.device_type
AND t1.channel = t13.channel
LEFT JOIN
(--新增注册设备数/注册用户数
SELECT t1.partition_date,device_os_type,device_type,channel
,count(DISTINCT t1.device_id) as login_dev_num
,count(DISTINCT t1.user_id) as login_user_num
,count(DISTINCT CASE WHEN t3.user_id is not NULL THEN t2.user_id END) as login_phone_user_num
FROM
(
SELECT partition_date,device_id,a.user_id as user_id
FROM online.ml_device_updates
LATERAL VIEW explode(user_list)a as user_id
WHERE partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <= regexp_replace(date_sub(current_date,1),'-','')
)t1
JOIN
(
SELECT user_id,regexp_replace(substr(date_joined,1,10),'-','') as date_joined
FROM online.ml_user_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND regexp_replace(substr(date_joined,1,10),'-','') >= regexp_replace(date_sub(current_date,30),'-','')
AND regexp_replace(substr(date_joined,1,10),'-','') <= regexp_replace(date_sub(current_date,1),'-','')
)t2
ON t2.user_id=t1.user_id and t2.date_joined=t1.partition_date
LEFT JOIN
(
SELECT user_id,auth_type,phone
FROM online.tl_hdfs_userextra_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND auth_type = '1'
)t3
ON t2.user_id = t3.user_id
LEFT JOIN
(
SELECT user_id
FROM
(
--医生账号
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--更美内网用户
SELECT user_id
FROM dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)a
GROUP BY user_id
)c
ON t2.user_id=c.user_id
JOIN
(
SELECT partition_date,device_id,device_type,t.channel,device_os_type
FROM
(
SELECT
partition_date
,m.device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as device_type
,array(CASE WHEN (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') THEN '渠道可疑'
WHEN tmp.is_ai_channel='true' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT code,is_spam,is_ai_channel,partition_day
FROM DIM.DIM_AI_CHANNEL_ZP_NEW
WHERE partition_day >=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(date_sub(current_date,2),'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(SELECT device_id,day_id
FROM pm.tl_pm_channel_d
WHERE partition_day = regexp_replace(date_sub(current_date,1),'-','')
GROUP BY device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(date_sub(current_date,30),'-','')
AND partition_date <=regexp_replace(date_sub(current_date,1),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
LATERAL VIEW explode(b.channel) t AS channel
)t4
ON t1.device_id = t4.device_id
AND t1.partition_date = t4.partition_date
WHERE (c.user_id is null or c.user_id = '')
GROUP BY t1.partition_date,device_os_type,device_type,channel
)t14
ON t1.partition_date = t14.partition_date
AND t1.device_os_type = t14.device_os_type
AND t1.device_type = t14.device_type
AND t1.channel = t14.channel
pm/daily_operation/job/step1_1.job 0 → 100644
View file @ d33f53e6
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_operation/job/step1_10.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_diary_view
\ No newline at end of file
pm/daily_operation/job/step1_11.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_problem_view
\ No newline at end of file
pm/daily_operation/job/step1_12.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_question_view
\ No newline at end of file
pm/daily_operation/job/step1_13.job 0 → 100644
View file @ d33f53e6
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/daily_operation/job/step1_14.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_view
\ No newline at end of file
pm/daily_operation/job/step1_15.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file
pm/daily_operation/job/step1_16.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
pm/daily_operation/job/step1_17.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file
pm/daily_operation/job/step1_18.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file
pm/daily_operation/job/step1_19.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/daily_operation/job/step1_2.job 0 → 100644
View file @ d33f53e6
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/daily_operation/job/step1_20.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_topicreply_view
\ No newline at end of file
pm/daily_operation/job/step1_21.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_reply_view
\ No newline at end of file
pm/daily_operation/job/step1_22.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file
pm/daily_operation/job/step1_23.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive bl bl_et_bg_trackingpushlog_inc_d
\ No newline at end of file
pm/daily_operation/job/step1_24.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_auroracallback_view
\ No newline at end of file
pm/daily_operation/job/step1_25.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_jpushuserdevicebinding_view
\ No newline at end of file
pm/daily_operation/job/step1_26.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file
pm/daily_operation/job/step1_27.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file
pm/daily_operation/job/step1_3.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/daily_operation/job/step1_4.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file
pm/daily_operation/job/step1_5.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive dim DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file
pm/daily_operation/job/step1_7.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml mid_ml_c_et_pe_preciseexposure_dimen_d
\ No newline at end of file
pm/daily_operation/job/step1_8.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_ck_click_dimen_d
\ No newline at end of file
pm/daily_operation/job/step1_9.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file
pm/daily_operation/job/step2.job 0 → 100644
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17,step1_18,step1_19,step1_20,step1_21,step1_22,step1_23,step1_24,step1_25,step1_26,step1_27
command=sh /home/bi/bi-report/lib/shell/hive daily_operation
\ No newline at end of file
pm/daily_operation/job/step3.job 0 → 100644
View file @ d33f53e6
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_operation/liudi@igengmei.com,wangxin@igengmei.com,xuepengfei@igengmei.com,zhaoyang@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/daily_operation/report/daily_operation.sql 0 → 100644
View file @ d33f53e6
SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(day_id,'yyyymmdd'),'yyyy-mm-dd') `日期`
,device_os_type `系统`
,active_type `新老`
,channel `渠道`
,dau as `dau（去除积分墙）`
,nvl(login_dev_num,0) as `新增注册设备数`
,nvl(login_user_num,0) as `新增注册账号`
,nvl(login_phone_user_num,0) as `新增注册手机号用户数`
,nvl(retention_1,0) as `次日留存率`
,nvl(retention_7,0) as `第7日留存率`
,nvl(retention_30,0) as `第30日留存率`
,nvl(per_open_times,0) as `单设备打开频次`
,nvl(home_feeds_ctr,0) as `信息流内容ctr`
,nvl(per_use_time,0) as `单设备app停留时长(M)`
,nvl(neirong_page_stay,0) as `单设备内容时长(M)`
,nvl(neirong_in_app_use_time,0) as `单设备内容时长/app停留时长`
,nvl(service_privacy_uv,0) as `到达隐私授权界面设备数`
,nvl(service_click_uv,0) as `同意隐私授权设备设备数`
,nvl(service_privacy_uv_in_dau,0) as `到达隐私授权界面设备数/DAU`
,nvl(service_click_uv_in_dau,0) as `同意隐私授权设备数/到达隐私授权界面设备数`
,nvl(open_push_uv,0) as `开启推送授权设备数`
,nvl(open_push_uv_in_dau,0) as `开启推送授权设备数/DAU`
,nvl(push_receive_uv_in_open_push_uv,0) as `次日接收推送设备数/当天开启推送授权设备数`
,nvl(click_uv,0) as `点击推送的设备数`
,nvl(non_click_uv,0) as `未点击推送的设备数`
,nvl(click_uv_in_dau,0) as `点击推送设备数/DAU`
,nvl(non_click_uv_in_dau,0) `未点击推送设备数/DAU`
,nvl(op_push_open_uv,0) `点击运营推送设备数`
,nvl(auto_push_open_uv,0) as `点击系统推送设备数`
,nvl(personal_push_open_uv,0) `点击个性化推送设备数`
,nvl(sign_push_open_uv,0) as `点击签到推送设备数`
,nvl(op_push_open_uv_in_receive,0) as `点击运营推送设备/接收到运营推送设备`
,nvl(auto_push_open_uv_in_receive,0) as `点击系统推送设备/接收到系统推送设备`
,nvl(personal_push_open_uv_in_receive,0) as `点击个性化推送设备/接收到个性化推送设备`
,nvl(sign_push_open_uv_in_receive,0) as `点击签到推送设备/接收到签到推送设备`
,nvl(question_num,0) as `主动提问数`
,nvl(answer_num,0) as `主动回答数`
,nvl(topic_num,0) as `主动发帖数`
,nvl(reply_num,0) as `真实评论数`
,nvl(question_num_in_dau,0) as `主动提问设备数/DAU`
,nvl(answer_num_in_dau,0) as `主动回答设备数/DAU`
,nvl(topic_num_in_dau,0) as `主动发帖设备数/DAU`
,nvl(reply_num_in_dau,0) as `主动评论设备数/DAU`
,nvl(neirongUV,0) as `内容UV`
,nvl(neirongPV,0) `内容PV`
,nvl(neirongUV_in_dau,0) `内容UV/DAU`
,nvl(neirongPV_in_UV,0) `内容PV/内容UV`
,nvl(result_more_UV,0) as `搜索结果综合页UV`
,nvl(result_meigou_UV,0) as `搜索结果美购页UV`
,nvl(result_con_UV,0) as `搜索结果商业页UV`
,nvl(result_more_UV_in_dau,0) as `搜索结果综合页UV/DAU`
,nvl(result_meigou_UV_in_dau,0) as `搜索结果美购页UV/DAU`
,nvl(result_con_UV_in_dau,0) as `搜索结果商业页UV/DAU`
,nvl(result_more_PV,0) as `搜索结果综合页PV`
,nvl(result_meigou_PV,0) as `搜索结果美购页PV`
,nvl(result_con_PV,0) as `搜索结果商业页PV`
,nvl(result_more_PV_in_UV,0) as `搜索结果综合页PV/搜索结果综合页UV`
,nvl(result_meigou_PV_in_UV,0) as `搜索结果美购页PV/搜索结果美购页UV`
,nvl(result_con_PV_in_UV,0) `搜索结果商业页PV/搜索结果商业页UV`
,nvl(result_more_PV_in_dau,0) `搜索结果综合页PV/DAU`
,nvl(result_meigou_PV_in_dau,0) `搜索结果美购页PV/DAU`
,nvl(result_con_PV_in_dau,0) as `搜索结果商业页PV/DAU`
,nvl(neirong_meigou_PV,0) as `来自内容页的美购页PV`
,nvl(neirong_con_PV,0) as `来自内容页的商业页PV`
,nvl(neirong_meigou_PV_in_dau,0) as `来自内容页的美购页PV/DAU`
,nvl(neirong_con_PV_in_dau,0) as `来自内容页的商业页PV/DAU`
,nvl(neirong_click_pv,0) as `内容页的推荐内容点击pv`
,nvl(wel_second_pv_in_neirongPV,0) as `来自内容页的商业二跳/内容pv`
,nvl(neirong_second_pv_in_neirongPV,0) as `来自内容页的内容二跳/内容pv`
FROM pm.tl_pm_operation_d
where ((partition_day ='20200824' and day_id<='20200726')
or (partition_day<regexp_replace(date_sub(current_date,1),'-','') and partition_day>'20200824' and day_id=regexp_replace(date_sub(FROM_UNIXTIME(UNIX_TIMESTAMP(partition_day,'yyyymmdd'),'yyyy-mm-dd'),29),'-',''))
or (partition_day=regexp_replace(date_sub(current_date,1),'-','') and day_id>=regexp_replace(date_sub(current_date,30),'-','')))
order by `日期`,`系统`,`新老`,`渠道`
pm/daily_push/etl/create_push_quality.sql 0 → 100644
View file @ d33f53e6
--***************************************************************
--*脚本名称:
--*功能: push质量数据
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_quality_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
op_only_wel double comment '{"chs_name":"只通过运营push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
op_push_first_wel double comment '{"chs_name":"先通过运营push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
op_open_first_wel double comment '{"chs_name":"先主动打开再运营push-漏到核心页","description":"","etl":"","value":"","remark":""}',
op_only_clue double comment '{"chs_name":"只通过运营push来-漏到线索","description":"","etl":"","value":"","remark":""}',
op_push_first_clue double comment '{"chs_name":"先通过运营push来-漏到线索","description":"","etl":"","value":"","remark":""}',
op_open_first_clue double comment '{"chs_name":"先主动打开再运营push-漏到线索","description":"","etl":"","value":"","remark":""}',
auto_only_wel double comment '{"chs_name":"只通过自动化push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
auto_push_first_wel double comment '{"chs_name":"先通过自动化push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
auto_open_first_wel double comment '{"chs_name":"先主动打开再自动化push-漏到核心页","description":"","etl":"","value":"","remark":""}',
auto_only_clue double comment '{"chs_name":"只通过自动化push来-漏到线索","description":"","etl":"","value":"","remark":""}',
auto_push_first_clue double comment '{"chs_name":"先通过自动化push来-漏到线索","description":"","etl":"","value":"","remark":""}',
auto_open_first_clue double comment '{"chs_name":"先主动打开再自动化push-漏到线索","description":"","etl":"","value":"","remark":""}',
)comment 'push质量数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/daily_push.sql
View file @ d33f53e6
......@@ -560,4 +560,338 @@ and t2.msg_id=t1.msg_id
and t2.cl_id=t1.device_id
group by t1.partition_day
,t1.device_os_type
,t1.push_type;
\ No newline at end of file
,t1.push_type;
--push质量数据
INSERT OVERWRITE TABLE pm.tl_pm_push_quality_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT partition_date as day_id
,device_os_type
,round(nvl((op_only_wel_pv/op_only_wel_uv)/(wel_pv/wel_uv),0),2) as op_only_wel
,round(nvl((op_push_first_wel_pv/op_push_first_wel_uv)/(wel_pv/wel_uv),0),2) as op_push_first_wel
,round(nvl((op_open_first_wel_pv/op_open_first_wel_uv)/(wel_pv/wel_uv),0),2) as op_open_first_wel
,round(nvl((op_only_clue_pv/op_only_clue_uv)/(clue_pv/clue_uv),0),2) as op_only_clue
,round(nvl((op_push_first_clue_pv/op_push_first_clue_uv)/(clue_pv/clue_uv),0),2) as op_push_first_clue
,round(nvl((op_open_first_clue_pv/op_open_first_clue_uv)/(clue_pv/clue_uv),0),2) as op_open_first_clue
,round(nvl((auto_only_wel_pv/auto_only_wel_uv)/(wel_pv/wel_uv),0),2) as auto_only_wel
,round(nvl((auto_push_first_wel_pv/auto_push_first_wel_uv)/(wel_pv/wel_uv),0),2) as auto_push_first_wel
,round(nvl((auto_open_first_wel_pv/auto_open_first_wel_uv)/(wel_pv/wel_uv),0),2) as auto_open_first_wel
,round(nvl((auto_only_clue_pv/auto_only_clue_uv)/(clue_pv/clue_uv),0),2) as auto_only_clue
,round(nvl((auto_push_first_clue_pv/auto_push_first_clue_uv)/(clue_pv/clue_uv),0),2) as auto_push_first_clue
,round(nvl((auto_open_first_clue_pv/auto_open_first_clue_uv)/(clue_pv/clue_uv),0),2) as auto_open_first_clue
FROM
(
SELECT mas.partition_date
,device_os_type
,count(distinct a.cl_id) as wel_uv
,sum(a.wel_pv) as wel_pv
,count(distinct b.device_id) as clue_uv
,sum(b.valid_pv) as clue_pv
,count(distinct case when c.push_type='运营push' and d.type='仅点击push' then a.cl_id end) as op_only_wel_uv
,sum(case when c.push_type='运营push' and d.type='仅点击push' then a.wel_pv end) as op_only_wel_pv
,count(distinct case when c.push_type='运营push' and d.type='仅点击push' then b.device_id end) as op_only_clue_uv
,sum(case when c.push_type='运营push' and d.type='仅点击push' then b.valid_pv end) as op_only_clue_pv
,count(distinct case when c.push_type='自动化push' and d.type='仅点击push' then a.cl_id end) as auto_only_wel_uv
,sum(case when c.push_type='自动化push' and d.type='仅点击push' then a.wel_pv end) as auto_only_wel_pv
,count(distinct case when c.push_type='自动化push' and d.type='仅点击push' then b.device_id end) as auto_only_clue_uv
,sum(case when c.push_type='自动化push' and d.type='仅点击push' then b.valid_pv end) as auto_only_clue_pv
,count(distinct case when c.push_type='运营push' and d.type='先通过push来后主动打开' then a.cl_id end) as op_push_first_wel_uv
,sum(case when c.push_type='运营push' and d.type='先通过push来后主动打开' then a.wel_pv end) as op_push_first_wel_pv
,count(distinct case when c.push_type='运营push' and d.type='先通过push来后主动打开' then b.device_id end) as op_push_first_clue_uv
,sum(case when c.push_type='运营push' and d.type='先通过push来后主动打开' then b.valid_pv end) as op_push_first_clue_pv
,count(distinct case when c.push_type='自动化push' and d.type='先通过push来后主动打开' then a.cl_id end) as auto_push_first_wel_uv
,sum(case when c.push_type='自动化push' and d.type='先通过push来后主动打开' then a.wel_pv end) as auto_push_first_wel_pv
,count(distinct case when c.push_type='自动化push' and d.type='先通过push来后主动打开' then b.device_id end) as auto_push_first_clue_uv
,sum(case when c.push_type='自动化push' and d.type='先通过push来后主动打开' then b.valid_pv end) as auto_push_first_clue_pv
,count(distinct case when c.push_type='运营push' and d.type='先主动打开后通过push来' then a.cl_id end) as op_open_first_wel_uv
,sum(case when c.push_type='运营push' and d.type='先主动打开后通过push来' then a.wel_pv end) as op_open_first_wel_pv
,count(distinct case when c.push_type='运营push' and d.type='先主动打开后通过push来' then b.device_id end) as op_open_first_clue_uv
,sum(case when c.push_type='运营push' and d.type='先主动打开后通过push来' then b.valid_pv end) as op_open_first_clue_pv
,count(distinct case when c.push_type='自动化push' and d.type='先主动打开后通过push来' then a.cl_id end) as auto_open_first_wel_uv
,sum(case when c.push_type='自动化push' and d.type='先主动打开后通过push来' then a.wel_pv end) as auto_open_first_wel_pv
,count(distinct case when c.push_type='自动化push' and d.type='先主动打开后通过push来' then b.device_id end) as auto_open_first_clue_uv
,sum(case when c.push_type='自动化push' and d.type='先主动打开后通过push来' then b.valid_pv end) as auto_open_first_clue_pv
FROM
(
SELECT partition_date
,device_os_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND active_type IN ('4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
LEFT JOIN
(
SELECT partition_date,cl_id,wel_pv
FROM
(
SELECT partition_date,cl_id,count(1) as wel_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<=regexp_replace(date_sub(current_date,1),'-','')
AND action = 'page_view'
AND page_name in ('welfare_detail','organization_detail','expert_detail')
GROUP BY partition_date,cl_id
)a
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d
where partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)a
on a.partition_date=mas.partition_date and a.cl_id=mas.device_id
LEFT JOIN
( --有效线索分日数据(按用户维度)
SELECT
t2.device_id,DATE_DAY,count(1) AS valid_pv
FROM
(
SELECT NVL(T1.USER_ID,T2.USER_ID) AS USER_ID,
NVL(T1.merchant_id,T2.merchant_id) AS merchant_id,
NVL(T1.DATE_MONTH,T2.CALL_MONTH) AS DATE_MONTH,
least(NVL(DATE_DAY,'99999999'),NVL(FIRST_CALL_DATE,'99999999')) AS DATE_DAY,
NVL(T1.CONSULT_NUM,0) AS CONSULT_NUM,
NVL(T1.PAY_NUM,0) AS PAY_NUM,
NVL(T2.call_num,0) AS CALL_NUM
FROM
(
SELECT
NVL(T3.USER_ID,T4.USER_ID) AS USER_ID,
NVL(T3.merchant_id,T4.merchant_id) AS merchant_id,
NVL(T3.CONSULT_MONTH,T4.PAY_MONTH) AS DATE_MONTH,
least(NVL(first_consult_date,'99999999'),NVL(FIRST_PAY_DATE,'99999999')) AS DATE_DAY,
NVL(T3.CONSULT_NUM,0) AS CONSULT_NUM,
NVL(T4.PAY_NUM,0) AS PAY_NUM
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS consult_month,min(partition_day) as first_consult_date,1 as CONSULT_NUM
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)T3
FULL JOIN
(
SELECT
T.USER_ID,
T.merchant_id,
regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','') AS PAY_MONTH,
MIN(regexp_replace(SUBSTR(T.PAY_DATE,1,10),'-','')) AS FIRST_PAY_DATE,
1 AS PAY_NUM
FROM ONLINE.ML_TRADE_ORDER_DETAIL_DAY T
WHERE
T.PARTITION_DATE = regexp_replace(date_sub(current_date(),1),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','') >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','')<=regexp_replace(date_sub(current_date,1),'-','')
AND T.is_pure_user='true'
GROUP BY T.USER_ID,T.merchant_id,regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','')
) T4
ON T3.USER_ID = T4.USER_ID AND T3.merchant_id = T4.merchant_id AND T3.CONSULT_MONTH = T4.PAY_MONTH
GROUP BY NVL(T3.USER_ID,T4.USER_ID),NVL(T3.merchant_id,T4.merchant_id),NVL(T3.CONSULT_MONTH,T4.PAY_MONTH),
least(NVL(first_consult_date,'99999999'),NVL(FIRST_PAY_DATE,'99999999')),NVL(T3.CONSULT_NUM,0),NVL(T4.PAY_NUM,0)
)T1
FULL JOIN
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as FIRST_CALL_DATE,1 AS call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date,1),'-','')
AND start_time<release_time --通话时长大于30秒
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)T2
ON T1.USER_ID = T2.USER_ID AND T1.merchant_id = T2.merchant_id AND T1.DATE_MONTH = T2.CALL_MONTH
)t1
JOIN
(
SELECT t.user_id,partition_date,if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and t.partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
)t2
on t1.user_id=t2.user_id and t1.DATE_DAY=t2.partition_date
GROUP BY t2.device_id,date_day
)b
ON b.date_day=mas.partition_date and b.device_id=mas.device_id
LEFT JOIN
(
SELECT a.partition_date,a.cl_id,
case when push_type in ('70','71') then '运营push'
when push_type not in ('70','71') and push_type is not null then '自动化push' end as push_type
FROM
(
SELECT a.partition_date,b.cl_id,b.message_id
FROM
(
--设备开启
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
and action='device_opened'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),app_session_id
)a
JOIN
(
--推送点击
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,params['message_id'] as message_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
and action='notification_open'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),params['message_id']
)b
on a.partition_date = b.partition_date
AND a.cl_id = b.cl_id
where abs(a.time_stamp-b.time_stamp)<=2
)a
LEFT join
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)b
on a.message_id=b.msg_id
group by a.partition_date,a.cl_id,
case when push_type in ('70','71') then '运营push'
when push_type not in ('70','71') and push_type is not null then '自动化push' end
)c
on c.cl_id=mas.device_id and c.partition_date=mas.partition_date
LEFT JOIN
(
SELECT t1.partition_date,t1.cl_id
,case when array_contains(t1.type,"点击push")=TRUE and size(t1.type)=1 then '仅点击push'
when array_contains(t1.type,"打开设备")=TRUE and size(t1.type)=1 then '仅打开设备'
when t2.type='首次push拉起' then '先通过push来后主动打开'
when t2.type ='首次非push拉起' then '先主动打开后通过push来' else null end as type
FROM
(
SELECT partition_date,cl_id,collect_set(type) as type
FROM
(
SELECT a.partition_date,a.cl_id
,case when abs(a.time_stamp-b.time_stamp)<=2 then '点击push'
else '打开设备' end as type
FROM
(
--设备开启
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
and action='device_opened'
and REGEXP_REPLACE(SUBSTR(time_str, 1, 10), '-', '')=partition_date
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19))
)a
left JOIN
(
--推送点击
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,params['message_id'] as message_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
and action='notification_open'
and REGEXP_REPLACE(SUBSTR(time_str, 1, 10), '-', '')=partition_date
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),params['message_id']
)b
on a.partition_date = b.partition_date
AND a.cl_id = b.cl_id
group by a.partition_date,a.cl_id,case when abs(a.time_stamp-b.time_stamp)<=2 then '点击push'else '打开设备' end
)a
group by partition_date,cl_id
)t1
left join
(
SELECT a.partition_date,a.cl_id,
case when abs(a.time_stamp-b.time_stamp)<=2 then '首次push拉起' else '首次非push拉起' end as type
FROM
(
--设备开启
SELECT partition_date,cl_id,min(unix_timestamp(substr(time_str,1,19))) as time_stamp
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
and action='device_opened'
and REGEXP_REPLACE(SUBSTR(time_str, 1, 10), '-', '')=partition_date
group by partition_date,cl_id
)a
left JOIN
(
--推送点击
SELECT partition_date,cl_id,min(unix_timestamp(substr(time_str,1,19))) as time_stamp
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and partition_date <= regexp_replace(date_sub(current_date(),1),'-','')
and action='notification_open'
and REGEXP_REPLACE(SUBSTR(time_str, 1, 10), '-', '')=partition_date
group by partition_date,cl_id
)b
on a.partition_date=b.partition_date and a.cl_id=b.cl_id
)t2
on t1.partition_date=t2.partition_date and t1.cl_id=t2.cl_id
)d
on c.cl_id=d.cl_id and c.partition_date=d.partition_date
group by mas.partition_date
,device_os_type
)t;
pm/daily_push/job/step1_10.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file
pm/daily_push/job/step1_11.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file
pm/daily_push/job/step1_12.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/daily_push/job/step1_13.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/daily_push/job/step1_14.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/daily_push/job/step1_15.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/daily_push/job/step1_16.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/daily_push/job/step1_17.job 0 → 100644
View file @ d33f53e6
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/daily_push/job/step2.job
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17
command=sh /home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file
pm/daily_push/job/step3.job
View file @ d33f53e6
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
pm/daily_recommend_strategy/en-cn.properties
View file @ d33f53e6
daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file
daily_recommend_strategy=首页推荐策略及内容页ctr日报
home_recommend_strategy=首页推荐策略日报
content_ctr=内容详情页灰度ctr
\ No newline at end of file
pm/daily_recommend_strategy/etl/create_content_grey.sql 0 → 100644
View file @ d33f53e6
--***************************************************************
--*脚本名称:
--*功能: 首页推荐策略日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_contentpage_ctr
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
page_name string comment '{"chs_name":"页面名称","description":"","etl":"","value":"","remark":""}',
content_pv bigint comment '{"chs_name":"内容详情页pv","description":"","etl":"","value":"","remark":""}',
content_uv bigint comment '{"chs_name":"内容详情页uv","description":"","etl":"","value":"","remark":""}',
wel_exp_pv bigint comment '{"chs_name":"你可能喜欢美购卡片曝光pv","description":"","etl":"","value":"","remark":""}',
content_exp_pv bigint comment '{"chs_name":"你可能喜欢内容卡片曝光pv","description":"","etl":"","value":"","remark":""}',
wel_click_pv bigint comment '{"chs_name":"你可能喜欢美购卡片点击pv","description":"","etl":"","value":"","remark":""}',
content_click_pv bigint comment '{"chs_name":"你可能喜欢内容卡片点击pv","description":"","etl":"","value":"","remark":""}',
slide_wel_click_pv bigint comment '{"chs_name":"横划卡片点击pv","description":"","etl":"","value":"","remark":""}',
self_wel_click_pv bigint comment '{"chs_name":"关联卡片点击pv","description":"","etl":"","value":"","remark":""}'
)comment '内容详情页卡片ctr'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_recommend_strategy/etl/create_daily_recommend_strategy.sql
View file @ d33f53e6
......@@ -14,94 +14,25 @@ SET mapreduce.job.queuename=data;
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_content_d
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
is_ai_channel string comment '{"chs_name":"是否AI渠道","description":"","etl":"","value":"","remark":""}',
dau int comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
content_uv int comment '{"chs_name":"内容详情页uv","description":"","etl":"","value":"","remark":""}',
content_pv int comment '{"chs_name":"内容详情页PV","description":"","etl":"","value":"","remark":""}',
per_content_uv string comment '{"chs_name":"内容详情页uv/dau","description":"","etl":"","value":"","remark":""}',
per_content_pv string comment '{"chs_name":"内容详情页pv/dau","description":"","etl":"","value":"","remark":""}',
retention_1 string comment '{"chs_name":"内容用户APP次留","description":"","etl":"","value":"","remark":""}',
retention_7 string comment '{"chs_name":"内容用户APP7留","description":"","etl":"","value":"","remark":""}',
retention_30 string comment '{"chs_name":"内容用户APP30留","description":"","etl":"","value":"","remark":""}',
avg_app_duration double comment '{"chs_name":"内容用户单设备App时长(m)","description":"","etl":"","value":"","remark":""}',
avg_content_stay double comment '{"chs_name":"内容用户单设备内容时长(m)","description":"","etl":"","value":"","remark":""}',
avg_open_times double comment '{"chs_name":"内容用户单设备打开次数","description":"","etl":"","value":"","remark":""}',
search_related_stay double comment '{"chs_name":"内容用户搜索相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
welfare_stay double comment '{"chs_name":"内容用户美购详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
content_question_stay double comment '{"chs_name":"内容用户问题详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
ai_related_stay double comment '{"chs_name":"内容用户AI相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
content_diary_stay double comment '{"chs_name":"内容用户日记详情页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
home_stay double comment '{"chs_name":"内容用户首页单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
conv_related_stay double comment '{"chs_name":"内容用户咨询相关页面单设备页面时长(m)","description":"","etl":"","value":"","remark":""}',
recommend_rate double comment '{"chs_name":"首页feeds推荐进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_recommend_pv double comment '{"chs_name":"首页feeds推荐进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
feeds_rate double comment '{"chs_name":"首页feeds非推荐进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_feeds_pv double comment '{"chs_name":"首页feeds非推荐进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
search_rate double comment '{"chs_name":"搜索进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_search_pv double comment '{"chs_name":"搜索进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
zone_rate double comment '{"chs_name":"内容聚合页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_zone_pv double comment '{"chs_name":"内容聚合页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
content_rate double comment '{"chs_name":"内容详情页推荐板块进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_from_content_pv double comment '{"chs_name":"内容详情页推荐板块进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
blank_rate double comment '{"chs_name":"无来源页面(大多数为push)进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_blank_pv double comment '{"chs_name":"无来源页面(大多数为push)进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
comment_rate double comment '{"chs_name":"评论列表页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_comment_pv double comment '{"chs_name":"评论列表页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
org_rate double comment '{"chs_name":"医生医院主页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_org_pv double comment '{"chs_name":"医生医院主页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
category_rate double comment '{"chs_name":"品类聚合页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_category_pv double comment '{"chs_name":"品类聚合页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
my_diary_rate double comment '{"chs_name":"我的日记页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_my_diary_pv double comment '{"chs_name":"我的日记页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
ai_rate double comment '{"chs_name":"AI报告页进入内容UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_ai_pv double comment '{"chs_name":"AI报告页进入内容PV/UV","description":"","etl":"","value":"","remark":""}',
create_topic_num int comment '{"chs_name":"真实发帖数","description":"","etl":"","value":"","remark":""}',
create_reply_num int comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
diary_uv int comment '{"chs_name":"日记UV","description":"","etl":"","value":"","remark":""}',
diary_pv int comment '{"chs_name":"日记PV","description":"","etl":"","value":"","remark":""}',
diary_rate double comment '{"chs_name":"日记UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_diary_pv double comment '{"chs_name":"日记PV/日记UV","description":"","etl":"","value":"","remark":""}',
diary_stay double comment '{"chs_name":"日记单设备时长(m)","description":"","etl":"","value":"","remark":""}',
post_uv int comment '{"chs_name":"帖子UV","description":"","etl":"","value":"","remark":""}',
post_pv int comment '{"chs_name":"帖子PV","description":"","etl":"","value":"","remark":""}',
post_rate double comment '{"chs_name":"帖子UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_post_pv double comment '{"chs_name":"帖子PV/帖子UV","description":"","etl":"","value":"","remark":""}',
post_stay double comment '{"chs_name":"帖子单设备时长(m)","description":"","etl":"","value":"","remark":""}',
question_uv int comment '{"chs_name":"问题UV","description":"","etl":"","value":"","remark":""}',
question_pv int comment '{"chs_name":"问题PV","description":"","etl":"","value":"","remark":""}',
question_rate double comment '{"chs_name":"问题UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_question_pv double comment '{"chs_name":"问题PV/问题UV","description":"","etl":"","value":"","remark":""}',
question_stay double comment '{"chs_name":"问题单设备时长(m)","description":"","etl":"","value":"","remark":""}',
question_answer_uv int comment '{"chs_name":"问答UV","description":"","etl":"","value":"","remark":""}',
question_answer_pv int comment '{"chs_name":"问答PV","description":"","etl":"","value":"","remark":""}',
question_answer_rate double comment '{"chs_name":"问答UV/问答UV","description":"","etl":"","value":"","remark":""}',
per_question_answer_pv double comment '{"chs_name":"问答PV/问答UV","description":"","etl":"","value":"","remark":""}',
question_answer_stay double comment '{"chs_name":"问答单设备时长(m)","description":"","etl":"","value":"","remark":""}',
answer_uv int comment '{"chs_name":"回答UV","description":"","etl":"","value":"","remark":""}',
answer_pv int comment '{"chs_name":"回答PV","description":"","etl":"","value":"","remark":""}',
answer_rate double comment '{"chs_name":"回答UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_answer_pv double comment '{"chs_name":"回答PV/回答UV","description":"","etl":"","value":"","remark":""}',
answer_stay double comment '{"chs_name":"回答单设备时长(m)","description":"","etl":"","value":"","remark":""}',
video_uv int comment '{"chs_name":"视频UV","description":"","etl":"","value":"","remark":""}',
video_pv int comment '{"chs_name":"视频PV","description":"","etl":"","value":"","remark":""}',
video_rate double comment '{"chs_name":"视频UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_video_pv double comment '{"chs_name":"视频PV/视频UV","description":"","etl":"","value":"","remark":""}',
video_stay double comment '{"chs_name":"视频单设备时长(m)","description":"","etl":"","value":"","remark":""}',
wiki_uv int comment '{"chs_name":"百科UV","description":"","etl":"","value":"","remark":""}',
wiki_pv int comment '{"chs_name":"百科PV","description":"","etl":"","value":"","remark":""}',
wiki_rate double comment '{"chs_name":"百科UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_wiki_pv double comment '{"chs_name":"百科PV/百科UV","description":"","etl":"","value":"","remark":""}',
wiki_stay double comment '{"chs_name":"百科单设备时长(m)","description":"","etl":"","value":"","remark":""}',
article_uv int comment '{"chs_name":"专栏UV","description":"","etl":"","value":"","remark":""}',
article_pv int comment '{"chs_name":"专栏PV","description":"","etl":"","value":"","remark":""}',
article_rate double comment '{"chs_name":"专栏UV/内容UV","description":"","etl":"","value":"","remark":""}',
per_article_pv double comment '{"chs_name":"专栏PV/专栏UV","description":"","etl":"","value":"","remark":""}',
article_stay double comment '{"chs_name":"专栏单设备时长(m)","description":"","etl":"","value":"","remark":""}'
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
card_click bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
card_exposure bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
navbar_search bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
highlight_word bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
self_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_content_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_special_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
transfer_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
video_consultation bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}'
)comment '内容日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/daily_recommend_strategy/etl/create_home_retention.sql 0 → 100644
View file @ d33f53e6
--***************************************************************
--*脚本名称:
--*功能: 首页推荐策略日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
card_click bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
card_exposure bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
navbar_search bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
highlight_word bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
self_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_content_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
recommend_special_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
transfer_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
video_consultation bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}'
)comment '内容日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ d33f53e6
......@@ -7,7 +7,9 @@ set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';--进制转换函数，支持各种进制转换
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT
......@@ -23,10 +25,10 @@ SELECT
NVL(sum(highlight_pv),0) as highlight_word,
NVL(sum(self_wel_pv),0) as self_welfare_card,
NVL(sum(recom_wel_pv),0)-NVL(sum(self_wel_pv),0) as recommend_welfare_card,--需要排除关联的商品卡片点击
NVL(sum(recom_content_pv),0) as recommend_content_card,
NVL(sum(recom_content_pv),0) as recommend_content_card,
NULL as recommend_special_card,
NULL as transfer_card,
NULL as video_consultation
NVL(sum(referral_pv),0) as transfer_card,
NVL(sum(video_pv),0) as video_consultation
FROM
(
SELECT partition_date
......@@ -51,23 +53,27 @@ FROM
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
v.recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
when transaction_type in ('samecity_ctr') then 'samecity_ctr'
when transaction_type in ('samecity_cvr') then 'samecity_cvr'
when transaction_type in ('hotspot_feed') then 'hotspot_feed'
END AS recommend_type,
case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end as card_content_type,
CASE when transaction_type in ('fmctr') then array('fmctr','合计')
WHEN transaction_type like '%ctr' THEN array('ctr预估','合计')
WHEN transaction_type like '%cvr' THEN array('cvr预估','合计')
WHEN transaction_type in ('-1','smr') THEN array('smr','合计')
when transaction_type in ('pgc','hotspot') then array('热点卡片','合计')
when transaction_type in ('newdata') then array('保量卡片')
when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')
when transaction_type in ('aistragegy') then array('新用户AI帖优先')
when transaction_type in ('excestragegy') then array('新用户精华帖优先')
when transaction_type in ('fixedstragegy') then array('新氧新用户策略一')
when transaction_type in ('fixedstragegy_video') then array('新氧新用户策略二') end
AS recommend_type,
card_id,
app_session_id
from online.ml_community_precise_exposure_detail
......@@ -76,47 +82,57 @@ FROM
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata','samecity_ctr','samecity_cvr','hotspot_feed')
AND card_content_type in ('qa','diary','user_post','answer')
AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or transaction_type like '%ctr' or transaction_type like '%cvr')
AND card_content_type in ('qa','diary','user_post','answer','special_pool')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
case when card_content_type in ('qa','answer') then 'qa'
when card_content_type in ('special_pool') then 'special' else card_content_type end,
cl_id,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
when transaction_type in ('samecity_ctr') then 'samecity_ctr'
when transaction_type in ('samecity_cvr') then 'samecity_cvr'
when transaction_type in ('hotspot_feed') then 'hotspot_feed' END,
CASE when transaction_type in ('fmctr') then array('fmctr','合计')
WHEN transaction_type like '%ctr' THEN array('ctr预估','合计')
WHEN transaction_type like '%cvr' THEN array('cvr预估','合计')
WHEN transaction_type in ('-1','smr') THEN array('smr','合计')
when transaction_type in ('pgc','hotspot') then array('热点卡片','合计')
when transaction_type in ('newdata') then array('保量卡片')
when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')
when transaction_type in ('aistragegy') then array('新用户AI帖优先')
when transaction_type in ('excestragegy') then array('新用户精华帖优先')
when transaction_type in ('fixedstragegy') then array('新氧新用户策略一')
when transaction_type in ('fixedstragegy_video') then array('新氧新用户策略二') end,
card_id,
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
LATERAL VIEW explode (a.recommend_type) v as recommend_type
group by partition_date,card_content_type,cl_id,v.recommend_type,card_id
)t2
on t1.device_id=t2.cl_id and t1.partition_date=t2.partition_date
LEFT JOIN
LEFT JOIN
(--卡片，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
v.recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
when params['transaction_type'] in ('samecity_ctr') then 'samecity_ctr'
when params['transaction_type'] in ('samecity_cvr') then 'samecity_cvr'
when params['transaction_type'] in ('hotspot_feed') then 'hotspot_feed'
END AS recommend_type,
case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end as card_content_type,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
WHEN params['transaction_type'] in ('-1','smr') THEN array('smr','合计')
when params['transaction_type'] in ('pgc','hotspot') then array('热点卡片','合计')
when params['transaction_type'] in ('newdata') then array('保量卡片')
when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')
when params['transaction_type'] in ('aistragegy') then array('新用户AI帖优先')
when params['transaction_type'] in ('excestragegy') then array('新用户精华帖优先')
when params['transaction_type'] in ('fixedstragegy') then array('新氧新用户策略一')
when params['transaction_type'] in ('fixedstragegy_video') then array('新氧新用户策略二')
end AS recommend_type,
params['card_id'] as card_id,
app_session_id
from online.bl_hdfs_maidian_updates
......@@ -124,23 +140,29 @@ FROM
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata','samecity_ctr','samecity_cvr','hotspot_feed')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
AND params['card_content_type'] in ('qa','diary','user_post','answer','special_pool')
GROUP BY partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
when params['transaction_type'] in ('samecity_ctr') then 'samecity_ctr'
when params['transaction_type'] in ('samecity_cvr') then 'samecity_cvr'
when params['transaction_type'] in ('hotspot_feed') then 'hotspot_feed' END,
case when params['card_content_type'] in ('qa','answer') then 'qa'
when params['card_content_type'] in ('special_pool') then 'special' else params['card_content_type'] end,
CASE when params['transaction_type'] in ('fmctr') then array('fmctr','合计')
WHEN params['transaction_type'] like '%ctr' THEN array('ctr预估','合计')
WHEN params['transaction_type'] like '%cvr' THEN array('cvr预估','合计')
WHEN params['transaction_type'] in ('-1','smr') THEN array('smr','合计')
when params['transaction_type'] in ('pgc','hotspot') then array('热点卡片','合计')
when params['transaction_type'] in ('newdata') then array('保量卡片')
when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')
when params['transaction_type'] in ('aistragegy') then array('新用户AI帖优先')
when params['transaction_type'] in ('excestragegy') then array('新用户精华帖优先')
when params['transaction_type'] in ('fixedstragegy') then array('新氧新用户策略一')
when params['transaction_type'] in ('fixedstragegy_video') then array('新氧新用户策略二') end,
params['card_id'],
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
LATERAL VIEW explode (a.recommend_type) v as recommend_type
group by partition_date,card_content_type,cl_id,v.recommend_type,card_id
)t3
on t2.partition_date=t3.partition_date
and t2.cl_id=t3.cl_id
......@@ -149,44 +171,72 @@ FROM
and t2.recommend_type=t3.recommend_type
LEFT JOIN
(--页面浏览时长
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,page_stay
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND referrer='home'
AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t4
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t4
on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id
and t4.business_id=t3.card_id
and t4.page_name=t3.card_content_type
LEFT JOIN
(--搜索框和点击行为
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,navbar_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t5
on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id
......@@ -194,43 +244,60 @@ FROM
and t5.page_name=t3.card_content_type
LEFT JOIN
(--点击高亮词
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,highlight_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t6
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t6
on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id
and t6.business_id=t3.card_id
and t6.page_name=t3.card_content_type
LEFT JOIN
(--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv
SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as self_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top' or params['in_page_pos']='bottom')
or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top'
or (params['in_page_pos']='bottom' and params['position'] is null and cl_type='android')
or (params['in_page_pos']='bottom' and params['card_but_pos'] is not null and cl_type='ios'))
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name IN ('diary_detail','topic_detail')
......@@ -238,11 +305,20 @@ FROM
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
group by partition_date,cl_id,business_id,page_name
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t7
on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id
......@@ -250,63 +326,160 @@ FROM
and t7.page_name=t3.card_content_type
LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv
SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as recom_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
or action='on_click_button' and params['button_name']='unfold' and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail')
or action='on_click_button' and params['button_name'] = 'more_recommendations')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t8
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t8
on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id
and t8.business_id=t3.card_id
and t8.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv
SELECT partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,count(1) as recom_content_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
group by partition_date,cl_id,business_id,page_name
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
group by partition_date,cl_id,coalesce(b.id,a.business_id),a.page_name
)t9
on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(--视频面诊点击
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,video_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as video_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button'
and params['button_name']='video_interview'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t10
on t10.partition_date=t3.partition_date
and t10.cl_id=t3.cl_id
and t10.business_id=t3.card_id
and t10.page_name=t3.card_content_type
LEFT JOIN
(--转诊按钮点击
select partition_date,cl_id,coalesce(b.id,a.business_id) as business_id,a.page_name,referral_pv
from
(
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end as page_name,
count(1) as referral_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_button'
and params['button_name']='referral'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','custom_special')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa'
when page_name in ('custom_special') then 'special' else null end
)a
left join
(
select id,visual_page_id,'special' as page_name
from tl.tl_zx_api_special_pool
where partition_day =regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by id,visual_page_id
)b
on a.business_id=b.visual_page_id and a.page_name=b.page_name
)t11
on t11.partition_date=t3.partition_date
and t11.cl_id=t3.cl_id
and t11.business_id=t3.card_id
and t11.page_name=t3.card_content_type
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
......@@ -318,7 +491,7 @@ FROM
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=t2.cl_id
LEFT JOIN
LEFT JOIN
(
SELECT partition_date,device_id
FROM
......@@ -372,4 +545,266 @@ FROM
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
\ No newline at end of file
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
INSERT OVERWRITE TABLE pm.tl_pm_contentpage_ctr PARTITION (PARTITION_DAY = ${partition_day})
SELECT mas.partition_date day_id
,device_os_type
,active_type
,grey_type
,case when t1.page_name in ('post_detail','user_post_detail','doctor_post_detail') then '帖子详情页'
when t1.page_name in ('diary_detail') then '日记本详情页'
when t1.page_name in ('topic_detail') then '日记帖详情页'
when t1.page_name in ('question_answer_detail') then '问答详情页' end as page_name
,nvl(sum(t1.pv),0) as content_pv
,nvl(count(distinct t1.cl_id),0) as content_uv
,nvl(sum(t4.pv),0) as wel_exp_pv
,nvl(sum(t5.pv),0) as content_exp_pv
,nvl(sum(t2.pv),0) as wel_click_pv
,nvl(sum(t3.pv),0) as content_click_pv
,nvl(sum(t6.pv),0) as slide_wel_click_pv
,nvl(sum(t7.pv),0) as self_wel_click_pv
FROM
(
SELECT
partition_date,device_id,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
, '灰度' AS grey_type
FROM online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
join
(
SELECT partition_date,cl_id,page_name,count(cl_id) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
AND action='page_view'
AND int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name
)t1
on mas.partition_date=t1.partition_date
and mas.device_id=t1.cl_id
left join
(--你可能喜欢美购卡片点击
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type']='service'
and params['in_page_pos']='bottom'
and params['tab_name'] is null--与横划区域区分
and ((params['position'] is not null and cl_type='android')
or (params['card_but_pos'] is null and cl_type='ios')) --与关联卡片区分
AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
AND int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t2
on t1.partition_date=t2.partition_date
and t1.cl_id=t2.cl_id
and t1.page_name=t2.page_name
left join
(--你可能喜欢内容卡片点击
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action='on_click_card'
and params['card_content_type']<>'service'
and params['in_page_pos']='bottom'
and params['tab_name'] is null
AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
AND int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t3
on t1.partition_date=t3.partition_date
and t1.cl_id=t3.cl_id
and t1.page_name=t3.page_name
left join
(--你可能喜欢美购卡片曝光
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
select partition_date,cl_id,page_name,business_id,card_id,app_session_id
from online.ml_community_precise_exposure_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
and card_content_type='service'
and action ='page_precise_exposure'
and params['tab_name']='feed_recommend'
and params['in_page_pos']='bottom'
and int(split(app_version,'\\.')[1]) >= 30
and is_exposure='1'
group by partition_date,cl_id,page_name,business_id,card_id,app_session_id
)a
group by partition_date,cl_id,page_name
)t4
on t1.partition_date=t4.partition_date
and t1.cl_id=t4.cl_id
and t1.page_name=t4.page_name
left join
(--你可能喜欢内容卡片曝光
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
select partition_date,cl_id,page_name,business_id,card_id,app_session_id
from online.ml_community_precise_exposure_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
and card_content_type in ('diary','user_post','answer')
and action ='page_precise_exposure'
and params['tab_name']='feed_recommend'
and params['in_page_pos']='bottom'
and int(split(app_version,'\\.')[1]) >= 30
and is_exposure='1'
group by partition_date,cl_id,page_name,business_id,card_id,app_session_id
)a
group by partition_date,cl_id,page_name
)t5
on t1.partition_date=t5.partition_date
and t1.cl_id=t5.cl_id
and t1.page_name=t5.page_name
left join
(--横划美购卡片点击
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_answer_detail')
and (( action='on_click_button' and params['button_name'] = 'more_recommendations')
or (action ='on_click_card' and params['in_page_pos']='recommand' and params['card_content_type']='service')
or (action ='on_click_card' and params['tab_name']='hot_recommendation' and params['card_content_type']='service'))
and int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t6
on t1.partition_date=t6.partition_date
and t1.cl_id=t6.cl_id
and t1.page_name=t6.page_name
left join
(--关联卡片
SELECT partition_date,cl_id,page_name,count(1) as pv
FROM
(
SELECT partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top'
or (params['in_page_pos']='bottom' and params['position'] is null and cl_type='android')
or (params['in_page_pos']='bottom' and params['card_but_pos'] is not null and cl_type='ios'))
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name in ('diary_detail','topic_detail')
AND params['position'] is null
and int(split(app_version,'\\.')[1]) >= 30
group by partition_date,cl_id,page_name,business_id,params['card_id'],app_session_id
)a
group by partition_date,cl_id,page_name
)t7
on t1.partition_date=t7.partition_date
and t1.cl_id=t7.cl_id
and t1.page_name=t7.page_name
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=mas.device_id
LEFT JOIN
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)t1
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
)dev
on mas.partition_date=dev.partition_date and mas.device_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
group by mas.partition_date
,device_os_type,active_type,grey_type
,case when t1.page_name in ('post_detail','user_post_detail','doctor_post_detail') then '帖子详情页'
when t1.page_name in ('diary_detail') then '日记本详情页'
when t1.page_name in ('topic_detail') then '日记帖详情页'
when t1.page_name in ('question_answer_detail') then '问答详情页' end
order by 1,2,3,4,5;
pm/daily_recommend_strategy/job/step3.job
View file @ d33f53e6
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/daily_recommend_strategy/report/content_ctr.sql 0 → 100644
View file @ d33f53e6
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
grey_type as `灰度`,
page_name as `页面`,
NVL(CONCAT(ROUND(wel_click_pv/wel_exp_pv*100,2),'%'),0) as `内容页你可能喜欢美购卡片点击PV/该类卡片曝光PV`,
NVL(CONCAT(ROUND(content_click_pv/content_exp_pv*100,2),'%'),0) as `内容页你可能喜欢内容卡片点击PV/该类卡片曝光PV`,
content_pv as `内容详情页pv`,
content_uv as `内容详情页uv`,
wel_exp_pv as `你可能喜欢美购卡片曝光pv`,
content_exp_pv as `你可能喜欢内容卡片曝光pv`,
wel_click_pv as `你可能喜欢美购卡片点击pv`,
content_click_pv as `你可能喜欢内容卡片点击pv`,
slide_wel_click_pv as `横划卡片点击pv`,
self_wel_click_pv as `关联卡片点击pv`
FROM pm.tl_pm_contentpage_ctr
WHERE partition_day>='20200730' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`设备类型`,`活跃类型`,`灰度`,`页面`
\ No newline at end of file
pm/daily_recommend_strategy/report/daily_recommend_strategy.sql → pm/daily_recommend_strategy/report/home_recommend_strategy.sql
View file @ d33f53e6
......@@ -4,21 +4,20 @@ SELECT
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)as `有效二跳pv`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-- '未配置' as `来自I的推荐专题点击pv`,
transfer_card as `来自I的转诊点击pv`,
video_consultation as `来自I的视频面诊点击pv`FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200824' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/kyc_q2/en-cn.properties 0 → 100644
View file @ d33f53e6
kyc_q2=用户兴趣引导日报及ai自测日报
ai_daily=AI自测漏斗数据
ai_qa=AI自测问答选项
kyc_qa_daily=用户兴趣引导日报
pm/kyc_q2/job/step1_1.job 0 → 100644
View file @ d33f53e6
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/kyc_q2/job/step1_10.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_zx_api_ai_qa_answer
\ No newline at end of file
pm/kyc_q2/job/step1_11.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_zx_api_ai_qa_question
\ No newline at end of file
pm/kyc_q2/job/step1_12.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file
pm/kyc_q2/job/step1_13.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_message_view
\ No newline at end of file
pm/kyc_q2/job/step1_14.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_conversationuserstatus_view
\ No newline at end of file
pm/kyc_q2/job/step1_15.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_message_detail
\ No newline at end of file
pm/kyc_q2/job/step1_16.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_doctor_relation_dimen_day
\ No newline at end of file
pm/kyc_q2/job/step1_17.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/kyc_q2/job/step1_18.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/kyc_q2/job/step1_19.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/kyc_q2/job/step1_2.job 0 → 100644
View file @ d33f53e6
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/kyc_q2/job/step1_20.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/kyc_q2/job/step1_21.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/kyc_q2/job/step1_3.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/kyc_q2/job/step1_4.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_zx_api_ai_qa_question_answer
\ No newline at end of file
pm/kyc_q2/job/step1_5.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/kyc_q2/job/step1_6.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
pm/kyc_q2/job/step1_7.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file
pm/kyc_q2/job/step1_8.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file
pm/kyc_q2/job/step1_9.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/kyc_q2/job/step2.job 0 → 100644
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17,step1_18,step1_19,step1_20,step1_21
command=curl -X GET http://localhost:8553/api/report/email/kyc_q2/weiyimin@igengmei.com/weiyimin@qq.com
pm/kyc_q2/report/ai_daily.sql 0 → 100644
View file @ d33f53e6
SET mapreduce.job.queuename=data;
set role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
SELECT mas.partition_date
,count(distinct t1.cl_id) as `AI自测首页uv`
,sum(t1.pv) as `AI自测首页pv`
,count(distinct t5.cl_id) as `第一个问题出现uv`
,sum(t5.pv) as `第一个问题出现pv`
,count(distinct t4.cl_id) as `第一层答案点击uv`
,sum(t4.pv) as `第一层答案点击pv`
,count(distinct t2.cl_id) as `完成AI自测uv`
,sum(t2.pv) as `完成AI自测pv`
,count(distinct t3.cl_id) as `落地页点击uv`
,sum(t3.pv) as `落地页点击pv`
FROM
(
SELECT
partition_date,device_id,device_os_type
,case when active_type in ('1','2') then '新增设备'
when active_type ='4' then '老活跃设备' end as active_type
FROM online.ml_device_day_active_status
where partition_date>='20200720' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
AND substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0--取灰度
)mas
left join
(--自测页浏览
SELECT partition_date,cl_id,count(1) as pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200720' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'ai_conversation_detail'
and action='page_view'
group by partition_date,cl_id
)t1
on mas.partition_date=t1.partition_date and mas.device_id=t1.cl_id
left join
(--完成自测（即接下来会出现卡片）
SELECT a.partition_date,a.cl_id,sum(1) as pv
FROM
(
SELECT partition_date,json_map(t2.result,'string,string')['answer_id'] answer_id
,json_map(t2.result,'string,string')['question_id'] question_id
,cl_id
from
(
SELECT partition_date,cl_id
,json_split(params['result_list']) as result_list
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200720' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'ai_conversation_detail'
and action='on_click_button'
and params['button_name']='ai_answer'
)t1
LATERAL VIEW EXPLODE(t1.result_list) t2 as result
)a
join
(
select ai_question_id,ai_answer_id
from tl.tl_zx_api_ai_qa_question_answer
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and next_type=1--推荐卡片
)b
on a.answer_id=b.ai_answer_id and a.question_id=b.ai_question_id
group by a.partition_date,a.cl_id
)t2
on mas.partition_date=t2.partition_date and mas.device_id=t2.cl_id
left join
(--落地页点击pv
SELECT partition_date,cl_id,count(1) as pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200720' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'ai_conversation_detail'
and action='on_click_card'
group by partition_date,cl_id
)t3
on mas.partition_date=t3.partition_date and mas.device_id=t3.cl_id
left join
(--第一层答案点击
SELECT a.partition_date,a.cl_id,count(1) as pv
FROM
(
SELECT partition_date,json_map(t2.result,'string,string')['answer_id'] answer_id
,cl_id
from
(
SELECT partition_date,cl_id
,json_split(params['result_list']) as result_list
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200720' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'ai_conversation_detail'
and action='on_click_button'
and params['button_name']='ai_answer'
)t1
LATERAL VIEW EXPLODE(t1.result_list) t2 as result
)a
JOIN
(
select ai_question_id,ai_answer_id
from tl.tl_zx_api_ai_qa_question_answer
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and ai_question_id=2
)b
on a.answer_id=b.ai_answer_id
group by a.partition_date,a.cl_id
)t4
on mas.partition_date=t4.partition_date and mas.device_id=t4.cl_id
left join
(--第一层答案曝光
SELECT partition_date,cl_id,count(1) as pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200720' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'ai_conversation_detail'
and action='report_status'
and (params['content_id']=2 or params['concent_id']=2)
group by partition_date,cl_id
)t5
on mas.partition_date=t5.partition_date and mas.device_id=t5.cl_id
left join
( -- 去掉疑似机构刷量的PV和UV
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on mas.device_id=spam_pv.device_id
left join
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
)a
join
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)b
on a.user_id=b.user_id
group by partition_date,device_id
)dev
on mas.partition_date=dev.partition_date and mas.device_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is NULL
group by mas.partition_date;
\ No newline at end of file
pm/kyc_q2/report/ai_qa.sql 0 → 100644
View file @ d33f53e6
SET mapreduce.job.queuename=data;
set role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';--进制转换函数，支持各种进制转换
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
select t2.title
,sum(pv) as `选择pv`
,count(distinct cl_id) as `选择uv`
from
(--自测页浏览
SELECT json_map(t2.result,'string,string')['answer_id'] answer_id
,json_map(t2.result,'string,string')['question_id'] question_id
,cl_id
,count(1) as pv
from
(
SELECT cl_id
,json_split(params['result_list']) as result_list
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200729' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'ai_conversation_detail'
and action='on_click_button'
and params['button_name']='ai_answer'
)t1
LATERAL VIEW EXPLODE(t1.result_list) t2 as result
group by json_map(t2.result,'string,string')['answer_id']
,json_map(t2.result,'string,string')['question_id']
,cl_id
)t1
JOIN
(
SELECT id,title
FROM tl.tl_zx_api_ai_qa_answer
where partition_day= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and is_online='true'
)t2
on t1.answer_id=t2.id
join
(
SELECT distinct device_id
FROM online.ml_device_day_active_status
where partition_date>='20200729' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
AND substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0--取灰度
)mas
on mas.device_id=t1.cl_id
left join
( -- 去掉疑似机构刷量的PV和UV
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on mas.device_id=spam_pv.device_id
left join
(
SELECT distinct device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>='20200729' AND partition_date<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
join
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)b
on a.user_id=b.user_id
)dev
on t1.cl_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is NULL
group by t2.title;
\ No newline at end of file
pm/kyc_q2/report/kyc_qa_daily.sql 0 → 100644
View file @ d33f53e6
SET mapreduce.job.queuename=data;
set role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
SELECT mas.partition_date as `日期`
,mas.device_os_type as `设备类型`
,mas.active_type as `活跃类型`
,case when binding.type>=1 then '是' else '否' end as `是否授权电话`
,count(distinct mas.device_id) as `dau`
,count(distinct case when register_num>0 then phone.cl_id end) as `新注册设备`
,count(distinct is_open.cl_id) as `授权推送设备数`
,count(distinct loc.cl_id) as `授权地域设备数`
,count(distinct t1.cl_id) as `kyc兴趣页面uv`
,count(distinct case when kyc_phone_pv>0 then t2.cl_id end) as `授权电话页面浏览uv`
,count(distinct case when phone_agree_pv>0 then t3.cl_id end) as `同意授权电话uv`
,count(distinct case when phone_disagree_pv>0 then t3.cl_id end) as `不同意授权电话uv`
,sum(clue_pv) as `授权电话线索数`
,sum(call_clue_pv) as `机构拨打电话数量`
,sum(valid_clue_pv) as `有效电话线索数>30s`
,count(distinct case when kyc_message_pv>0 then t2.cl_id end) as `授权私信页面浏览uv`
,count(distinct case when message_agree_pv>0 then t3.cl_id end) as `同意发送私信uv`
,count(distinct case when message_disagree_pv>0 then t3.cl_id end) as `不同意发送私信uv`
,sum(t4.conv_pv) as `发送私信对话数`
,count(distinct case when t4.view_pv>0 then t4.cl_id end ) as `查看私信用户数`
,sum(t4.view_pv) as `查看私信对话数`
,sum(t4.valid_conv_pv) as `有效私信对话数`
,count(distinct case when t7.login_pv>0 then t7.cl_id end) as `一键登录点击uv`
,count(distinct case when t7.quit_pv>0 then t7.cl_id end) as `跳过点击uv`
,count(distinct case when t7.wechat_pv>0 then t7.cl_id end) as `微信点击uv`
,count(distinct case when t7.microblog_pv>0 then t7.cl_id end) as `微博点击uv`
,count(distinct case when t7.qq_pv>0 then t7.cl_id end) as `qq点击uv`
,count(distinct case when t7.apple_pv>0 then t7.cl_id end) as `苹果点击uv`
,count(distinct case when t7.change_account_pv>0 then t7.cl_id end) as `其他账号点击uv`
,count(distinct case when t8.page_pv>0 then t8.cl_id end) as `无法识别一键登录页，显示手机号登录页浏览uv`
,count(distinct case when t8.login_pv>0 then t8.cl_id end) as `手机号登录页手机登录点击uv`
,count(distinct case when t8.wechat_pv>0 then t8.cl_id end) as `手机号登录页微信点击uv`
,count(distinct case when t8.microblog_pv>0 then t8.cl_id end) as `手机号登录页微博点击uv`
,count(distinct case when t8.qq_pv>0 then t8.cl_id end) as `手机号登录页qq点击uv`
FROM
(
SELECT
partition_date,device_id,device_os_type
,case when active_type in ('1','2') then '新增设备'
when active_type ='4' then '老活跃设备' end as active_type
FROM online.ml_device_day_active_status
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and device_id is not null
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
AND substr(convup(setencryption(device_id,'sha-1'),16,10),-2,2)%20 = 0--取灰度
)mas
left join
(
select a.partition_date,a.cl_id --开启push设备
from
(
SELECT partition_date,cl_id,max(time_str) as max_time_str
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='is_open_push'
group by partition_date,cl_id
)a
left join
(
SELECT partition_date,
cl_id,time_str,COALESCE(params['type'],params['tyep']) as type
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='is_open_push'
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.time_str=a.max_time_str
where type=1
group by a.partition_date,a.cl_id
)is_open
on is_open.partition_date=mas.partition_date and is_open.cl_id=mas.device_id
left join
(--授权电话设备
select partition_date,cl_id
,count(case when params['is_new_register']=1 then 1 end) as register_num
,count(case when params['is_new_register']=0 then 1 end) as login_num
from online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action ='login_success'
and page_name in ('new_register','one_click_login','login_phone','login_password')
and params['button_name'] in ('qq','microblog','wechat','login','quick_login','phone','密码登录','手机登录')
and params['status']='succeed'
group by partition_date,cl_id
)phone
on phone.partition_date=mas.partition_date and phone.cl_id=mas.device_id
left join
(--手机号授权
SELECT partition_date,cl_id,sum(type) as type
FROM
(
select partition_date,cl_id,user_id
from online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and user_id <>''
group by partition_date,cl_id,user_id
)a
join
(
select user_id,case when length(phone)=11 then '1' else 0 end as type
from online.tl_hdfs_userextra_view
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)b
on a.user_id=b.user_id
group by partition_date,cl_id
)binding
on binding.partition_date=mas.partition_date and binding.cl_id=mas.device_id
left join
(--地理位置授权
SELECT partition_date,cl_id
,count(1) as pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'start_question'
and ((get_json_object(params['extra_param'], '$.kyc_id')=1 and action='page_view')
or (params['kyc_id']=1 and action='report_status'))
and device['lng'] not in ('0','0.0') and device['lat'] not in ('0','0.0')
group by partition_date,cl_id
)loc
on loc.partition_date=mas.partition_date and loc.cl_id=mas.device_id
left join
(--兴趣页面浏览（question_id=198）
SELECT partition_date,cl_id,count(1) as pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name = 'start_question'
and ((get_json_object(params['extra_param'], '$.kyc_id') in (1,2) and action='page_view' and get_json_object(params['extra_param'], '$.question_id') in (198))
or (params['kyc_id']in (1,2) and action='report_status' and params['result_list'] like '%"question_id":198%'))
group by partition_date,cl_id
)t1
on mas.partition_date=t1.partition_date and mas.device_id=t1.cl_id
left join
(--页面浏览
SELECT partition_date,cl_id
,count(distinct case when page_name='kyc_message' then 1 end) as kyc_message_pv
,count(distinct case when page_name='kyc_phone' then 1 end) as kyc_phone_pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('kyc_message','kyc_phone')
and action='page_view'
group by partition_date,cl_id
)t2
on mas.partition_date=t2.partition_date and mas.device_id=t2.cl_id
left join
(--kyc点击同意和不同意
SELECT partition_date,cl_id
,count( case when page_name='kyc_phone' and params['button_name']='agree' then 1 end) as phone_agree_pv
,count( case when page_name='kyc_phone' and params['button_name']='no_need' then 1 end) as phone_disagree_pv
,count( case when page_name='kyc_message' and params['button_name']='agree' then 1 end) as message_agree_pv
,count( case when page_name='kyc_message' and params['button_name']='no_need' then 1 end) as message_disagree_pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('kyc_message','kyc_phone')
and action='on_click_button'
and ((params['popup_name']='kyc_disagree' and params['button_name']='no_need')
or params['button_name']='agree')
group by partition_date,cl_id
)t3
on mas.partition_date=t3.partition_date and mas.device_id=t3.cl_id
left join
(
SELECT b.partition_date,b.cl_id
,count(distinct b.conversation_id) as conv_pv
,count(distinct c.doctor_id,c.user_id) as view_pv
,count(distinct d.conversation_id) as valid_conv_pv
FROM
(
SELECT b1.conversation_id,a.cl_id,b2.user_id,b2.doctor_id,a.partition_date
FROM
(--点击同意
SELECT partition_date,cl_id,user_id
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and params['button_name'] = 'agree'
and action='on_click_button'
and page_name='kyc_message'
group by partition_date,cl_id,user_id
)a
join
(
SELECT conversation_id,user_id,regexp_replace(to_date(send_time) ,'-','') as send_date
FROM online.tl_hdfs_message_view
WHERE partition_date =regexp_replace(DATE_SUB(current_date,1) ,'-','')
and regexp_replace(to_date(send_time) ,'-','')>='20200801'
and regexp_replace(to_date(send_time) ,'-','')<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and content like '%对%感兴趣'
group by conversation_id,user_id,regexp_replace(to_date(send_time) ,'-','')
)b1
on a.partition_date=b1.send_date and a.user_id=b1.user_id
join
(--conversation对应的user_id和doctor_id
SELECT conversation_id,t.doctor_id,t.user_id
FROM
(
SELECT conversation_id,collect_set(doctor_id) as doctor_id,collect_set(user_id) as user_id
FROM
(
SELECT t2.conversation_id
,t3.user_id as doctor_id
,case when t3.user_id is null then t2.user_id end user_id
FROM
(
SELECT conversation_id,user_id
FROM online.tl_hdfs_conversationuserstatus_view
WHERE partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by conversation_id,user_id
)t2
left join
(
SELECT user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
GROUP BY user_id
)t3
ON t2.user_id = t3.user_id
)a
group by conversation_id
)b
LATERAL VIEW EXPLODE(b.doctor_id) t as doctor_id
LATERAL VIEW EXPLODE(b.user_id) t as user_id
)b2
on b1.conversation_id=b2.conversation_id and b1.user_id=b2.user_id
)b
left join
(--查看私信
SELECT partition_date
,split(params['business_id'],'_')[0] as doctor_id
,split(params['business_id'],'_')[1] as user_id
,cl_id
,count(1) as view_pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name='conversation_detail'
and action='page_view'
group by partition_date
,split(params['business_id'],'_')[0]
,split(params['business_id'],'_')[1]
,cl_id
)c
on b.partition_date=c.partition_date
and b.doctor_id=c.doctor_id
and b.user_id=c.user_id
and b.cl_id=c.cl_id
left join
(--有效私信
SELECT t1.partition_date,t1.conversation_id,t1.user_id,t1.cl_id
FROM
(
SELECT partition_date, conversation_id ,user_id,cl_id,count(distinct msg_id) as msg_num
FROM online.ml_community_message_detail
WHERE partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND app['is_system']=0
AND msg_type <> '3'
AND target_doctor_id is not null
AND target_doctor_id <> ''
--去除更美客服聊天,即去掉所长大人、返现小助手、更美小秘书、小美医生、更美用户体验中心、更美测试
AND target_user_id NOT in ('23899734','515333','31605392','22','24481051','30174022','22691355','22144926','602329')
GROUP BY partition_date,conversation_id,user_id,cl_id
having msg_num>=3
)t1
JOIN --除去医生主动联系的（即认为对话框的第一句话是医生发的）
(
SELECT a.send_date,a.conversation_id
FROM
(
SELECT conversation_id, send_date,user_id
FROM
(
SELECT conversation_id,REGEXP_REPLACE(SUBSTR(send_time, 1, 10), '-', '') as send_date,user_id
,row_number () over (PARTITION BY REGEXP_REPLACE(SUBSTR(send_time, 1, 10), '-', ''),conversation_id ORDER BY send_time asc) rn
FROM online.tl_hdfs_message_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND REGEXP_REPLACE(SUBSTR(send_time, 1, 10), '-', '') >= '20200801'
AND REGEXP_REPLACE(SUBSTR(send_time, 1, 10), '-', '') <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
WHERE rn = 1
GROUP BY conversation_id,send_date,user_id
)a
LEFT JOIN
(
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
)b
ON a.user_id = b.user_id
WHERE b.user_id is NULL
GROUP BY a.send_date,a.conversation_id
)t2
on t1.partition_date=t2.send_date and t1.conversation_id=t2.conversation_id
)d
on b.partition_date=d.partition_date
and b.conversation_id=d.conversation_id
and b.user_id=d.user_id
and b.cl_id=d.cl_id
group by b.partition_date,b.cl_id
)t4
on mas.partition_date=t4.partition_date and mas.device_id=t4.cl_id
left join
(--授权的电话线索数
SELECT partition_date,cl_id,count(distinct cl_id,hospital_id) as clue_pv
from
(
SELECT partition_date,cl_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id
)t5
on mas.partition_date=t5.partition_date and mas.device_id=t5.cl_id
left join
(--授权的有效电话线索数
SELECT a.partition_date,cl_id
,count(1) as call_clue_pv
,count(case when type='有效' then 1 end) as valid_clue_pv
FROM
(
SELECT partition_date,cl_id,user_id,hospital_id
from
(
SELECT partition_date,cl_id,user_id,json_split(params['hospital_id'],'string') as hospital_ids
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name in ('kyc_phone')
and action='on_click_button'
and params['button_name']='agree'
and params['hospital_id'] like '%[%'--其实传的是doctor_id
)a
LATERAL VIEW explode(hospital_ids) b AS hospital_id
group by partition_date,cl_id,user_id,hospital_id
)a
join
(
SELECT doctor_id,merchant_id
FROM online.ml_doctor_relation_dimen_day
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND doctor_is_merchant='true' --医生是商户
GROUP BY doctor_id,merchant_id
)b
on a.hospital_id=b.doctor_id
join
(
SELECT user_id,merchant_id,partition_date,type
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
,case when unix_timestamp(substr(start_time,1,19))+30<=unix_timestamp(substr(release_time,1,19)) then '有效' else '无效' end as type
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>='20200801'
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,partition_date,type
)c
on a.user_id=c.user_id and b.merchant_id=c.merchant_id
group by a.partition_date,cl_id
)t6
on mas.partition_date=t6.partition_date and mas.device_id=t6.cl_id
left join
(
SELECT partition_date,cl_id
,count(case when params['button_name']='quick_login' then 1 end) as login_pv
,count(case when params['button_name']='quit' then 1 end) as quit_pv
,count(case when params['button_name']='wechat' then 1 end) as wechat_pv
,count(case when params['button_name']='microblog' then 1 end) as microblog_pv
,count(case when params['button_name']='qq' then 1 end) as qq_pv
,count(case when params['button_name']='apple' then 1 end) as apple_pv
,count(case when params['button_name']='change_account' then 1 end) as change_account_pv
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name='one_click_login'
and action='on_click_button'
and int(split(app_version,'\\.')[1]) >= 31
group by partition_date,cl_id
)t7
on mas.partition_date=t7.partition_date and mas.device_id=t7.cl_id
left join
(
SELECT a.partition_date,a.cl_id
,count(case when action='on_click_button' and button_name='login' then 1 end) as login_pv
,count(case when action='on_click_button' and button_name='wechat' then 1 end) as wechat_pv
,count(case when action='on_click_button' and button_name='microblog' then 1 end) as microblog_pv
,count(case when action='on_click_button' and button_name='qq' then 1 end) as qq_pv
,count(case when action='page_view'then 1 end) as page_pv
from
(
SELECT partition_date,cl_id,action,params['button_name'] as button_name
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and int(split(app_version,'\\.')[1]) >= 31
and page_name='login_phone'
)a
left join
(--不会出现一键登录页
SELECT partition_date,cl_id
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
and page_name='one_click_login'
and int(split(app_version,'\\.')[1]) >= 31
group by partition_date,cl_id
)b
on a.partition_date=b.partition_date and a.cl_id=b.cl_id
where b.cl_id is null
group by a.partition_date,a.cl_id
)t8
on mas.partition_date=t8.partition_date and mas.device_id=t8.cl_id
left join
( -- 去掉疑似机构刷量的PV和UV
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff
)spam_pv
on mas.device_id=spam_pv.device_id
left join
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>='20200801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
)a
join
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)b
on a.user_id=b.user_id
group by partition_date,device_id
)dev
on mas.partition_date=dev.partition_date and mas.device_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is NULL
group by mas.partition_date,mas.device_os_type,mas.active_type,case when binding.type>=1 then '是' else '否' end
order by `日期`,`设备类型`,`活跃类型`,`是否授权电话`;
pm/meigou_detail_page/job/step2.job
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,dailiang@igengmei.com,zhanghaoyu@igengmei.com,cp-sunyinghe@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
pm/meigou_detail_page/report/meigou_detail_page_dispense_pv.sql
View file @ d33f53e6
......@@ -41,7 +41,7 @@ FROM
SELECT t1.partition_date as `日期`
,t1.device_type as `设备类型`
,t1.active_type as `活跃类型`
,t2.channel as `渠道`
,t1.channel as `渠道`
,count(case when type='美购详情页' then cl_id end ) as `美购详情页浏览pv`
,count(case when type='加购物车' then cl_id end ) as `加购物车点击pv`
,count(case when type='评价列表页' then cl_id end ) as `评价列表页点击pv`
......@@ -71,6 +71,7 @@ FROM
AND partition_date<regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
and params['is_first']=1
)pv
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
......@@ -265,30 +266,37 @@ FROM
)click
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
......@@ -298,12 +306,13 @@ FROM
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON click.cl_id=mas.device_id
AND click.partition_date=mas.partition_date
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T1
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
pm/meigou_detail_page/report/meigou_detail_page_dispense_uv.sql
View file @ d33f53e6
--美购详情页的分发
SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
select `日期`,`设备类型`,`活跃类型`,`渠道`
,`美购详情页浏览uv`
,(`加购物车点击uv`+`评价列表页点击uv`+`立即支付点击uv`+`私信及电话咨询uv` +`选择美购项目点击uv`
+`机构主页点击uv`+`机构的其他美购点击uv`+`推荐美购点击uv`+`收藏点击uv`
......@@ -36,10 +36,10 @@ SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
FROM
(
SELECT t1.partition_date as `日期`
select t1.partition_date as `日期`
,t1.device_type as `设备类型`
,t1.active_type as `活跃类型`
,t2.channel as `渠道`
,t1.channel as `渠道`
,count(distinct case when type='美购详情页' then cl_id end ) as `美购详情页浏览uv`
,count(distinct case when type='加购物车' then cl_id end ) as `加购物车点击uv`
,count(distinct case when type='评价列表页' then cl_id end ) as `评价列表页点击uv`
......@@ -57,236 +57,244 @@ FROM
FROM
(
SELECT click.partition_date,click.cl_id,click.type,mas.active_type,mas.device_type,mas.channel
FROM
select click.partition_date,click.cl_id,click.type,mas.active_type,mas.device_type,mas.channel
from
( --浏览需要去掉疑似机构刷量的设备
SELECT uv.partition_date as partition_date,uv.cl_id as cl_id,type
FROM
select uv.partition_date as partition_date,uv.cl_id as cl_id,type
from
(
SELECT partition_date,cl_id,'美购详情页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
select partition_date,cl_id,'美购详情页' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='page_view'
and params['is_first']=1
)uv
LEFT JOIN
left join
( -- 去掉疑似机构刷量的UV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on uv.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
where spam_pv.device_id is null
UNION ALL
union all
SELECT partition_date,cl_id, '加购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_add'
select partition_date,cl_id, '加购物车' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='welfare_multiattribute_click_add'
UNION ALL
union all
SELECT partition_date,cl_id, '右上角购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_navbar_cart'
select partition_date,cl_id, '右上角购物车' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_navbar_cart'
UNION ALL
union all
--点击查看更多评论
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
select partition_date,cl_id, '评价列表页' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_comment'
and action='welfare_detail_click_comment'
UNION ALL
union all
--点击外显的评价标签
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='service_comment_click_tag'
UNION ALL
select partition_date,cl_id, '评价列表页' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='service_comment_click_tag'
union all
--点击外显的日记卡片
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_detail_comment_click_diary_card'
UNION ALL
SELECT partition_date,cl_id, '立即支付' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_buy'
UNION ALL
select partition_date,cl_id, '评价列表页' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='welfare_detail_comment_click_diary_card'
union all
select partition_date,cl_id, '立即支付' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='welfare_multiattribute_click_buy'
union all
--点击选择更美项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='sku_choose'
UNION ALL
select partition_date,cl_id, '选择美购项目' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_button'
and params['button_name']='sku_choose'
union all
--7.22新版本--点击外露的美购项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='service_sku'
UNION ALL
select partition_date,cl_id, '选择美购项目' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_button'
and params['button_name']='service_sku'
union all
--7.22新版本--点击查看更多
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='sku_all'
UNION ALL
select partition_date,cl_id, '选择美购项目' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_button'
and params['button_name']='sku_all'
union all
--点击私信按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
select partition_date,cl_id, '私信及电话咨询' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
and action='welfare_detail_click_message'
UNION ALL
union all
--7.20之后灰度上线的快捷咨询按钮和预约面诊按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name'] in ('question_tag','appointment')
UNION ALL
select partition_date,cl_id, '私信及电话咨询' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_button'
and params['button_name'] in ('question_tag','appointment')
union all
--点击机构下面的在线咨询和电话咨询
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
select partition_date,cl_id, '私信及电话咨询' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea_contact'
AND params['connect_type'] in ('onlineconsult','phone')
and action='welfare_detail_click_curearea_contact'
and params['connect_type'] in ('onlineconsult','phone')
UNION ALL
union all
SELECT partition_date,cl_id, '机构主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
select partition_date,cl_id, '机构主页' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
AND params['cure_type']='organization'
and action='welfare_detail_click_curearea'
and params['cure_type']='organization'
UNION ALL
union all
SELECT partition_date,cl_id, '机构的其他美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
select partition_date,cl_id, '机构的其他美购' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_seller_service_item'
and action='welfare_detail_click_seller_service_item'
UNION ALL
union all
SELECT partition_date,cl_id, '机构导航' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
select partition_date,cl_id, '机构导航' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_address'
UNION ALL
SELECT partition_date,cl_id, '推荐美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date <regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_card'
AND params['card_content_type']='service'
AND params['tab_name']='推荐'
UNION ALL
SELECT partition_date,cl_id, '收藏' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_favor'
AND params['favor_type']='service'
AND params['motion']='do'
UNION ALL
SELECT partition_date,cl_id, '医生主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
and action='welfare_detail_click_address'
union all
select partition_date,cl_id, '推荐美购' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date <regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_card'
and params['card_content_type']='service'
and params['tab_name']='推荐'
union all
select partition_date,cl_id, '收藏' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='on_click_favor'
and params['favor_type']='service'
and params['motion']='do'
union all
select partition_date,cl_id, '医生主页' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
AND params['cure_type']='doctor'
and action='welfare_detail_click_curearea'
and params['cure_type']='doctor'
UNION ALL
union all
SELECT partition_date,cl_id, '分享' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_click_share'
select partition_date,cl_id, '分享' as type
from online.bl_hdfs_maidian_updates
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
and partition_date < regexp_replace((current_date),'-','')
and page_name = 'welfare_detail'
and action='page_click_share'
)click
JOIN
join
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
......@@ -296,12 +304,13 @@ FROM
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON click.cl_id=mas.device_id
AND click.partition_date=mas.partition_date
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T1
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
pm/meigou_detail_page/report/meigou_detail_page_source_pv.sql
View file @ d33f53e6
......@@ -31,16 +31,16 @@ SELECT T1.partition_date as `日期`
,concat(round(T3.org_add_pv/T2.organization_detail_pv*100,2),'%') as `医院主页-点击加车率`
,concat(round(T3.org_buy_pv/T2.organization_detail_pv*100,2),'%') as `医院主页-点击立即支付率`
,concat(round(T3.org_msg_pv/T2.organization_detail_pv*100,2),'%') as `医院主页-点击私信咨询率`
,T1.category_pv as `品类聚合页pv`
,T2.category_pv as `品类聚合页-转化pv`
,T3.category_add_pv as `品类聚合页-转化点击加车pv`
,T3.category_buy_pv as `品类聚合页-转化点击立即支付pv`
,T3.category_msg_pv as `品类聚合页-转化点击私信咨询pv`
,concat(round(T2.category_pv/T1.category_pv*100,2),'%') as `品类聚合页-转化pv率`
,concat(round(T2.category_pv/T1.wel_pv*100,2),'%') as `品类聚合页-转化美购pv占比`
,concat(round(T3.category_add_pv/T2.category_pv*100,2),'%') as `品类聚合页-点击加车率`
,concat(round(T3.category_buy_pv/T2.category_pv*100,2),'%') as `品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_pv/T2.category_pv*100,2),'%') as `品类聚合页-点击私信咨询率`
,T1.category_pv as `1级品类聚合页pv`
,T2.category_pv as `1级品类聚合页-转化pv`
,T3.category_add_pv as `1级品类聚合页-转化点击加车pv`
,T3.category_buy_pv as `1级品类聚合页-转化点击立即支付pv`
,T3.category_msg_pv as `1级品类聚合页-转化点击私信咨询pv`
,concat(round(T2.category_pv/T1.category_pv*100,2),'%') as `1级品类聚合页-转化pv率`
,concat(round(T2.category_pv/T1.wel_pv*100,2),'%') as `1级品类聚合页-转化美购pv占比`
,concat(round(T3.category_add_pv/T2.category_pv*100,2),'%') as `1级品类聚合页-点击加车率`
,concat(round(T3.category_buy_pv/T2.category_pv*100,2),'%') as `1级品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_pv/T2.category_pv*100,2),'%') as `1级品类聚合页-点击私信咨询率`
,T1.conversation_detail_pv as `私信详情页pv`
,T2.conversation_detail_pv as `私信详情页-转化pv`
,T3.conv_add_pv as `私信详情页-转化点击加车pv`
......@@ -51,16 +51,16 @@ SELECT T1.partition_date as `日期`
,concat(round(T3.conv_add_pv/T2.conversation_detail_pv*100,2),'%') as `私信详情页-点击加车率`
,concat(round(T3.conv_buy_pv/T2.conversation_detail_pv*100,2),'%') as `私信详情页-点击立即支付率`
,concat(round(T3.conv_msg_pv/T2.conversation_detail_pv*100,2),'%') as `私信详情页-点击私信咨询率`
,T1.welfare_list_pv as `美购列表页pv`
,T2.welfare_list_pv as `美购列表页-转化pv`
,T3.wel_list_add_pv as `美购列表页-转化点击加车pv`
,T3.wel_list_buy_pv as `美购列表页-转化点击立即支付pv`
,T3.wel_list_msg_pv as `美购列表页-转化点击私信咨询pv`
,concat(round(T2.welfare_list_pv/T1.welfare_list_pv*100,2),'%') as `美购列表页-转化pv率`
,concat(round(T2.welfare_list_pv/T1.wel_pv*100,2),'%') as `美购列表页-转化美购pv占比`
,concat(round(T3.wel_list_add_pv/T2.welfare_list_pv*100,2),'%') as `美购列表页-点击加车率`
,concat(round(T3.wel_list_buy_pv/T2.welfare_list_pv*100,2),'%') as `美购列表页-点击立即支付率`
,concat(round(T3.wel_list_msg_pv/T2.welfare_list_pv*100,2),'%') as `美购列表页-点击私信咨询率`
,T1.welfare_list_pv as `2级品类聚合页pv`
,T2.welfare_list_pv as `2级品类聚合页-转化pv`
,T3.wel_list_add_pv as `2级品类聚合页-转化点击加车pv`
,T3.wel_list_buy_pv as `2级品类聚合页-转化点击立即支付pv`
,T3.wel_list_msg_pv as `2级品类聚合页-转化点击私信咨询pv`
,concat(round(T2.welfare_list_pv/T1.welfare_list_pv*100,2),'%') as `2级品类聚合页-转化pv率`
,concat(round(T2.welfare_list_pv/T1.wel_pv*100,2),'%') as `2级品类聚合页-转化美购pv占比`
,concat(round(T3.wel_list_add_pv/T2.welfare_list_pv*100,2),'%') as `2级品类聚合页-点击加车率`
,concat(round(T3.wel_list_buy_pv/T2.welfare_list_pv*100,2),'%') as `2级品类聚合页-点击立即支付率`
,concat(round(T3.wel_list_msg_pv/T2.welfare_list_pv*100,2),'%') as `2级品类聚合页-点击私信咨询率`
,T1.my_cart_pv as`购物车pv`
,T2.my_cart_pv as`购物车-转化pv`
,T3.cart_add_pv as `购物车-转化点击加车pv`
......@@ -177,7 +177,7 @@ FROM
SELECT t1.partition_date
,t1.device_type
,t1.active_type
,t2.channel
,t1.channel
,count(case when page_name = 'welfare_detail' then t1.cl_id end) as wel_pv
,count(case when page_name = 'search_result_more' then t1.cl_id end) as search_result_more_pv
,count(case when page_name = 'search_result_welfare' then t1.cl_id end) as search_result_welfare_pv
......@@ -205,42 +205,52 @@ FROM
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='page_view'
and params['is_first']=1
)t1
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
......@@ -253,15 +263,14 @@ FROM
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T1
LEFT JOIN
(--不同referrer下的美购详情页pv/uv
SELECT t1.partition_date
,t1.device_type
,t1.active_type
,t2.channel
,t1.channel
,count(case when referrer = 'welfare_detail' then t1.cl_id end) as wel_pv
,count(case when referrer = 'search_result_more' then t1.cl_id end) as search_result_more_pv
,count(case when referrer = 'search_result_welfare' then t1.cl_id end) as search_result_welfare_pv
......@@ -289,42 +298,52 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
and params['is_first']=1
)t1
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
......@@ -337,8 +356,7 @@ LEFT JOIN
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T2
ON T1.partition_date=T2.partition_date
and T1.device_type=T2.device_type
......@@ -349,7 +367,7 @@ LEFT JOIN
SELECT t1.partition_date
,t1.device_type
,t1.active_type
,t2.channel
,t1.channel
,count(case when referrer = 'welfare_detail' and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as wel_add_pv
,count(case when referrer = 'welfare_detail' and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as wel_buy_pv
,count(case when referrer = 'welfare_detail' and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as wel_msg_pv
......@@ -444,39 +462,48 @@ LEFT JOIN
)t1
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
......@@ -489,8 +516,7 @@ LEFT JOIN
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T3
ON T1.partition_date=T3.partition_date
and T1.device_type=T3.device_type
......
pm/meigou_detail_page/report/meigou_detail_page_source_uv.sql
View file @ d33f53e6
......@@ -25,16 +25,16 @@ SELECT T1.partition_date as `日期`
,concat(round(T3.org_add_uv/T2.organization_detail_uv*100,2),'%') as `医院主页-点击加车率`
,concat(round(T3.org_buy_uv/T2.organization_detail_uv*100,2),'%') as `医院主页-点击立即支付率`
,concat(round(T3.org_msg_uv/T2.organization_detail_uv*100,2),'%') as `医院主页-点击私信咨询率`
,T1.category_uv as `品类聚合页uv`
,T2.category_uv as `品类聚合页-转化uv`
,T3.category_add_uv as `品类聚合页-转化点击加车uv`
,T3.category_buy_uv as `品类聚合页-转化点击立即支付uv`
,T3.category_msg_uv as `品类聚合页-转化点击私信咨询uv`
,concat(round(T2.category_uv/T1.category_uv*100,2),'%') as `品类聚合页-转化uv率`
,concat(round(T2.category_uv/T1.wel_uv*100,2),'%') as `品类聚合页-转化美购uv占比`
,concat(round(T3.category_add_uv/T2.category_uv*100,2),'%') as `品类聚合页-点击加车率`
,concat(round(T3.category_buy_uv/T2.category_uv*100,2),'%') as `品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_uv/T2.category_uv*100,2),'%') as `品类聚合页-点击私信咨询率`
,T1.category_uv as `1级品类聚合页uv`
,T2.category_uv as `1级品类聚合页-转化uv`
,T3.category_add_uv as `1级品类聚合页-转化点击加车uv`
,T3.category_buy_uv as `1级品类聚合页-转化点击立即支付uv`
,T3.category_msg_uv as `1级品类聚合页-转化点击私信咨询uv`
,concat(round(T2.category_uv/T1.category_uv*100,2),'%') as `1级品类聚合页-转化uv率`
,concat(round(T2.category_uv/T1.wel_uv*100,2),'%') as `1级品类聚合页-转化美购uv占比`
,concat(round(T3.category_add_uv/T2.category_uv*100,2),'%') as `1级品类聚合页-点击加车率`
,concat(round(T3.category_buy_uv/T2.category_uv*100,2),'%') as `1级品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_uv/T2.category_uv*100,2),'%') as `1级品类聚合页-点击私信咨询率`
,T1.conversation_detail_uv as `私信详情页uv`
,T2.conversation_detail_uv as `私信详情页-转化uv`
,T3.conv_add_uv as `私信详情页-转化点击加车uv`
......@@ -45,16 +45,16 @@ SELECT T1.partition_date as `日期`
,concat(round(T3.conv_add_uv/T2.conversation_detail_uv*100,2),'%') as `私信详情页-点击加车率`
,concat(round(T3.conv_buy_uv/T2.conversation_detail_uv*100,2),'%') as `私信详情页-点击立即支付率`
,concat(round(T3.conv_msg_uv/T2.conversation_detail_uv*100,2),'%') as `私信详情页-点击私信咨询率`
,T1.welfare_list_uv as `美购列表页uv`
,T2.welfare_list_uv as `美购列表页-转化uv`
,T3.wel_list_add_uv as `美购列表页-转化点击加车uv`
,T3.wel_list_buy_uv as `美购列表页-转化点击立即支付uv`
,T3.wel_list_msg_uv as `美购列表页-转化点击私信咨询uv`
,concat(round(T2.welfare_list_uv/T1.welfare_list_uv*100,2),'%') as `美购列表页-转化uv率`
,concat(round(T2.welfare_list_uv/T1.wel_uv*100,2),'%') as `美购列表页-转化美购uv占比`
,concat(round(T3.wel_list_add_uv/T2.welfare_list_uv*100,2),'%') as `美购列表页-点击加车率`
,concat(round(T3.wel_list_buy_uv/T2.welfare_list_uv*100,2),'%') as `美购列表页-点击立即支付率`
,concat(round(T3.wel_list_msg_uv/T2.welfare_list_uv*100,2),'%') as `美购列表页-点击私信咨询率`
,T1.welfare_list_uv as `2级品类聚合页uv`
,T2.welfare_list_uv as `2级品类聚合页-转化uv`
,T3.wel_list_add_uv as `2级品类聚合页-转化点击加车uv`
,T3.wel_list_buy_uv as `2级品类聚合页-转化点击立即支付uv`
,T3.wel_list_msg_uv as `2级品类聚合页-转化点击私信咨询uv`
,concat(round(T2.welfare_list_uv/T1.welfare_list_uv*100,2),'%') as `2级品类聚合页-转化uv率`
,concat(round(T2.welfare_list_uv/T1.wel_uv*100,2),'%') as `2级品类聚合页-转化美购uv占比`
,concat(round(T3.wel_list_add_uv/T2.welfare_list_uv*100,2),'%') as `2级品类聚合页-点击加车率`
,concat(round(T3.wel_list_buy_uv/T2.welfare_list_uv*100,2),'%') as `2级品类聚合页-点击立即支付率`
,concat(round(T3.wel_list_msg_uv/T2.welfare_list_uv*100,2),'%') as `2级品类聚合页-点击私信咨询率`
,T1.my_cart_uv as`购物车uv`
,T2.my_cart_uv as`购物车-转化uv`
,T3.cart_add_uv as `购物车-转化点击加车uv`
......@@ -171,7 +171,7 @@ FROM
SELECT t1.partition_date
,t1.device_type
,t1.active_type
,t2.channel
,t1.channel
,count(distinct case when page_name = 'welfare_detail' then t1.cl_id end) as wel_uv
,count(distinct case when page_name = 'search_result_more' then t1.cl_id end) as search_result_more_uv
,count(distinct case when page_name = 'search_result_welfare' then t1.cl_id end) as search_result_welfare_uv
......@@ -199,42 +199,52 @@ FROM
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='page_view'
and params['is_first']=1
)t1
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
......@@ -247,15 +257,14 @@ FROM
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T1
LEFT JOIN
(--不同referrer下的美购详情页uv/uv
SELECT t1.partition_date
,t1.device_type
,t1.active_type
,t2.channel
,t1.channel
,count(distinct case when referrer = 'welfare_detail' then t1.cl_id end) as wel_uv
,count(distinct case when referrer = 'search_result_more' then t1.cl_id end) as search_result_more_uv
,count(distinct case when referrer = 'search_result_welfare' then t1.cl_id end) as search_result_welfare_uv
......@@ -283,42 +292,52 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
and params['is_first']=1
)t1
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
......@@ -331,8 +350,7 @@ LEFT JOIN
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T2
ON T1.partition_date=T2.partition_date
and T1.device_type=T2.device_type
......@@ -343,7 +361,7 @@ LEFT JOIN
SELECT t1.partition_date
,t1.device_type
,t1.active_type
,t2.channel
,t1.channel
,count(distinct case when referrer = 'welfare_detail' and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as wel_add_uv
,count(distinct case when referrer = 'welfare_detail' and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as wel_buy_uv
,count(distinct case when referrer = 'welfare_detail' and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as wel_msg_uv
......@@ -438,39 +456,48 @@ LEFT JOIN
)t1
JOIN
(
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
select partition_date,device_type,device_id,active_type,t.channel
from
(
select partition_date,device_os_type as device_type,m.device_id
,case when active_type = '4' then '老活跃设备'
when active_type in ('1','2') then '新增设备' end as active_type
,array(case when (a.device_id is not null or b.device_id is not null
or first_channel_source_type like '%xinyouxingkong%'
or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%') then '渠道可疑'
when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
from online.ml_device_day_active_status m
LEFT JOIN
(select code,is_spam,is_ai_channel,partition_day
from DIM.DIM_AI_CHANNEL_ZP_NEW
where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
LEFT JOIN
(select distinct device_id
from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
LEFT JOIN
(select device_id,day_id
from pm.tl_pm_channel_d
where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by device_id,day_id)b
ON m.device_id = b.device_id AND m.partition_date = b.day_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)a
LATERAL VIEW explode(a.channel) t AS channel
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
......@@ -483,8 +510,7 @@ LEFT JOIN
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
)T3
ON T1.partition_date=T3.partition_date
and T1.device_type=T3.device_type
......
pm/search_related/etl/search_related.sql
View file @ d33f53e6
......@@ -33,7 +33,8 @@ FROM
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='新首页-大家都在看' then '新首页-大家都在看'
when action='旧首页-大家都在看' then '旧首页-大家都在看'
when action='热搜词' then '美购首页热搜词' else null end as search_entrance
,count(cl_id) as search_pv
,count(distinct cl_id) as search_uv
......@@ -64,7 +65,7 @@ FROM
union all
SELECT partition_date,cl_id,null as page_name,'大家都在看' as action, null as referrer_link
SELECT partition_date,cl_id,null as page_name,'新首页-大家都在看' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
......@@ -73,6 +74,15 @@ FROM
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date,cl_id,page_name,'旧首页-大家都在看' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date>='20200801' and partition_date <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'do_search'
and params['input_type']='everyone_watch'
and params['tab']='精选'
and page_name='home'
union all
SELECT partition_date,cl_id,page_name,'热搜词' as action, null as referrer_link
......@@ -104,7 +114,8 @@ FROM
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='新首页-大家都在看' then '新首页-大家都在看'
when action='旧首页-大家都在看' then '旧首页-大家都在看'
when action='热搜词' then '美购首页热搜词' else null end
having search_entrance is not null
)t
......
pm/sign_daily/en-cn.properties 0 → 100644
View file @ d33f53e6
sign_daily=签到用户质量日报
\ No newline at end of file
pm/sign_daily/etl/create_sign_daily.sql 0 → 100644
View file @ d33f53e6
--***************************************************************
--*脚本名称:
--*功能: 签到用户质量日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_sign_deepaction_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
user_type string comment '{"chs_name":"用户类型","description":"","etl":"","value":"","remark":""}',
active_num bigint comment '{"chs_name":"活跃设备数","description":"","etl":"","value":"","remark":""}',
wel_pv bigint comment '{"chs_name":"商品核心页浏览pv","description":"","etl":"","value":"","remark":""}',
wel_uv bigint comment '{"chs_name":"商品核心页浏览uv","description":"","etl":"","value":"","remark":""}',
clue_pv bigint comment '{"chs_name":"有效线索人次","description":"","etl":"","value":"","remark":""}',
clue_uv bigint comment '{"chs_name":"有效线索人次","description":"","etl":"","value":"","remark":""}'
)comment '签到用户质量日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/sign_daily/etl/sign_daily.sql 0 → 100644
View file @ d33f53e6
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--线索日报
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
INSERT OVERWRITE TABLE pm.tl_pm_sign_deepaction_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT mas.partition_date as day_id
,device_os_type
,case when c.device_id is null then '非签到用户'
when c.sign_date=c.first_sign_date then '首日签到用户'
when c.sign_date>c.first_sign_date then '非首日签到用户' end as user_type
,count(distinct mas.device_id) as active_num
,sum(wel_pv) as wel_pv
,count(distinct case when wel_pv <> 0 then a.cl_id end) as wel_uv
,sum(CASE WHEN b.device_id is NOT NULL THEN b.valid_pv END) as clue_pv
,count(distinct case when valid_pv <> 0 then b.device_id end) as clue_uv
FROM
(
SELECT partition_date
,device_os_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','') AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND active_type IN ('4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
LEFT JOIN
(
SELECT partition_date,cl_id,wel_pv
FROM
(
SELECT partition_date,cl_id,count(1) as wel_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND action = 'page_view'
AND page_name in ('welfare_detail','organization_detail','expert_detail')
GROUP BY partition_date,cl_id
)a
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d
where partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)a
on a.partition_date=mas.partition_date and a.cl_id=mas.device_id
LEFT JOIN
( --有效线索分日数据(按用户维度)
SELECT t2.device_id,date_day,count(1) as valid_pv
FROM
(
SELECT NVL(T1.USER_ID,T2.USER_ID) AS USER_ID,
NVL(T1.merchant_id,T2.merchant_id) AS merchant_id,
NVL(T1.DATE_MONTH,T2.CALL_MONTH) AS DATE_MONTH,
least(NVL(DATE_DAY,'99999999'),NVL(FIRST_CALL_DATE,'99999999')) AS DATE_DAY,
NVL(T1.CONSULT_NUM,0) AS CONSULT_NUM,
NVL(T1.PAY_NUM,0) AS PAY_NUM,
NVL(T2.call_num,0) AS CALL_NUM
FROM
(
SELECT
NVL(T3.USER_ID,T4.USER_ID) AS USER_ID,
NVL(T3.merchant_id,T4.merchant_id) AS merchant_id,
NVL(T3.CONSULT_MONTH,T4.PAY_MONTH) AS DATE_MONTH,
least(NVL(first_consult_date,'99999999'),NVL(FIRST_PAY_DATE,'99999999')) AS DATE_DAY,
NVL(T3.CONSULT_NUM,0) AS CONSULT_NUM,
NVL(T4.PAY_NUM,0) AS PAY_NUM
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS consult_month,min(partition_day) as first_consult_date,1 as CONSULT_NUM
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <=regexp_replace(date_sub(current_date(),1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)T3
FULL JOIN
(
SELECT
T.USER_ID,
T.merchant_id,
regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','') AS PAY_MONTH,
MIN(regexp_replace(SUBSTR(T.PAY_DATE,1,10),'-','')) AS FIRST_PAY_DATE,
1 AS PAY_NUM
FROM ONLINE.ML_TRADE_ORDER_DETAIL_DAY T
WHERE
T.PARTITION_DATE = regexp_replace(date_sub(current_date(),1),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','') >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','')<=regexp_replace(date_sub(current_date(),1),'-','')
AND T.is_pure_user='true'
GROUP BY T.USER_ID,T.merchant_id,regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','')
) T4
ON T3.USER_ID = T4.USER_ID AND T3.merchant_id = T4.merchant_id AND T3.CONSULT_MONTH = T4.PAY_MONTH
GROUP BY NVL(T3.USER_ID,T4.USER_ID),NVL(T3.merchant_id,T4.merchant_id),NVL(T3.CONSULT_MONTH,T4.PAY_MONTH),
least(NVL(first_consult_date,'99999999'),NVL(FIRST_PAY_DATE,'99999999')),NVL(T3.CONSULT_NUM,0),NVL(T4.PAY_NUM,0)
)T1
FULL JOIN
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as FIRST_CALL_DATE,1 AS call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
AND start_time<release_time--电话拨通
and start_time>call_time
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)T2
ON T1.USER_ID = T2.USER_ID AND T1.merchant_id = T2.merchant_id AND T1.DATE_MONTH = T2.CALL_MONTH
)t1
JOIN
(
SELECT t.user_id,partition_date,if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates t
WHERE t.partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and t.partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
)t2
on t1.user_id=t2.user_id and t1.date_day=t2.partition_date
GROUP BY t2.device_id,date_day
)b
ON b.date_day=mas.partition_date and b.device_id=mas.device_id
LEFT JOIN
(
SELECT device_id,sign_date,first_sign_date
FROM
(
SELECT user_id, REGEXP_REPLACE(substr(sign_time,1,10), '-', '') as sign_date
FROM online.tl_hdfs_api_user_sign_v2_view --20190601之后使用的用户签到表
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(sign_time,1,10),'-','') >='20190601'
and regexp_replace(substr(sign_time,1,10),'-','') <=regexp_replace(date_sub(current_date(),1),'-','')
and sign_type <> '2' --2表示补签
union all
SELECT user_id, REGEXP_REPLACE(substr(sign_time,1,10), '-', '') as sign_date
FROM online.tl_hdfs_api_sign_record_view --20190601之前使用的用户签到表
WHERE partition_date = '20190531'
and regexp_replace(substr(sign_time,1,10),'-','') >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and regexp_replace(substr(sign_time,1,10),'-','') <'20190601'
and sign_type <> '3' --3表示补签
)a
left join
(
SELECT user_id,REGEXP_REPLACE(min(sign_date), '-', '') as first_sign_date
FROM
(
SELECT user_id, substr(sign_time,1,10) as sign_date
FROM online.tl_hdfs_api_user_sign_v2_view --20190601之后使用的用户签到表
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
and regexp_replace(substr(sign_time,1,10),'-','') >='20190601'
and regexp_replace(substr(sign_time,1,10),'-','') <=regexp_replace(date_sub(current_date(),1),'-','')
and sign_type <> '2' --2表示补签
union all
SELECT user_id, substr(sign_time,1,10) as sign_date
FROM online.tl_hdfs_api_sign_record_view --20190601之前使用的用户签到表 ,缺少20190101-20190226以及201905整月的数据
WHERE partition_date = '20190531'
and regexp_replace(substr(sign_time,1,10),'-','') >=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and regexp_replace(substr(sign_time,1,10),'-','') <'20190601'
and sign_type <> '3' --3表示补签
)first
group by user_id
)a_first
on a.user_id=a_first.user_id
LEFT JOIN
(
SELECT user_id --医生账号
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(date_sub(current_date(),1),'-','')
)b
ON a.user_id = b.user_id
JOIN
(
select t.user_id,partition_date,
if(size(t.device_list) > 0, device_list [ 0 ], '') device_id
from online.ml_user_updates t
where t.partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
and t.partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
)c
on a.user_id=c.user_id and a.sign_date=c.partition_date
where b.user_id is null
group by device_id,sign_date,first_sign_date
)c
on c.device_id=mas.device_id and c.sign_date=mas.partition_date
group by mas.partition_date
,device_os_type
,case when c.device_id is null then '非签到用户'
when c.sign_date=c.first_sign_date then '首日签到用户'
when c.sign_date>c.first_sign_date then '非首日签到用户' end
order by 1,2,3
pm/sign_daily/job/step1_1.job 0 → 100644
View file @ d33f53e6
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/sign_daily/job/step1_10.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/sign_daily/job/step1_11.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/sign_daily/job/step1_12.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/sign_daily/job/step1_13.job 0 → 100644
View file @ d33f53e6
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/sign_daily/job/step1_2.job 0 → 100644
View file @ d33f53e6
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/sign_daily/job/step1_3.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/sign_daily/job/step1_4.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file
pm/sign_daily/job/step1_5.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_user_sign_v2_view
\ No newline at end of file
pm/sign_daily/job/step1_6.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file
pm/sign_daily/job/step1_7.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
pm/sign_daily/job/step1_8.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file
pm/sign_daily/job/step1_9.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/sign_daily/job/step2.job 0 → 100644
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
command=sh /home/bi/bi-report/lib/shell/hive sign_daily
\ No newline at end of file
pm/sign_daily/job/step3.job 0 → 100644
View file @ d33f53e6
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/sign_daily/zhaoyang@igengmei.com,zhangwen@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/sign_daily/report/sign_daily.sql 0 → 100644
View file @ d33f53e6
SELECT day_id as `日期`
,round((sum(CASE WHEN device_os_type = 'ios' AND user_type='非首日签到用户' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'ios' and user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'ios' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'ios' THEN active_num END)),2) as `单个iOS非首日签到贡献商品核心页PV/单个iOS老贡献商品核心页PV`
,round((sum(CASE WHEN device_os_type = 'ios' AND user_type='非首日签到用户' THEN clue_pv END)/sum(CASE WHEN device_os_type = 'ios' AND user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'ios' THEN clue_pv END)/sum(CASE WHEN device_os_type = 'ios' THEN active_num END)),2) as `单个iOS非首日签到贡献有效线索人次/单个iOS老贡献有效线索人次`
,round((sum(CASE WHEN device_os_type = 'android' AND user_type='非首日签到用户' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'android' and user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'android' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'android' THEN active_num END)),2) as `单个android非首日签到贡献商品核心页PV/单个android老贡献商品核心页PV`
,round((sum(CASE WHEN device_os_type = 'android' AND user_type='非首日签到用户' THEN clue_pv END)/sum(CASE WHEN device_os_type = 'android' AND user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'android' THEN clue_pv END)/sum(CASE WHEN device_os_type = 'android' THEN active_num END)),2) as `单个android非首日签到贡献有效线索人次/单个android老贡献有效线索人次`
FROM pm.tl_pm_sign_deepaction_d
where partition_day in ('20200901','202001001','20201101','20201201','20210101',regexp_replace(date_sub(current_date,1),'-',''))
group by day_id
order by `日期`
pm/wel_clue_by_channel/en-cn.properties 0 → 100644
View file @ d33f53e6
clue_by_channel=线索pv
welfare_by_channel=核心页pv
wel_clue_by_channel=分渠道3日及7日内核心页和线索pv
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_1.job 0 → 100644
View file @ d33f53e6
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_10.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_2.job 0 → 100644
View file @ d33f53e6
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_3.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_4.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_5.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_6.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_7.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_8.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file
pm/wel_clue_by_channel/job/step1_9.job 0 → 100644
View file @ d33f53e6
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file
pm/wel_clue_by_channel/job/step2.job 0 → 100644
View file @ d33f53e6
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10
command=curl -X GET http://localhost:8553/api/report/email/wel_clue_by_channel/huchunhe@igengmei.com,wangjun@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file
pm/wel_clue_by_channel/report/clue_by_channel.sql 0 → 100644
View file @ d33f53e6
SELECT
first_active_date as `日期`
,channel as `渠道`
,device_os_type as `平台`
,COUNT(DISTINCT device.device_id) AS `当天新活量`
,nvl(SUM(IF(valid.date_day=device.first_active_date, valid.clue_num, NULL)),0) AS `当天有效线索`
,nvl(SUM(IF(valid.date_day>=device.first_active_date AND valid.date_day<=date_add(device.first_active_date,2), valid.clue_num, NULL)),0) AS `3日内有效线索`
,nvl(SUM(IF(valid.date_day>=device.first_active_date AND valid.date_day<=date_add(device.first_active_date,6), valid.clue_num, NULL)),0) AS `7日内有效线索`
FROM
(
SELECT
first_active_date,device_id,device_os_type
,IF(ios_device.channel IS NOT NULL, ios_device.channel, first_channel_source_type) AS channel
FROM
(
SELECT
device_id,device_os_type
,first_channel_source_type
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS first_active_date
FROM online.ml_device_day_active_status
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND active_type in ('1','2')
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)device
LEFT JOIN
( SELECT channel,idfa
FROM
(
SELECT channel,idfa,row_number () over (PARTITION BY idfa ORDER BY active_time asc) rn--,substr(active_time,1,7) as new_month
FROM online.tl_hdfs_ios_idfa_tmp--设备表，区分App Store 明细渠道名字
)a
WHERE rn = 1
)ios_device
ON ios_device.idfa = device.device_id
GROUP BY first_active_date,device_id,device_os_type,IF(ios_device.channel IS NOT NULL, ios_device.channel, first_channel_source_type)
)device
LEFT JOIN
( --有效线索分日数据(按用户维度)
SELECT
t2.device_id,from_unixtime(unix_timestamp(t1.DATE_DAY,'yyyyMMdd'),'yyyy-MM-dd') DATE_DAY,count(1) AS clue_num
FROM
(
SELECT NVL(T1.USER_ID,T2.USER_ID) AS USER_ID,
NVL(T1.merchant_id,T2.merchant_id) AS merchant_id,
NVL(T1.DATE_MONTH,T2.CALL_MONTH) AS DATE_MONTH,
least(NVL(DATE_DAY,'99999999'),NVL(FIRST_CALL_DATE,'99999999')) AS DATE_DAY,
NVL(T1.CONSULT_NUM,0) AS CONSULT_NUM,
NVL(T1.PAY_NUM,0) AS PAY_NUM,
NVL(T2.call_num,0) AS CALL_NUM
FROM
(
SELECT
NVL(T3.USER_ID,T4.USER_ID) AS USER_ID,
NVL(T3.merchant_id,T4.merchant_id) AS merchant_id,
NVL(T3.CONSULT_MONTH,T4.PAY_MONTH) AS DATE_MONTH,
least(NVL(first_consult_date,'99999999'),NVL(FIRST_PAY_DATE,'99999999')) AS DATE_DAY,
NVL(T3.CONSULT_NUM,0) AS CONSULT_NUM,
NVL(T4.PAY_NUM,0) AS PAY_NUM
FROM
(
SELECT user_id,merchant_id,SUBSTR(partition_day,1,6) AS consult_month,min(partition_day) as first_consult_date,1 as CONSULT_NUM
FROM ml.ml_c_et_msg_conversation_dimen_inc_d --新仓库私信表
WHERE partition_day >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_day <= regexp_replace(date_sub(current_date,1),'-','')
and is_valid='true'
GROUP BY user_id,merchant_id,SUBSTR(partition_day,1,6)
)T3
FULL JOIN
(
SELECT
T.USER_ID,
T.merchant_id,
regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','') AS PAY_MONTH,
MIN(regexp_replace(SUBSTR(T.PAY_DATE,1,10),'-','')) AS FIRST_PAY_DATE,
1 AS PAY_NUM
FROM ONLINE.ML_TRADE_ORDER_DETAIL_DAY T
WHERE
T.PARTITION_DATE = regexp_replace(date_sub(current_date(),1),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','') >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND regexp_replace(SUBSTR(T.pay_date,1,10),'-','')<=regexp_replace(date_sub(current_date,1),'-','')
AND T.is_pure_user='true'
GROUP BY T.USER_ID,T.merchant_id,regexp_replace(SUBSTR(T.PAY_DATE,1,7),'-','')
) T4
ON T3.USER_ID = T4.USER_ID AND T3.merchant_id = T4.merchant_id AND T3.CONSULT_MONTH = T4.PAY_MONTH
GROUP BY NVL(T3.USER_ID,T4.USER_ID),NVL(T3.merchant_id,T4.merchant_id),NVL(T3.CONSULT_MONTH,T4.PAY_MONTH),
least(NVL(first_consult_date,'99999999'),NVL(FIRST_PAY_DATE,'99999999')),NVL(T3.CONSULT_NUM,0),NVL(T4.PAY_NUM,0)
)T1
FULL JOIN
(
SELECT user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','') AS CALL_MONTH,
MIN(partition_date) as FIRST_CALL_DATE,1 AS call_num
FROM
(
SELECT sub_id,REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '') as partition_date
FROM tl.tl_gm_sl_ali_virtual_phone_call_detail --通话记录表，call_type呼叫类型
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>=REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date,1),'-','')
AND start_time<release_time --通话时长大于30秒
)a
LEFT JOIN
(
SELECT id,sub_id
FROM tl.tl_gm_sl_ali_virtual_phone_binding --阿里记录表
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)b
ON a.sub_id = b.sub_id
LEFT JOIN
(
SELECT id,platform_binding_id
FROM tl.tl_gm_sl_virtual_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)c
ON b.id = c.platform_binding_id
LEFT JOIN
(
SELECT phone_binding_id,lead_task_id
FROM tl.tl_gm_sl_lead_task_phone_binding
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)d
ON c.id = d.phone_binding_id
JOIN
(
SELECT id,user_id,merchant_id
FROM tl.tl_gm_sl_lead_task --线索任务表（用户点击授权后记入该表）
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND source='2' --用户行为电话授权
)e
ON d.lead_task_id = e.id
GROUP BY user_id,merchant_id,regexp_replace(SUBSTR(a.partition_date,1,6),'-','')
)T2
ON T1.USER_ID = T2.USER_ID AND T1.merchant_id = T2.merchant_id AND T1.DATE_MONTH = T2.CALL_MONTH
)t1
JOIN
(
SELECT user_id,partition_date,if(size(device_list) > 0, device_list [ 0 ], '') device_id
FROM online.ml_user_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
)t2
on t1.user_id=t2.user_id and t1.DATE_DAY=t2.partition_date
GROUP BY t2.device_id,from_unixtime(unix_timestamp(t1.DATE_DAY,'yyyyMMdd'),'yyyy-MM-dd')
)valid
ON device.device_id = valid.device_id
GROUP BY
first_active_date,channel,device_os_type
\ No newline at end of file
pm/wel_clue_by_channel/report/welfare_by_channel.sql 0 → 100644
View file @ d33f53e6
SELECT
first_active_date as `日期`
,channel as `渠道`
,device_os_type as `平台`
,COUNT(DISTINCT device.device_id) AS `当天新活量`
,nvl(SUM(IF(pv.action_date=device.first_active_date, pv.wel_pv, NULL)),0) AS `当日核心页pv`
,nvl(SUM(IF(pv.action_date>=device.first_active_date AND pv.action_date<=date_add(device.first_active_date,2), pv.wel_pv, NULL)),0) AS `3日内核心页pv`
,nvl(SUM(IF(pv.action_date>=device.first_active_date AND pv.action_date<=date_add(device.first_active_date,6), pv.wel_pv, NULL)),0) AS `7日内核心页pv`
FROM
(
SELECT
first_active_date,device_id,device_os_type
,IF(ios_device.channel IS NOT NULL, ios_device.channel, first_channel_source_type) AS channel
FROM
(
SELECT
device_id,device_os_type
,first_channel_source_type
,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS first_active_date
FROM online.ml_device_day_active_status
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND active_type in ('1','2')
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)device
LEFT JOIN
( SELECT channel,idfa
FROM
(
SELECT channel,idfa,row_number () over (PARTITION BY idfa ORDER BY active_time asc) rn--,substr(active_time,1,7) as new_month
FROM online.tl_hdfs_ios_idfa_tmp--设备表，区分App Store 明细渠道名字
)a
WHERE rn = 1
)ios_device
ON ios_device.idfa = device.device_id
GROUP BY first_active_date,device_id,device_os_type,IF(ios_device.channel IS NOT NULL, ios_device.channel, first_channel_source_type)
)device
LEFT JOIN
(
SELECT action_date,cl_id,count(1) as wel_pv
FROM
(
SELECT concat_ws('-',substr(partition_date,0,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS action_date,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(trunc(date_sub(current_date(),1),'MM'),'-','')
AND partition_date <=regexp_replace(date_sub(current_date(),1),'-','')
AND page_name in ('welfare_detail','organization_detail','expert_detail')
AND action = 'page_view'
)a
LEFT JOIN
( -- 2.去掉疑似机构刷量的PV和UV
SELECT device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
)spam_pv
on a.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
GROUP BY action_date,cl_id
)pv
ON device.device_id = pv.cl_id
GROUP BY
first_active_date,channel,device_os_type
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment