Skip to content

B
bi-report
Commit 9b3e3517 authored by 赵建伟's avatar 赵建伟
Browse files
Options

Merge branch 'yindanlei' into 'master' 

Yindanlei

See merge request !34
parents aa806dfc 7d452b9e
Show whitespace changes
Inline Side-by-side
Showing with 271 additions and 651 deletions
pm/fangan_day/etl/fangan_day.sql
View file @ 9b3e3517
......@@ -182,7 +182,8 @@ LEFT JOIN
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>='20200701'
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
AND start_time+ interval 30 second <= release_time --通话时长大于30秒
--AND start_time+ interval 30 second <= release_time --通话时长大于30秒
AND unix_timestamp(substr(start_time,1,19))+30<=unix_timestamp(substr(release_time,1,19))
)a
LEFT JOIN
(
......@@ -298,7 +299,8 @@ LEFT JOIN
WHERE partition_day=regexp_replace(date_sub(current_date(),1),'-','')
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')>='20200701'
AND REGEXP_REPLACE(SUBSTR(call_time,0,10), '-', '')<=regexp_replace(date_sub(current_date(),1),'-','')
AND start_time+ interval 30 second <= release_time --通话时长大于30秒
--AND start_time+ interval 30 second <= release_time --通话时长大于30秒
AND unix_timestamp(substr(start_time,1,19))+30<=unix_timestamp(substr(release_time,1,19))
)a
LEFT JOIN
(
......
pm/search_daily_data/etl/create_search_daily_data.sql
View file @ 9b3e3517
......@@ -4,7 +4,7 @@
--*业务名称: pm
--*输入数据:
--*作者: yindanlei@igengmei.com
--*更新时间:2020-07-13
--*更新时间:2020-08-07
--***************************************************************
--设置全局变量&UDF
......@@ -17,37 +17,21 @@ USE pm;
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_d
(
day_id string comment '{"chs_name":"日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"平台","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
channel string comment '{"chs_name":"渠道类型","description":"","etl":"","value":"","remark":""}',
dau int comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
all_search_uv int comment '{"chs_name":"总完成搜索uv(含首页猜你喜欢、美购首页大家都在搜)","description":"","etl":"","value":"","remark":""}',
all_search_uv/dau double comment '{"chs_name":"总完成搜索uv/DAU","description":"","etl":"","value":"","remark":""}',
all_search_pv/all_search_uv double comment '{"chs_name":"总完成搜索pv/总完成搜索uv","description":"","etl":"","value":"","remark":""}',
search_result_more_all_click_pv/search_result_more_pv double comment '{"chs_name":"搜索综合页正向点击pv/搜索综合页面pv","description":"","etl":"","value":"","remark":""}',
search_result_more_card_click_pv/search_result_more_pv double comment '{"chs_name":"搜索结果综合页所有卡片点击pv(含AI)/搜索结果综合页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_welfare_card_click_pv/search_result_welfare_pv double comment '{"chs_name":"搜索结果美购页所有卡片点击pv/搜索结果美购页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_hospital_card_click_pv/search_result_hospital_pv double comment '{"chs_name":"搜索结果医院页所有卡片点击pv/搜索结果医院页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_doctor_card_click_pv/search_result_doctor_pv double comment '{"chs_name":"搜索结果医生页所有卡片点击pv/搜索结果医生页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_diary_card_click_pv/search_result_diary_pv double comment '{"chs_name":"搜索结果日记页所有卡片点击pv/搜索结果日记页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_post_card_click_pv/search_result_post_pv double comment '{"chs_name":"搜索结果帖子页所有卡片点击pv/搜索结果帖子页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_wiki_card_click_pv/search_result_wiki_pv double comment '{"chs_name":"搜索结果百科页所有卡片点击pv/搜索结果百科页浏览pv","description":"","etl":"","value":"","remark":""}',
all_search_click_pv int comment '{"chs_name":"点击搜索框pv(724及以上)","description":"","etl":"","value":"","remark":""}',
navbar_search_pv/all_search_click_pv double comment '{"chs_name":"通过搜索框完成搜索总pv/点击搜索框pv(724及以上)","description":"","etl":"","value":"","remark":""}',
sug_search_pv/sug_out_pv double comment '{"chs_name":"sug搜索pv/sug弹窗弹出pv(726及以上)","description":"","etl":"","value":"","remark":""}',
home_youlike_search_pv/home_you_like_grey_type_pv double comment '{"chs_name":"首页猜你喜欢点击pv/灰度设备首页浏览pv(723及以上5%灰度)","description":"","etl":"","value":"","remark":""}',
faxain_search_pv/search_home_pv double comment '{"chs_name":"搜索首页发现搜索pv/搜索首页浏览pv","description":"","etl":"","value":"","remark":""}',
lishi_search_pv/search_home_pv double comment '{"chs_name":"搜索首页历史搜索pv/搜索首页浏览pv","description":"","etl":"","value":"","remark":""}',
allsee_search_pv/search_home_pv double comment '{"chs_name":"搜索首页大家都在看搜索pv/搜索首页浏览pv","description":"","etl":"","value":"","remark":""}',
home_morenci_search_pv/home_search_click_pv double comment '{"chs_name":"首页默认词搜索pv/首页搜索框点击pv(727及以上)","description":"","etl":"","value":"","remark":""}',
welfare_home_morenci_search_pv/welfare_home_search_click_pv double comment '{"chs_name":"美购首页默认词搜索pv/美购首页搜索框点击pv(727及以上)","description":"","etl":"","value":"","remark":""}',
content_detail_search_pv/content_detail_search_grey_pv double comment '{"chs_name":"内容详情页点击搜索按钮完成默认词搜索pv/灰度设备内容详情页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_welfare_pv/all_search_result_pv double comment '{"chs_name":"搜索结果美购页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_hospital_pv/all_search_result_pv double comment '{"chs_name":"搜索结果医院页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_doctor_pv/all_search_result_pv double comment '{"chs_name":"搜索结果医生页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_diary_pv/all_search_result_pv double comment '{"chs_name":"搜索结果日记页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_post_pv/all_search_result_pv double comment '{"chs_name":"搜索结果帖子页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}',
search_result_wiki_pv/all_search_result_pv double comment '{"chs_name":"搜索结果百科页浏览pv/搜索结果(全部tab)页浏览pv","description":"","etl":"","value":"","remark":""}'
all_search_uv int comment '{"chs_name":"完成搜索uv","description":"","etl":"","value":"","remark":""}',
all_search_pv int comment '{"chs_name":"完成搜索pv","description":"","etl":"","value":"","remark":""}',
all_search_uv/dau double comment '{"chs_name":"完成搜索的用户比例","description":"","etl":"","value":"","remark":""}',
all_search_pv/all_search_uv double comment '{"chs_name":"搜索用户人均使用次数","description":"","etl":"","value":"","remark":""}',
all_search_pv/search_home_pv double comment '{"chs_name":"搜索成功率","description":"","etl":"","value":"","remark":""}',
sug_search_pv_726/sug_out_pv_726 double comment '{"chs_name":"sug使用率","description":"","etl":"","value":"","remark":""}',
referrer_search_hexin_pv/all_search_uv double comment '{"chs_name":"来源于搜索的核心页pv/搜索uv","description":"","etl":"","value":"","remark":""}',
referrer_search_neirong_pv/all_search_uv double comment '{"chs_name":"来源于搜索的内容页pv/搜索uv","description":"","etl":"","value":"","remark":""}',
search_hexin_two_pv/referrer_search_hexin_pv double comment '{"chs_name":"来源于搜索的核心页二跳率","description":"","etl":"","value":"","remark":""}',
search_neirong_two_pv/referrer_search_neirong_pv double comment '{"chs_name":"来源于搜索的内容页二跳率","description":"","etl":"","value":"","remark":""}',
referrer_search_neirong_pagestay double comment '{"chs_name":"来源于搜索的内容平均浏览时长(m)","description":"","etl":"","value":"","remark":""}'
)comment '搜索日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
......
pm/search_daily_data/etl/search_daily_data.sql
View file @ 9b3e3517
......@@ -14,163 +14,76 @@ CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_search_d PARTITION (PARTITION_DAY = ${partition_day})
--搜索日报-英赫
SELECT
concat(substr(partition_date,0,4),'-',substr(partition_date,5,2),'-',substr(partition_date,7,2)) as day_id
,device_os_type as device_os_type
,active_type as active_type
,channel as channel
,dau as dau
,all_search_uv as all_search_uv
partition_date as day_id
,device_os_type
,active_type
,channel
,dau
,all_search_uv
,all_search_pv
,if(dau <> 0 ,concat(cast((all_search_uv/dau)*100 as decimal(18,2)),'%') , '-') as all_search_uv/dau
,if(all_search_uv <> 0 ,concat(cast((all_search_pv/all_search_uv) as decimal(18,2)),'') , '-') as all_search_pv/all_search_uv
,if(search_result_more_pv <> 0 ,concat(cast(((search_result_more_some_click_pv + ai_more_card_click_pv)/search_result_more_pv)*100 as decimal(18,2)),'%') , '-') as search_result_more_all_click_pv/search_result_more_pv
,if(search_result_more_pv <> 0 ,concat(cast((ai_more_card_click_pv/search_result_more_pv)*100 as decimal(18,2)),'%') , '-') as search_result_more_card_click_pv/search_result_more_pv
,if(search_result_welfare_pv <> 0 ,concat(cast((welfare_card_click_pv/search_result_welfare_pv)*100 as decimal(18,2)),'%') , '-') as search_result_welfare_card_click_pv/search_result_welfare_pv
,if(search_result_hospital_pv <> 0 ,concat(cast((hospital_card_click_pv/search_result_hospital_pv)*100 as decimal(18,2)),'%') , '-') as search_result_hospital_card_click_pv/search_result_hospital_pv
,if(search_result_doctor_pv <> 0 ,concat(cast((doctor_card_click_pv/search_result_doctor_pv)*100 as decimal(18,2)),'%') , '-') as search_result_doctor_card_click_pv/search_result_doctor_pv
,if(search_result_diary_pv <> 0 ,concat(cast((diary_card_click_pv/search_result_diary_pv)*100 as decimal(18,2)),'%') , '-') as search_result_diary_card_click_pv/search_result_diary_pv
,if(search_result_post_pv <> 0 ,concat(cast((post_card_click_pv/search_result_post_pv)*100 as decimal(18,2)),'%') , '-') as search_result_post_card_click_pv/search_result_post_pv
,if(search_result_wiki_pv <> 0 ,concat(cast((wiki_card_click_pv/search_result_wiki_pv)*100 as decimal(18,2)),'%') , '-') as search_result_wiki_card_click_pv/search_result_wiki_pv
,all_search_click_pv_724 as all_search_click_pv
,if(all_search_click_pv_724 <> 0 ,concat(cast((navbar_search_pv_724/all_search_click_pv_724)*100 as decimal(18,2)),'%') , '-') as navbar_search_pv/all_search_click_pv
,if(sug_out_pv_726 <> 0 ,concat(cast((sug_search_pv_726/sug_out_pv_726)*100 as decimal(18,2)),'%') , '-') as sug_search_pv/sug_out_pv
,if(home_you_like_grey_type_pv_723 <> 0 ,concat(cast((home_youlike_search_pv/home_you_like_grey_type_pv_723)*100 as decimal(18,2)),'%') , '-') as home_youlike_search_pv/home_you_like_grey_type_pv
,if(search_home_pv <> 0 ,concat(cast((faxain_search_pv/search_home_pv)*100 as decimal(18,2)),'%') , '-') as faxain_search_pv/search_home_pv
,if(search_home_pv <> 0 ,concat(cast((lishi_search_pv/search_home_pv)*100 as decimal(18,2)),'%') , '-') as lishi_search_pv/search_home_pv
,if(search_home_pv <> 0 ,concat(cast((allsee_search_pv/search_home_pv)*100 as decimal(18,2)),'%') , '-') as allsee_search_pv/search_home_pv
,if(home_search_click_pv_727 <> 0 ,concat(cast((home_morenci_search_pv_727/home_search_click_pv_727)*100 as decimal(18,2)),'%') , '-') as home_morenci_search_pv/home_search_click_pv
,if(welfare_home_search_click_pv_727 <> 0 ,concat(cast((welfare_home_morenci_search_pv_727/welfare_home_search_click_pv_727)*100 as decimal(18,2)),'%') , '-') as welfare_home_morenci_search_pv/welfare_home_search_click_pv
,if(content_detail_search_grey_pv_725 <> 0 ,concat(cast((content_detail_search_pv/content_detail_search_grey_pv_725)*100 as decimal(18,2)),'%') , '-') as content_detail_search_pv/content_detail_search_grey_pv
,if(all_search_result_pv <> 0 ,concat(cast((search_result_welfare_pv/all_search_result_pv)*100 as decimal(18,2)),'%') , '-') as search_result_welfare_pv/all_search_result_pv
,if(all_search_result_pv <> 0 ,concat(cast((search_result_hospital_pv/all_search_result_pv)*100 as decimal(18,2)),'%') , '-') as search_result_hospital_pv/all_search_result_pv
,if(all_search_result_pv <> 0 ,concat(cast((search_result_doctor_pv/all_search_result_pv)*100 as decimal(18,2)),'%') , '-') as search_result_doctor_pv/all_search_result_pv
,if(all_search_result_pv <> 0 ,concat(cast((search_result_diary_pv/all_search_result_pv)*100 as decimal(18,2)),'%') , '-') as search_result_diary_pv/all_search_result_pv
,if(all_search_result_pv <> 0 ,concat(cast((search_result_post_pv/all_search_result_pv)*100 as decimal(18,2)),'%') , '-') as search_result_post_pv/all_search_result_pv
,if(all_search_result_pv <> 0 ,concat(cast((search_result_wiki_pv/all_search_result_pv)*100 as decimal(18,2)),'%') , '-') as search_result_wiki_pv/all_search_result_pv
,if(search_home_pv <> 0 ,concat(cast((all_search_pv/search_home_pv)*100 as decimal(18,2)),'%') , '-') as all_search_pv/search_home_pv
,if(sug_out_pv_726 <> 0 ,concat(cast((sug_search_pv_726/sug_out_pv_726)*100 as decimal(18,2)),'%') , '-') as sug_search_pv_726/sug_out_pv_726
,if(all_search_uv <> 0 ,concat(cast((referrer_search_hexin_pv/all_search_uv) as decimal(18,2)),'') , '-') as referrer_search_hexin_pv/all_search_uv
,if(all_search_uv <> 0 ,concat(cast((referrer_search_neirong_pv/all_search_uv) as decimal(18,2)),'') , '-') as referrer_search_neirong_pv/all_search_uv
,if(referrer_search_hexin_pv <> 0 ,concat(cast((search_hexin_two_pv/referrer_search_hexin_pv)*100 as decimal(18,2)),'%') , '-') as search_hexin_two_pv/referrer_search_hexin_pv
,if(referrer_search_neirong_pv <> 0 ,concat(cast((search_neirong_two_pv/referrer_search_neirong_pv)*100 as decimal(18,2)),'%') , '-') as search_neirong_two_pv/referrer_search_neirong_pv
,if(referrer_search_neirong_uv_1000 <> 0 ,concat(cast((referrer_search_neirong_pagestay/referrer_search_neirong_uv_1000/60) as decimal(18,2)),'') , '-') as referrer_search_neirong_pagestay
FROM
(
SELECT
coalesce(t1.partition_date,t2.partition_date,t3.partition_date,t4.partition_date,t5.partition_date,t6.partition_date,t7.partition_date) as partition_date
,coalesce(t1.active_type,t2.active_type,t3.active_type,t4.active_type,t5.active_type,t6.active_type,t7.active_type) as active_type
,coalesce(t1.device_os_type,t2.device_os_type,t3.device_os_type,t4.device_os_type,t5.device_os_type,t6.device_os_type,t7.device_os_type) as device_os_type
,coalesce(t1.channel,t2.channel,t3.channel,t4.channel,t5.channel,t6.channel,t7.channel) as channel
coalesce(t1.partition_date,t2.partition_date,t3.partition_date,t4.partition_date,t5.partition_date,t6.partition_date) as partition_date
,coalesce(t1.active_type,t2.active_type,t3.active_type,t4.active_type,t5.active_type,t6.active_type) as active_type
,coalesce(t1.device_os_type,t2.device_os_type,t3.device_os_type,t4.device_os_type,t5.device_os_type,t6.device_os_type) as device_os_type
,coalesce(t1.channel,t2.channel,t3.channel,t4.channel,t5.channel,t6.channel) as channel
,coalesce(t1.dau,0) as dau
,coalesce(t2.all_search_click_uv_724,0) as all_search_click_uv_724 --724及以上全部搜索框点击uv
,coalesce(t2.all_search_click_pv_724,0) as all_search_click_pv_724 --724及以上全部搜索框点击pv
,coalesce(t2.home_search_click_uv_727,0) as home_search_click_uv_727 --725及以上首页搜索框点击uv
,coalesce(t2.home_search_click_pv_727,0) as home_search_click_pv_727 --725及以上首页搜索框点击pv
,coalesce(t2.welfare_home_search_click_uv_727,0) as welfare_home_search_click_uv_727 --725及以上美购首页搜索框点击uv
,coalesce(t2.welfare_home_search_click_pv_727,0) as welfare_home_search_click_pv_727 --725及以上美购首页搜索框点击pv
,coalesce(t2.content_search_click_uv_724,0) as content_search_click_uv_724 --内容详情页搜索框点击uv(725上10%灰度)
,coalesce(t2.content_search_click_pv_724,0) as content_search_click_pv_724 --内容详情页搜索框点击pv(725上10%灰度)
,coalesce(t3.all_search_uv,0) as all_search_uv --全部搜索uv
,coalesce(t3.all_search_pv,0) as all_search_pv --全部搜索pv
,coalesce(t3.navbar_search_uv_724,0) as navbar_search_uv_724 --724及以上通过搜索框搜索uv
,coalesce(t3.navbar_search_pv_724,0) as navbar_search_pv_724 --724及以上通过搜索框搜索pv
,coalesce(t3.sug_search_uv,0) as sug_search_uv --sug搜索uv
,coalesce(t3.sug_search_pv,0) as sug_search_pv --sug搜索pv
,coalesce(t3.home_youlike_search_uv,0) as home_youlike_search_uv --首页猜你喜欢搜索uv(725上5%灰度)
,coalesce(t3.home_youlike_search_pv,0) as home_youlike_search_pv --首页猜你喜欢搜索pv(725上5%灰度)
,coalesce(t3.faxain_search_uv,0) as faxain_search_uv --发现搜索uv
,coalesce(t3.faxain_search_pv,0) as faxain_search_pv --发现搜索pv
,coalesce(t3.lishi_search_uv,0) as lishi_search_uv --历史搜索uv
,coalesce(t3.lishi_search_pv,0) as lishi_search_pv --历史搜索pv
,coalesce(t3.allsee_search_uv,0) as allsee_search_uv --大家都在看搜索uv
,coalesce(t3.allsee_search_pv,0) as allsee_search_pv --大家都在看搜索pv
,coalesce(t3.home_morenci_search_uv_727,0) as home_morenci_search_uv_727 --725及以上首页默认词搜索uv，725do_search加的page_name参数
,coalesce(t3.home_morenci_search_pv_727,0) as home_morenci_search_pv_727 --725及以上首页默认词搜索uv，725do_search加的page_name参数
,coalesce(t3.welfare_home_morenci_search_uv_727,0) as welfare_home_morenci_search_uv_727 --725及以上美购首页默认词搜索uv，725do_search加的page_name参数
,coalesce(t3.welfare_home_morenci_search_pv_727,0) as welfare_home_morenci_search_pv_727 --725及以上美购首页默认词搜索uv，725do_search加的page_name参数
,coalesce(t3.content_detail_search_uv,0) as content_detail_search_uv --内容详情页默认词搜索uv
,coalesce(t3.content_detail_search_pv,0) as content_detail_search_pv --内容详情页默认词搜索pv
,coalesce(t3.sug_search_uv_726,0) as sug_search_uv_726 --726及以上版本的sug搜索uv
,coalesce(t3.sug_search_pv_726,0) as sug_search_pv_726 --726及以上版本的sug搜索pv
,coalesce(t4.home_you_like_grey_type_uv_723,0) as home_you_like_grey_type_uv_723 --723及以上首页-猜你喜欢灰度设备的首页uv
,coalesce(t4.home_you_like_grey_type_pv_723,0) as home_you_like_grey_type_pv_723 --723及以上首页-猜你喜欢灰度设备的首页pv
,coalesce(t4.content_detail_search_grey_uv_725,0) as content_detail_search_grey_uv_725 --725及以上内容详情页-搜索框灰度设备的内容详情页uv
,coalesce(t4.content_detail_search_grey_pv_725,0) as content_detail_search_grey_pv_725 --725及以上内容详情页-搜索框灰度设备的内容详情页pv
,coalesce(t4.search_home_uv,0) as search_home_uv
,coalesce(t4.search_home_pv,0) as search_home_pv
,coalesce(t4.all_search_result_uv,0) as all_search_result_uv
,coalesce(t4.all_search_result_pv,0) as all_search_result_pv
,coalesce(t4.search_result_more_uv,0) as search_result_more_uv
,coalesce(t4.search_result_more_pv,0) as search_result_more_pv
,coalesce(t4.search_result_welfare_uv,0) as search_result_welfare_uv
,coalesce(t4.search_result_welfare_pv,0) as search_result_welfare_pv
,coalesce(t4.search_result_diary_uv,0) as search_result_diary_uv
,coalesce(t4.search_result_diary_pv,0) as search_result_diary_pv
,coalesce(t4.search_result_post_uv,0) as search_result_post_uv
,coalesce(t4.search_result_post_pv,0) as search_result_post_pv
,coalesce(t4.search_result_wiki_uv,0) as search_result_wiki_uv
,coalesce(t4.search_result_wiki_pv,0) as search_result_wiki_pv
,coalesce(t4.search_result_doctor_uv,0) as search_result_doctor_uv
,coalesce(t4.search_result_doctor_pv,0) as search_result_doctor_pv
,coalesce(t4.search_result_hospital_uv,0) as search_result_hospital_uv
,coalesce(t4.search_result_hospital_pv,0) as search_result_hospital_pv
,coalesce(t5.more_card_click_uv,0) as more_card_click_uv --搜索结果综合页卡片点击uv(不含AI)
,coalesce(t5.more_card_click_pv,0) as more_card_click_pv --搜索结果综合页卡片点击pv(不含AI)
,coalesce(t5.ai_more_card_click_uv,0) as ai_more_card_click_uv --搜索结果综合页卡片点击uv(含AI)
,coalesce(t5.ai_more_card_click_pv,0) as ai_more_card_click_pv --搜索结果综合页卡片点击pv(含AI)
,coalesce(t5.welfare_card_click_uv,0) as welfare_card_click_uv
,coalesce(t5.welfare_card_click_pv,0) as welfare_card_click_pv
,coalesce(t5.diary_card_click_uv,0) as diary_card_click_uv
,coalesce(t5.diary_card_click_pv,0) as diary_card_click_pv
,coalesce(t5.post_card_click_uv,0) as post_card_click_uv
,coalesce(t5.post_card_click_pv,0) as post_card_click_pv
,coalesce(t5.wiki_card_click_uv,0) as wiki_card_click_uv
,coalesce(t5.wiki_card_click_pv,0) as wiki_card_click_pv
,coalesce(t5.doctor_card_click_uv,0) as doctor_card_click_uv
,coalesce(t5.doctor_card_click_pv,0) as doctor_card_click_pv
,coalesce(t5.hospital_card_click_uv,0) as hospital_card_click_uv
,coalesce(t5.hospital_card_click_pv,0) as hospital_card_click_pv
,coalesce(t4.search_home_uv,0) as search_home_uv --搜索首页uv
,coalesce(t4.search_home_pv,0) as search_home_pv --搜索首页pv
,coalesce(t3.sug_search_uv_726,0) as sug_search_uv_726 --726以上sug搜索uv
,coalesce(t3.sug_search_pv_726,0) as sug_search_pv_726 --726以上sug搜索pv
,coalesce(t2.home_search_click_uv_724,0) as home_search_click_uv_724 --724及以上搜索首页搜索框点击uv
,coalesce(t2.home_search_click_pv_724,0) as home_search_click_pv_724 --724及以上搜索首页搜索框点击pv
,coalesce(t4.referrer_search_hexin_pv,0) as referrer_search_hexin_pv --来源于搜索的核心页pv
,coalesce(t4.referrer_search_neirong_pv,0) as referrer_search_neirong_pv --来源于搜索的内容页pv
,coalesce(t5.search_hexin_two_pv,0) as search_hexin_two_pv --来源于搜索的核心页二跳pv
,coalesce(t5.search_neirong_two_pv,0) as search_neirong_two_pv --来源于搜索的内容页二跳pv
,coalesce(t4.referrer_search_neirong_uv_1000,0) as referrer_search_neirong_uv_1000 --来源于搜索的内容页uv，限制停留时长
,coalesce(t4.referrer_search_neirong_pagestay,0) as referrer_search_neirong_pagestay --来源于搜索的内容页的总停留时长(s)
,coalesce(t6.sug_out_uv_726,0) as sug_out_uv_726
,coalesce(t6.sug_out_pv_726,0) as sug_out_pv_726
,coalesce(t7.search_result_more_some_click_pv,0) as search_result_more_some_click_pv
FROM
( --dau
SELECT mas.partition_date,t2.active_type,t2.device_os_type,t2.channel,count(DISTINCT mas.device_id) as dau
FROM
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
partition_date,m.device_id
,array(device_os_type ,'合计') as device_os_type
,array(case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END ,'合计') as active_type
,array(CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
,count(distinct m.device_id) as dau
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
GROUP BY partition_date,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end),device_os_type
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END)
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND active_type in ('1','2','4')
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
LATERAL VIEW explode(mas.device_os_type) t2 AS device_os_type
LATERAL VIEW explode(mas.active_type) t2 AS active_type
GROUP BY mas.partition_date,t2.active_type,t2.device_os_type,t2.channel
)t1
full JOIN
(
......@@ -180,68 +93,46 @@ FROM
,active_type
,device_os_type
,channel
,count(click.cl_id) as all_search_click_pv_724
,count(distinct click.cl_id) as all_search_click_uv_724
,count(case when page_name = 'home' AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) then click.cl_id else null end) as home_search_click_pv_727
,count(distinct case when page_name = 'home' AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) then click.cl_id else null end) as home_search_click_uv_727
,count(case when page_name = 'welfare_home' AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) then click.cl_id else null end) as welfare_home_search_click_pv_727
,count(distinct case when page_name = 'welfare_home' AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) then click.cl_id else null end) as welfare_home_search_click_uv_727
,count(case when page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect') then click.cl_id else NULL END) as content_search_click_pv_724
,count(distinct case when page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail',
'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect') then click.cl_id else null end) as content_search_click_uv_724
,count(click.cl_id) as home_search_click_pv_724
,count(distinct click.cl_id) as home_search_click_uv_724
FROM
(
SELECT cl_id,partition_date,action,params['page_name'] as page_name,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action = 'on_click_navbar_search'
AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 24)
AND page_name in ('search_home','search_home_more','search_home_welfare','search_home_diary','search_home_wiki','search_home_post','search_home_hospital','search_home_doctor')
)click
JOIN
(
SELECT partition_date,device_id,t2.active_type,t2.channel,t2.device_os_type
FROM
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,m.device_id
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
partition_date,m.device_id
,array(device_os_type ,'合计') as device_os_type
,array(case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END ,'合计') as active_type
,array(CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND active_type in ('1','2','4')
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
LATERAL VIEW explode(mas.device_os_type) t2 AS device_os_type
LATERAL VIEW explode(mas.active_type) t2 AS active_type
)dev_channel
on dev_channel.device_id = click.cl_id
AND dev_channel.partition_date = click.partition_date
LEFT JOIN
(
SELECT device_id as cl_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
)spam_pv
on click.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
GROUP BY click.partition_date,active_type,device_os_type,channel
)t2
on t2.partition_date = t1.partition_date
......@@ -258,47 +149,19 @@ FROM
,channel
,count(click.cl_id) as all_search_pv
,count(distinct click.cl_id) as all_search_uv
,count(case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 24)
AND input_type not in ('首页-猜你喜欢','美购首页-大家都在搜','详情页默认词','高亮词','related_search') then click.cl_id else NULL end) as navbar_search_pv_724
,count(distinct case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 24)
AND input_type not in ('首页-猜你喜欢','美购首页-大家都在搜','详情页默认词','高亮词','related_search') then click.cl_id else NULL end) as navbar_search_uv_724
,count(case when input_type in ('发现') then click.cl_id else null end) as faxain_search_pv
,count(distinct case when input_type in ('发现') then click.cl_id else null end) as faxain_search_uv
,count(case when input_type in ('历史') then click.cl_id else null end) as lishi_search_pv
,count(distinct case when input_type in ('历史') then click.cl_id else null end) as lishi_search_uv
,count(case when input_type in ('大家都在看') then click.cl_id else null end) as allsee_search_pv
,count(distinct case when input_type in ('大家都在看') then click.cl_id else null end) as allsee_search_uv
,count(case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) AND input_type = '默认词' AND referrer_link_last in ('home') then click.cl_id else null end) as home_morenci_search_pv_727
,count(distinct case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) AND input_type = '默认词' AND referrer_link_last in ('home') then click.cl_id else null end) as home_morenci_search_uv_727
,count(case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) AND input_type = '默认词' AND referrer_link_last in ('welfare_home') then click.cl_id else null end) as welfare_home_morenci_search_pv_727
,count(distinct case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) AND input_type = '默认词' AND referrer_link_last in ('welfare_home') then click.cl_id else null end) as welfare_home_morenci_search_uv_727
,count(case when input_type = '详情页默认词' then click.cl_id else null end) as content_detail_search_pv
,count(distinct case when input_type = '详情页默认词' then click.cl_id else null end) as content_detail_search_uv
,count(case when input_type = '输入' then click.cl_id else null end) as shuru_search_pv
,count(distinct case when input_type = '输入' then click.cl_id else null end) as shuru_search_uv
,count(case when input_type = '默认词' then click.cl_id else null end) as morenci_search_pv
,count(distinct case when input_type = '默认词' then click.cl_id else null end) as morenci_search_uv
,count(case when input_type in ('联想','聚合模块') then click.cl_id else null end) as sug_search_pv
,count(distinct case when input_type in ('联想','聚合模块') then click.cl_id else null end) as sug_search_uv
,count(case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 26) AND input_type in ('联想','聚合模块') then click.cl_id else null end) as sug_search_pv_726
,count(distinct case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 26) AND input_type in ('联想','聚合模块') then click.cl_id else null end) as sug_search_uv_726
,count(case when input_type in ('首页-猜你喜欢') then click.cl_id else null end) as home_youlike_search_pv
,count(distinct case when input_type in ('首页-猜你喜欢') then click.cl_id else null end) as home_youlike_search_uv
,count(case when int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 26 AND input_type in ('联想','聚合模块') then click.cl_id else null end) as sug_search_pv_726
,count(distinct case when int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 26 AND input_type in ('联想','聚合模块') then click.cl_id else null end) as sug_search_uv_726
FROM
(
SELECT cl_id,partition_date,action,params['page_name'] as page_name,params['input_type'] as input_type,app_version
,(json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]) as referrer_link_last
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action in ('do_search','search_result_click_search')
union all
SELECT cl_id,partition_date,action,'home' as page_name,'首页-猜你喜欢' as input_type,app_version,'其它' as referrer_link_last
SELECT cl_id,partition_date,action,'home' as page_name,'首页-猜你喜欢' as input_type,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
......@@ -306,67 +169,50 @@ FROM
--AND page_name='home' android的page_name为空
union all
SELECT cl_id,partition_date,action,params['page_name'] as page_name,'美购首页-大家都在搜' as input_type,app_version,'其它' as referrer_link_last
SELECT cl_id,partition_date,action,params['page_name'] as page_name,'美购首页-大家都在搜' as input_type,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date() ,'-','')
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action = 'on_click_card'
AND params['page_name']='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
SELECT cl_id,partition_date,action,params['page_name'] as page_name,'高亮词' as input_type,app_version,'其它' as referrer_link_last
SELECT cl_id,partition_date,action,params['page_name'] as page_name,'高亮词' as input_type,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date() ,'-','')
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action = 'on_click_card'
AND params['card_type'] ='highlight_word'
)click
JOIN
(
SELECT partition_date,device_id,t2.active_type,t2.channel,t2.device_os_type
FROM
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,m.device_id
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
partition_date,m.device_id
,array(device_os_type ,'合计') as device_os_type
,array(case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END ,'合计') as active_type
,array(CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND active_type in ('1','2','4')
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
LATERAL VIEW explode(mas.device_os_type) t2 AS device_os_type
LATERAL VIEW explode(mas.active_type) t2 AS active_type
)dev_channel
on dev_channel.device_id = click.cl_id
AND dev_channel.partition_date = click.partition_date
LEFT JOIN
(
SELECT device_id as cl_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
)spam_pv
on click.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
GROUP BY click.partition_date,active_type,device_os_type,channel
)t3
on t3.partition_date = t1.partition_date
......@@ -381,95 +227,62 @@ FROM
,active_type
,device_os_type
,channel
,count(case when home_you_like_grey_type = '灰度' AND page_name = 'home' AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 23) then page.cl_id else NULL end) as home_you_like_grey_type_pv_723
,count(distinct case when home_you_like_grey_type = '灰度' AND page_name = 'home' AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 23) then page.cl_id else NULL end) as home_you_like_grey_type_uv_723
,count(case when page_name in ('search_home','search_home_more','search_home_welfare','search_home_diary','search_home_wiki','search_home_post','search_home_hospital','search_home_doctor') then page.cl_id else NULL end) as search_home_pv
,count(distinct case when page_name in ('search_home','search_home_more','search_home_welfare','search_home_diary','search_home_wiki','search_home_post','search_home_hospital','search_home_doctor') then page.cl_id else NULL end) as search_home_uv
,count(case when page_name in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,count(CASE when referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki') and page_name in ('welfare_detail','organization_detail','expert_detail') THEN page.cl_id else NULL END) as referrer_search_hexin_pv
,count(CASE when referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki') then page.cl_id else NULL end) as all_search_result_pv
,count(distinct case when page_name in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_wiki') and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail'
,'question_answer_detail','article_detail') THEN page.cl_id else NULL END) as referrer_search_neirong_pv
,count(DISTINCT CASE WHEN referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki') then page.cl_id else NULL end) as all_search_result_uv
,count(case when ((content_search_grey_type = '灰度' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail'
,'video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect'))
or (answer_grey_type = '灰度' AND page_name in ('answer_detail','question_answer_detail')))
AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 25) then page.cl_id else NULL end) as content_detail_search_grey_pv_725
,count(distinct case when content_search_grey_type = '灰度' AND page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail',
'question_answer_detail','video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect')
AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 25) then page.cl_id else NULL end) as content_detail_search_grey_uv_725
,count(case when page_name in ('search_result_more') then page.cl_id else NULL end) as search_result_more_pv
,count(distinct case when page_name in ('search_result_more') then page.cl_id else NULL end) as search_result_more_uv
,count(case when page_name in ('search_result_welfare') then page.cl_id else NULL end) as search_result_welfare_pv
,count(distinct case when page_name in ('search_result_welfare') then page.cl_id else NULL end) as search_result_welfare_uv
,count(case when page_name in ('search_result_diary') then page.cl_id else NULL end) as search_result_diary_pv
,count(distinct case when page_name in ('search_result_diary') then page.cl_id else NULL end) as search_result_diary_uv
,count(case when page_name in ('search_result_post') then page.cl_id else NULL end) as search_result_post_pv
,count(distinct case when page_name in ('search_result_post') then page.cl_id else NULL end) as search_result_post_uv
,count(case when page_name in ('search_result_wiki') then page.cl_id else NULL end) as search_result_wiki_pv
,count(distinct case when page_name in ('search_result_wiki') then page.cl_id else NULL end) as search_result_wiki_uv
,count(case when page_name in ('search_result_doctor') then page.cl_id else NULL end) as search_result_doctor_pv
,count(distinct case when page_name in ('search_result_doctor') then page.cl_id else NULL end) as search_result_doctor_uv
,count(case when page_name in ('search_result_hospital') then page.cl_id else NULL end) as search_result_hospital_pv
,count(distinct case when page_name in ('search_result_hospital') then page.cl_id else NULL end) as search_result_hospital_uv
,'search_result_wiki') and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail'
,'question_answer_detail','article_detail') and page_stay >= '0' and page_stay < '1000' THEN page.cl_id else NULL END) as referrer_search_neirong_uv_1000
,sum(CASE WHEN referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki') and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail'
,'question_answer_detail','article_detail') and page_stay >= '0' and page_stay < '1000' THEN page.page_stay else NULL END) as referrer_search_neirong_pagestay
FROM
(
SELECT cl_id,partition_date,referrer,params['page_name'] as page_name
,(CASE WHEN substr(convup(sha1(cl_id),16,10),-2,2)%20=0 then '灰度' else '非灰' end) as home_you_like_grey_type,app_version
,(case when (partition_date <= '20200511' AND substr(convup(sha1(cl_id),16,10),-1) = '0') or (partition_date >= '20200512' AND partition_date <= '20200706' AND substr(convup(sha1(cl_id),16,10),-1) in ('0','1','2','3','4') or partition_date >= '20200707') then '灰度' else '非灰' end) as content_search_grey_type
,(case when (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 27) or (substr(convup(sha1(cl_id),16,10),-2,2)%20=0) then '灰度' else '非灰' END) as answer_grey_type
SELECT cl_id,partition_date,page_name,params['referrer'] as referrer,page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action='page_view'
AND params['page_name'] in ('search_home','search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare','search_result_wiki'
,'search_home_more','search_home_welfare','search_home_diary','search_home_wiki','search_home_post','search_home_hospital','search_home_doctor'
,'home','diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail',
'question_answer_detail','video_steep','article_detail','wiki_detail','product_detail','wiki_brand','wiki_collect')
AND page_name in ('search_home','search_home_more','search_home_welfare','search_home_diary','search_home_wiki','search_home_post','search_home_hospital','search_home_doctor'
,'diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail'
,'question_answer_detail','article_detail','welfare_detail','organization_detail','expert_detail')
)page
JOIN
(
SELECT partition_date,device_id,t2.active_type,t2.channel,t2.device_os_type
FROM
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,m.device_id
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
partition_date,m.device_id
,array(device_os_type ,'合计') as device_os_type
,array(case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END ,'合计') as active_type
,array(CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND active_type in ('1','2','4')
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
LATERAL VIEW explode(mas.device_os_type) t2 AS device_os_type
LATERAL VIEW explode(mas.active_type) t2 AS active_type
)dev_channel
on dev_channel.device_id = page.cl_id
AND dev_channel.partition_date = page.partition_date
LEFT JOIN
(
SELECT device_id as cl_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
)spam_pv
on page.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
GROUP BY page.partition_date,active_type,device_os_type,channel
)t4
on t4.partition_date = t1.partition_date
......@@ -477,144 +290,95 @@ FROM
AND t4.device_os_type = t1.device_os_type
AND t4.channel = t1.channel
full JOIN
( --搜索结果卡片点击事件
( --搜索结果页的二跳率
SELECT
card_click.partition_date as partition_date
,active_type
,device_os_type
,channel
,count(case when page_name = 'search_result_more' then card_click.cl_id else null end) as ai_more_card_click_pv
,count(distinct case when page_name = 'search_result_more' then card_click.cl_id else null end) as ai_more_card_click_uv
,count(case when page_name = 'search_result_more' AND action <> 'search_result_more_click_face' then card_click.cl_id else null end) as more_card_click_pv
,count(distinct case when page_name = 'search_result_more' AND action <> 'search_result_more_click_face' then card_click.cl_id else null end) as more_card_click_uv
,count(case when page_name = 'search_result_welfare' then card_click.cl_id else null end) as welfare_card_click_pv
,count(distinct case when page_name = 'search_result_welfare' then card_click.cl_id else null end) as welfare_card_click_uv
,count(case when page_name = 'search_result_diary' then card_click.cl_id else null end) as diary_card_click_pv
,count(distinct case when page_name = 'search_result_diary' then card_click.cl_id else null end) as diary_card_click_uv
,count(case when page_name = 'search_result_post' then card_click.cl_id else null end) as post_card_click_pv
,count(distinct case when page_name = 'search_result_post' then card_click.cl_id else null end) as post_card_click_uv
,count(case when page_name = 'search_result_wiki' then card_click.cl_id else null end) as wiki_card_click_pv
,count(distinct case when page_name = 'search_result_wiki' then card_click.cl_id else null end) as wiki_card_click_uv
,count(case when page_name = 'search_result_doctor' then card_click.cl_id else null end) as doctor_card_click_pv
,count(distinct case when page_name = 'search_result_doctor' then card_click.cl_id else null end) as doctor_card_click_uv
,count(case when page_name = 'search_result_hospital' then card_click.cl_id else null end) as hospital_card_click_pv
,count(distinct case when page_name = 'search_result_hospital' then card_click.cl_id else null end) as hospital_card_click_uv
,count(case when type = 'hexin' then card_click.cl_id else null end) as search_hexin_two_pv
,count(case when type = 'neirong' then card_click.cl_id else null end) as search_neirong_two_pv
FROM
(
SELECT
T.cl_id AS cl_id,
T.action AS action,
T.partition_date as partition_date,
(CASE
WHEN T.action = 'search_result_welfare_click_item' OR (T.action = 'goto_welfare_detail' AND T.params [ 'from' ] = 'search_result_welfare_recommend')
THEN 'search_result_welfare' --ok
WHEN T.action IN ('search_result_wiki_click_item','search_result_wiki_click_recommend_wiki')
THEN 'search_result_wiki'
WHEN action IN ('on_click_doctor_card','search_result_click_recommend_item')
OR (action = 'on_click_hospital_card' AND params [ 'page_name' ] = 'search_result_more')
OR (action = 'search_result_click_search' AND params [ 'tab' ] = '综合' and params [ 'input_type' ] = 'related_search')
THEN 'search_result_more'
WHEN action = 'on_click_diary_card' AND params [ 'page_name' ] = 'search_result_diary'
THEN 'search_result_diary'
WHEN action = 'search_result_hospital_click_item'
THEN 'search_result_hospital'
WHEN action = 'search_result_doctor_click_item'
THEN 'search_result_doctor'
WHEN action = 'on_click_topic_card' AND params [ 'page_name' ] = 'search_result_post'
THEN 'search_result_post'
ELSE params [ 'page_name' ] END) AS page_name,
T.app_version AS app_version,
T.params['position'] AS position,
CASE
WHEN T.action IN ('search_result_click_recommend_item','search_result_welfare_click_item')
OR (action = 'goto_welfare_detail' and params [ 'from' ] = 'search_result_welfare_recommend')
THEN 'service'
WHEN (T.action = 'search_result_click_infomation_item' AND coalesce(T.params [ 'business_type' ],T.params['post_type']) = '0')
OR (T.action = 'on_click_diary_card' AND T.params [ 'page_name' ] = 'search_result_more' AND T.cl_type = 'android')
OR (T.action IN ('on_click_diary_card','search_result_click_infomation_item') AND T.params [ 'page_name' ] = 'search_result_diary')
THEN 'diary'
WHEN T.action = 'search_result_click_infomation_item' AND coalesce(T.params [ 'business_type' ],T.params['post_type']) = '5'
THEN 'answer'
WHEN T.action = 'search_result_click_infomation_item' AND coalesce(T.params [ 'business_type' ],T.params['post_type']) = '9'
THEN 'article'
WHEN T.action = 'search_result_click_infomation_item' AND coalesce(T.params [ 'business_type' ],T.params['post_type']) = '11'
THEN 'user_post'
WHEN T.action IN ('search_result_wiki_click_item','search_result_wiki_click_recommend_wiki')
THEN 'wiki'
WHEN T.action = 'search_result_hospital_click_item'
OR (T.action = 'on_click_hospital_card' AND T.params [ 'page_name' ] = 'search_result_more')
THEN 'hospital'
WHEN (T.action = 'on_click_doctor_card' AND T.params [ 'page_name' ] = 'search_result_more')
OR (T.action = 'search_result_doctor_click_item')
THEN 'doctor'
WHEN T.action = 'on_click_topic_card' AND T.params [ 'page_name' ] IN ('search_result_post','search_result_more') AND T.params [ 'card_content_type' ] = 'user_post'
THEN 'user_post'
WHEN T.action = 'on_click_topic_card' AND T.params [ 'page_name' ] IN ('search_result_post','search_result_more') AND T.params [ 'card_content_type' ] = 'doctor_post'
THEN 'doctor_post'
WHEN (T.action = 'search_result_click_search' AND params [ 'tab' ] = '综合' and params [ 'input_type' ] = 'related_search')
THEN 'related_search'
ELSE coalesce(T.params [ 'business_type' ],T.params['post_type']) END AS card_content_type
FROM online.bl_hdfs_maidian_updates T
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND ((T.action IN ('on_click_doctor_card', 'on_click_hospital_card') AND params [ 'page_name' ] = 'search_result_more')
OR (T.action='search_result_click_search' AND params [ 'tab' ] = '综合' and params['input_type']='related_search')
OR T.action in ('search_result_welfare_click_item', 'search_result_wiki_click_item','search_result_click_recommend_item','goto_welfare_detail','on_click_diary_card',
'search_result_hospital_click_item', 'search_result_doctor_click_item','search_result_click_infomation_item','search_result_wiki_click_recommend_wiki')
OR (T.action = 'on_click_topic_card' AND T.params [ 'page_name' ] IN ('search_result_post','search_result_more'))
OR T.action = 'search_result_more_click_face')
AND SIZE(split(T.app_version,'\\.')) >= 3
AND (T.params [ 'page_name' ] LIKE '%search_result%' OR T.params [ 'page_name' ] IS NULL)
AND (case when T.action='search_result_click_infomation_item' and T.params['tab']='综合' then T.params['input_type']<>'related_search' else 1=1 end)
( --核心页二跳率
SELECT cl_id,partition_date,'hexin' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND (referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki')
or (params['referrer_link'] like '%[%' and json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1] in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare','search_result_wiki')))
AND ((action in ('welfare_multiattribute_click_add','welfare_multiattribute_click_buy') AND page_name = 'welfare_detail')
or action = 'welfare_detail_click_message'
or (action = 'on_click_button' AND page_name = 'welfare_detail' AND params['button_name'] in ('question_tag','appointment'))
or (action = 'welfare_detail_click_curearea_contact' AND params['connect_type'] in ('phone','onlineconsult')))
UNION ALL
SELECT cl_id,partition_date,'hexin' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','') --医院主页、医生主页点击咨询和浮层提问
AND action = 'on_click_button'
AND page_name in ('organization_detail','expert_detail')
AND (params['button_name'] in ('online_consult','phone_consult','put_question','discount_pay') OR params['button_name'] like "%？%")
AND params['referrer_link'] like '%[%'
and (referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki')
or (params['referrer_link'] like '%[%' and json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1] in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare','search_result_wiki')))
UNION ALL
--内容二跳率-点击搜索框、搜索按钮、高亮词、美购卡片、推荐内容卡片
SELECT cl_id,partition_date,'neirong' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail','article_detail')
AND (referrer in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare'
,'search_result_wiki')
or (params['referrer_link'] like '%[%' and json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1] in ('search_result_diary','search_result_doctor','search_result_hospital','search_result_more'
,'search_result_more_infomation','search_result_more_user','search_result_post','search_result_welfare','search_result_wiki')))
AND (action in ('on_click_navbar_search','do_search')
or (action='on_click_card' and params['card_type']='highlight_word')
or (action='on_click_card' and params['card_content_type'] in ('service','qa','diary','user_post','answer'))
or (action='on_click_button' and params['button_name']='unfold'))
)card_click
JOIN
(
SELECT partition_date,device_id,t2.active_type,t2.channel,t2.device_os_type
FROM
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,m.device_id
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
partition_date,m.device_id
,array(device_os_type ,'合计') as device_os_type
,array(case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END ,'合计') as active_type
,array(CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND active_type in ('1','2','4')
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
LATERAL VIEW explode(mas.device_os_type) t2 AS device_os_type
LATERAL VIEW explode(mas.active_type) t2 AS active_type
)dev_channel
on dev_channel.device_id = card_click.cl_id
AND dev_channel.partition_date = card_click.partition_date
LEFT JOIN
(
SELECT device_id as cl_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
)spam_pv
on card_click.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
GROUP BY card_click.partition_date,active_type,device_os_type,channel
)t5
on t5.partition_date = t1.partition_date
AND t5.active_type = t1.active_type
AND t5.device_os_type = t1.device_os_type
AND t5.channel = t1.channel
full JOIN
( --sug弹窗弹出--预计排期726修复
SELECT
......@@ -628,154 +392,47 @@ FROM
(
SELECT cl_id,partition_date,action,params['page_name'] as page_name
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND action = 'searchsug_view'
AND (int(split(app_version,'\\.')[0]) = 7 AND int(split(app_version,'\\.')[1]) >= 26)
)click
JOIN
(
SELECT partition_date,device_id,t2.active_type,t2.channel,t2.device_os_type
FROM
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,m.device_id
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
partition_date,m.device_id
,array(device_os_type ,'合计') as device_os_type
,array(case WHEN active_type = '4' THEN '老活'
WHEN active_type in ('1','2') then '新增' END ,'合计') as active_type
,array(CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
where partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
AND active_type in ('1','2','4')
) mas
LATERAL VIEW explode(mas.channel) t2 AS channel
LATERAL VIEW explode(mas.device_os_type) t2 AS device_os_type
LATERAL VIEW explode(mas.active_type) t2 AS active_type
)dev_channel
on dev_channel.device_id = click.cl_id
AND dev_channel.partition_date = click.partition_date
LEFT JOIN
(
SELECT device_id as cl_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
)spam_pv
on click.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
GROUP BY click.partition_date,active_type,device_os_type,channel
)t6
on t6.partition_date = t1.partition_date
AND t6.active_type = t1.active_type
AND t6.device_os_type = t1.device_os_type
AND t6.channel = t1.channel
full JOIN
(
--搜索结果综合页的点击,包括搜索框点击、tab切换、美购-查看更多按钮、百科-查看更多按钮、相关搜索
SELECT
page.partition_date as partition_date
,active_type
,device_os_type
,channel
,count(page.cl_id) as search_result_more_some_click_pv
,count(distinct page.cl_id) as search_result_more_some_click_uv
FROM
(
SELECT cl_id,partition_date,params['page_name'] as page_name,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND action='on_click_navbar_search'
AND params['page_name'] = 'search_result_more'
union all
SELECT cl_id,partition_date,params['page_name'] as page_name,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND action = 'search_result_more_click_more'
AND params['page_name'] = 'search_result_more'
AND params['business_type'] in ('service','wiki')
union all
SELECT cl_id,partition_date,'search_result_more' as page_name,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND action in ('do_search','search_result_click_search')
AND params['input_type'] = 'related_search'
)t
ORDER BY `日期`,`平台`,`活跃类型`,`渠道类型`
union all
SELECT cl_id,partition_date,'search_result_more' as page_name,app_version
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND action = 'search_result_click_tab'
AND params['referrer_tab_name'] = '综合'
)page
JOIN
(
SELECT
partition_date
,(case when active_type in (1,2) then '新增' when active_type = 4 then '老活' else NULL end) as active_type
,device_os_type
,m.device_id
,(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%jingmeng%' or first_channel_source_type like '%longyuzhixing%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and channel_ai.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI渠道' ELSE '非AI渠道' END) as channel
FROM online.ml_device_day_active_status m
LEFT JOIN (SELECT col1,col2 FROM pm.tl_pm_ydl WHERE col3='0204_danlei_channel') channel_ai
on first_channel_source_type = channel_ai.col1
LEFT JOIN (SELECT DISTINCT device_id FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),2),'-',''))a
ON m.device_id = a.device_id
WHERE partition_date >= REGEXP_REPLACE(date_sub(current_date(),2),'-','')
AND partition_date < regexp_replace(current_date ,'-','')
AND active_type in (1,2,4)
AND first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev_channel
on dev_channel.device_id = page.cl_id
AND dev_channel.partition_date = page.partition_date
LEFT JOIN
(
SELECT device_id as cl_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
)spam_pv
on page.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
GROUP BY page.partition_date,active_type,device_os_type,channel
)t7
on t7.partition_date = t1.partition_date
AND t7.active_type = t1.active_type
AND t7.device_os_type = t1.device_os_type
AND t7.channel = t1.channel
)t
ORDER BY `日期`,`平台`,`活跃类型`,`渠道类型`;
pm/search_daily_data/job/step1_2.job
View file @ 9b3e3517
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive al al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_ydl
\ No newline at end of file
pm/search_daily_data/job/step1_3.job
View file @ 9b3e3517
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_ydl
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/search_daily_data/job/step1_4.job deleted 100644 → 0
View file @ aa806dfc
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/search_daily_data/job/step1_5.job deleted 100644 → 0
View file @ aa806dfc
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/search_daily_data/job/step2.job
View file @ 9b3e3517
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
dependencies=step1_1,step1_2,step1_3
command=/home/bi/bi-report/lib/shell/hive search_daily_data
\ No newline at end of file
pm/search_daily_data/job/step3.job
View file @ 9b3e3517
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/search_daily_data/dengguangyu@igengmei.com/liudi@igengmei.com,wangxin@igengmei.com,duanyingrong@igengmei.com,lixiaofang@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,wangzhuo@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/search_daily_data/cp-sunyinghe@igengmei.com@igengmei.com/wangsijing@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file
pm/search_daily_data/report/search_daily_data.sql
View file @ 9b3e3517
......@@ -2,36 +2,20 @@
SELECT
day_id AS `日期`
,device_os_type AS `设备类型`
,device_os_type AS `平台`
,active_type AS `活跃类型`
,channel AS `渠道类型`
,dau AS `DAU`
,all_search_uv AS `总完成搜索uv(含首页猜你喜欢、美购首页大家都在搜)`
,all_search_uv/dau AS `总完成搜索uv/DAU`
,all_search_pv/all_search_uv AS `总完成搜索pv/总完成搜索uv`
,search_result_more_all_click_pv/search_result_more_pv AS `搜索综合页正向点击pv/搜索综合页面pv`
,search_result_more_card_click_pv/search_result_more_pv AS `搜索结果综合页所有卡片点击pv(含AI)/搜索结果综合页浏览pv`
,search_result_welfare_card_click_pv/search_result_welfare_pv AS `搜索结果美购页所有卡片点击pv/搜索结果美购页浏览pv`
,search_result_hospital_card_click_pv/search_result_hospital_pv AS `搜索结果医院页所有卡片点击pv/搜索结果医院页浏览pv`
,search_result_doctor_card_click_pv/search_result_doctor_pv AS `搜索结果医生页所有卡片点击pv/搜索结果医生页浏览pv`
,search_result_diary_card_click_pv/search_result_diary_pv AS `搜索结果日记页所有卡片点击pv/搜索结果日记页浏览pv`
,search_result_post_card_click_pv/search_result_post_pv AS `搜索结果帖子页所有卡片点击pv/搜索结果帖子页浏览pv`
,search_result_wiki_card_click_pv/search_result_wiki_pv AS `搜索结果百科页所有卡片点击pv/搜索结果百科页浏览pv`
,all_search_click_pv AS `点击搜索框pv(724及以上)`
,navbar_search_pv/all_search_click_pv AS `通过搜索框完成搜索总pv/点击搜索框pv(724及以上)`
,sug_search_pv/sug_out_pv AS `sug搜索pv/sug弹窗弹出pv(726及以上)`
,home_youlike_search_pv/home_you_like_grey_type_pv AS `首页猜你喜欢点击pv/灰度设备首页浏览pv(723及以上5%灰度)`
,faxain_search_pv/search_home_pv AS `搜索首页发现搜索pv/搜索首页浏览pv`
,lishi_search_pv/search_home_pv AS `搜索首页历史搜索pv/搜索首页浏览pv`
,allsee_search_pv/search_home_pv AS `搜索首页大家都在看搜索pv/搜索首页浏览pv`
,home_morenci_search_pv/home_search_click_pv AS `首页默认词搜索pv/首页搜索框点击pv(727及以上)`
,welfare_home_morenci_search_pv/welfare_home_search_click_pv AS `美购首页默认词搜索pv/美购首页搜索框点击pv(727及以上)`
,content_detail_search_pv/content_detail_search_grey_pv AS `内容详情页点击搜索按钮完成默认词搜索pv/灰度设备内容详情页浏览pv`
,search_result_welfare_pv/all_search_result_pv AS `搜索结果美购页浏览pv/搜索结果(全部tab)页浏览pv`
,search_result_hospital_pv/all_search_result_pv AS `搜索结果医院页浏览pv/搜索结果(全部tab)页浏览pv`
,search_result_doctor_pv/all_search_result_pv AS `搜索结果医生页浏览pv/搜索结果(全部tab)页浏览pv`
,search_result_diary_pv/all_search_result_pv AS `搜索结果日记页浏览pv/搜索结果(全部tab)页浏览pv`
,search_result_post_pv/all_search_result_pv AS `搜索结果帖子页浏览pv/搜索结果(全部tab)页浏览pv`
,search_result_wiki_pv/all_search_result_pv AS `搜索结果百科页浏览pv/搜索结果(全部tab)页浏览pv`
,all_search_uv AS `完成搜索uv`
,all_search_pv AS `完成搜索pv`
,all_search_uv/dau AS `完成搜索的用户比例`
,all_search_pv/all_search_uv AS `搜索用户人均使用次数`
,all_search_pv/search_home_pv AS `搜索成功率`
,sug_search_pv_726/sug_out_pv_726 AS `sug使用率`
,referrer_search_hexin_pv/all_search_uv AS `来源于搜索的核心页pv/搜索uv`
,referrer_search_neirong_pv/all_search_uv AS `来源于搜索的内容页pv/搜索uv`
,search_hexin_two_pv/referrer_search_hexin_pv AS `来源于搜索的核心页二跳率`
,search_neirong_two_pv/referrer_search_neirong_pv AS `来源于搜索的内容页二跳率`
,referrer_search_neirong_pagestay AS `来源于搜索的内容平均浏览时长(m)`
FROM pm.tl_pm_search_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment