Skip to content

B
bi-report
Commit 26739b20 authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

push codes

parent 4734938d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 631 deletions
conf/en-cn.properties deleted 100644 → 0
View file @ 4734938d
demo=示例程序-demo
demo_bak=示例程序-demo_bak
demo_param=带参数的示例程序
demo_java=JAVA实例程序
meigou_detail_page=美购详情页来源与分发
meigou_detail_page_dispense_pv=美购详情页分发pv
meigou_detail_page_dispense_uv=美购详情页分发uv
meigou_detail_page_source_pv=美购详情页来源pv
meigou_detail_page_source_uv=美购详情页来源uv
daily_content_data=内容日报-新
search_click_path=分搜索入口的搜索框点击数据
search_input_type=分搜索方式近7天数据
search_result_click_rate=分搜索词转化率数据
search_related=分搜索方式及搜索词日报
daily_recommend_strategy=首页推荐策略日报
jobs/daily_content_data/step1_1.job deleted 100644 → 0
View file @ 4734938d
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/daily_content_data/step1_10.job deleted 100644 → 0
View file @ 4734938d
#step1_10.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_view
\ No newline at end of file
jobs/daily_content_data/step1_11.job deleted 100644 → 0
View file @ 4734938d
#step1_11.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_topicreply_view
\ No newline at end of file
jobs/daily_content_data/step1_12.job deleted 100644 → 0
View file @ 4734938d
#step1_12.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_reply_view
\ No newline at end of file
jobs/daily_content_data/step1_13.job deleted 100644 → 0
View file @ 4734938d
#step1_13.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file
jobs/daily_content_data/step1_14.job deleted 100644 → 0
View file @ 4734938d
#step1_14.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive al al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
\ No newline at end of file
jobs/daily_content_data/step1_2.job deleted 100644 → 0
View file @ 4734938d
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/daily_content_data/step1_3.job deleted 100644 → 0
View file @ 4734938d
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file
jobs/daily_content_data/step1_4.job deleted 100644 → 0
View file @ 4734938d
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
jobs/daily_content_data/step1_5.job deleted 100644 → 0
View file @ 4734938d
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file
jobs/daily_content_data/step1_6.job deleted 100644 → 0
View file @ 4734938d
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file
jobs/daily_content_data/step1_7.job deleted 100644 → 0
View file @ 4734938d
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_diary_view
\ No newline at end of file
jobs/daily_content_data/step1_8.job deleted 100644 → 0
View file @ 4734938d
#step1_8.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_problem_view
\ No newline at end of file
jobs/daily_content_data/step1_9.job deleted 100644 → 0
View file @ 4734938d
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_question_view
\ No newline at end of file
jobs/daily_content_data/step2.job deleted 100644 → 0
View file @ 4734938d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
command=/home/bi/bi-report/lib/shell/hive.sh daily_content_data_insert
\ No newline at end of file
jobs/daily_content_data/step3.job deleted 100644 → 0
View file @ 4734938d
#step3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_content_d
\ No newline at end of file
jobs/daily_content_data/step4.job deleted 100644 → 0
View file @ 4734938d
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaojianwei@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com/daily_content_data
\ No newline at end of file
jobs/daily_recommend_strategy/step1_1.job deleted 100644 → 0
View file @ 4734938d
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/daily_recommend_strategy/step1_2.job deleted 100644 → 0
View file @ 4734938d
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
jobs/daily_recommend_strategy/step1_3.job deleted 100644 → 0
View file @ 4734938d
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/daily_recommend_strategy/step1_4.job deleted 100644 → 0
View file @ 4734938d
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
jobs/daily_recommend_strategy/step1_5.job deleted 100644 → 0
View file @ 4734938d
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
jobs/daily_recommend_strategy/step1_6.job deleted 100644 → 0
View file @ 4734938d
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file
jobs/daily_recommend_strategy/step1_7.job deleted 100644 → 0
View file @ 4734938d
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file
jobs/daily_recommend_strategy/step1_8.job deleted 100644 → 0
View file @ 4734938d
#step1_8.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file
jobs/daily_recommend_strategy/step1_9.job deleted 100644 → 0
View file @ 4734938d
#step1_9.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
jobs/daily_recommend_strategy/step2.job deleted 100644 → 0
View file @ 4734938d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/lib/shell/hive.sh daily_recommend_strategy_insert
\ No newline at end of file
jobs/daily_recommend_strategy/step3.job deleted 100644 → 0
View file @ 4734938d
#step3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_recommend_strategy_d
\ No newline at end of file
jobs/daily_recommend_strategy/step4.job deleted 100644 → 0
View file @ 4734938d
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com/daily_recommend_strategy
\ No newline at end of file
jobs/demo/step.job deleted 100644 → 0
View file @ 4734938d
#step.job
type=command
command=/home/bi/bi-report/lib/shell/hive.sh demo
\ No newline at end of file
jobs/meigou_detail_page/step1_1.job deleted 100644 → 0
View file @ 4734938d
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/meigou_detail_page/step1_2.job deleted 100644 → 0
View file @ 4734938d
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/meigou_detail_page/step1_3.job deleted 100644 → 0
View file @ 4734938d
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
jobs/meigou_detail_page/step2.job deleted 100644 → 0
View file @ 4734938d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com,zhaojianwei@igengmei.com/meigou_detail_page
\ No newline at end of file
jobs/search_related/step1_1.job deleted 100644 → 0
View file @ 4734938d
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/search_related/step1_2.job deleted 100644 → 0
View file @ 4734938d
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
jobs/search_related/step1_3.job deleted 100644 → 0
View file @ 4734938d
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/search_related/step1_4.job deleted 100644 → 0
View file @ 4734938d
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
jobs/search_related/step1_5.job deleted 100644 → 0
View file @ 4734938d
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
jobs/search_related/step2.job deleted 100644 → 0
View file @ 4734938d
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
command=/home/bi/bi-report/lib/shell/hive.sh search_related_insert
\ No newline at end of file
jobs/search_related/step3_1.job deleted 100644 → 0
View file @ 4734938d
#step3-1.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_click_path_d
\ No newline at end of file
jobs/search_related/step3_2.job deleted 100644 → 0
View file @ 4734938d
#step3-2.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_input_type_d
\ No newline at end of file
jobs/search_related/step3_3.job deleted 100644 → 0
View file @ 4734938d
#step3-3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_result_click_rate_d
\ No newline at end of file
jobs/search_related/step4.job deleted 100644 → 0
View file @ 4734938d
#step4.job
type=command
dependencies=step3_1,step3_2,step3_3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,duanyingrong@igengmei.com,dengguangyu@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com/search_related
\ No newline at end of file
shells/daily_content_data/daily_content_data.sh deleted 100755 → 0
View file @ 4734938d
This diff is collapsed.
shells/search_related/search_related.sh deleted 100755 → 0
View file @ 4734938d
This diff is collapsed.
sqls/daily_content_data/daily_content_data.sql deleted 100644 → 0
View file @ 4734938d
--内容日报
SELECT
day_id AS `日期`
,device_os_type AS `设备类型`
,active_type AS `活跃类型`
,is_ai_channel AS `是否AI渠道`
,dau AS `DAU`
,content_uv AS `内容详情页UV`
,content_pv AS `内容详情页PV`
,per_content_uv AS `内容UV/DAU`
,per_content_pv AS `内容PV/内容UV`
,retention_1 AS `内容用户APP次留`
,retention_7 AS `内容用户APP7留`
,retention_30 AS `内容用户APP30留`
,avg_app_duration AS `内容用户单设备App时长(m)`
,avg_content_stay AS `内容用户单设备内容时长(m)`
,avg_open_times AS `内容用户单设备打开次数`
,search_related_stay AS `内容用户搜索相关页面单设备页面时长(m)`
,welfare_stay AS `内容用户美购详情页单设备页面时长(m)`
,content_question_stay AS `内容用户问题详情页单设备页面时长(m)`
,ai_related_stay AS `内容用户AI相关页面单设备页面时长(m)`
,content_diary_stay AS `内容用户日记详情页单设备页面时长(m)`
,home_stay AS `内容用户首页单设备页面时长(m)`
,conv_related_stay AS `内容用户咨询相关页面单设备页面时长(m)`
,recommend_rate AS `首页feeds推荐进入内容UV/内容UV`
,per_recommend_pv AS `首页feeds推荐进入内容PV/UV`
,feeds_rate AS `首页feeds非推荐进入内容UV/内容UV`
,per_feeds_pv AS `首页feeds非推荐进入内容PV/UV`
,search_rate AS `搜索进入内容UV/内容UV`
,per_search_pv AS `搜索进入内容PV/UV`
,zone_rate AS `内容聚合页进入内容UV/内容UV`
,per_zone_pv AS `内容聚合页进入内容PV/UV`
,content_rate AS `内容详情页推荐板块进入内容UV/内容UV`
,per_from_content_pv AS `内容详情页推荐板块进入内容PV/UV`
,blank_rate AS `无来源页面(大多数为push)进入内容UV/内容UV`
,per_blank_pv AS `无来源页面(大多数为push)进入内容PV/UV`
,comment_rate AS `评论列表页进入内容UV/内容UV`
,per_comment_pv AS `评论列表页进入内容PV/UV`
,org_rate AS `医生医院主页进入内容UV/内容UV`
,per_org_pv AS `医生医院主页进入内容PV/UV`
,category_rate AS `品类聚合页进入内容UV/内容UV`
,per_category_pv AS `品类聚合页进入内容PV/UV`
,my_diary_rate AS `我的日记页进入内容UV/内容UV`
,per_my_diary_pv AS `我的日记页进入内容PV/UV`
,ai_rate AS `AI报告页进入内容UV/内容UV`
,per_ai_pv AS `AI报告页进入内容PV/UV`
,diary_uv AS `日记UV`
,diary_pv AS `日记PV`
,diary_rate AS `日记UV/内容UV`
,per_diary_pv AS `日记PV/日记UV`
,diary_stay AS `日记单设备时长(m)`
,post_uv AS `帖子UV`
,post_pv AS `帖子PV`
,post_rate AS `帖子UV/内容UV`
,per_post_pv AS `帖子PV/帖子UV`
,post_stay AS `帖子单设备时长(m)`
,question_uv AS `问题UV`
,question_pv AS `问题PV`
,question_rate AS `问题UV/内容UV`
,per_question_pv AS `问题PV/问题UV`
,question_stay AS `问题单设备时长(m)`
,question_answer_uv AS `问答UV`
,question_answer_pv AS `问答PV`
,question_answer_rate AS `问答UV/内容UV`
,per_question_answer_pv AS `问答PV/问答UV`
,question_answer_stay AS `问答单设备时长(m)`
,answer_uv AS `回答UV`
,answer_pv AS `回答PV`
,answer_rate AS `回答UV/内容UV`
,per_answer_pv AS `回答PV/回答UV`
,answer_stay AS `回答单设备时长(m)`
,video_uv AS `视频UV`
,video_pv AS `视频PV`
,video_rate AS `视频UV/内容UV`
,per_video_pv AS `视频PV/视频UV`
,video_stay AS `视频单设备时长(m)`
,wiki_uv AS `百科UV`
,wiki_pv AS `百科PV`
,wiki_rate AS `百科UV/内容UV`
,per_wiki_pv AS `百科PV/百科UV`
,wiki_stay AS `百科单设备时长(m)`
,article_uv AS `专栏UV`
,article_pv AS `专栏PV`
,article_rate AS `专栏UV/内容UV`
,per_article_pv AS `专栏PV/专栏UV`
,article_stay AS `专栏单设备时长(m)`
FROM pm.tl_pm_content_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
sqls/daily_recommend_strategy/daily_recommend_strategy.sql deleted 100644 → 0
View file @ 4734938d
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`
sqls/demo/demo.sql deleted 100644 → 0
View file @ 4734938d
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT day_id as day_id
,device_os_type as device_os_type
,active_type as active_type
,v.search_entrance as search_entrance
,sum(search_pv) as click_search_pv
,sum(search_uv) as click_search_uv
FROM
(
SELECT day_id,device_os_type,active_type,array(search_entrance,'合计') as search_entrance
,search_pv,search_uv
FROM
(
SELECT t1.partition_date as day_id
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end as search_entrance
,count(cl_id) as search_pv
,count(distinct cl_id) as search_uv
FROM
(
SELECT partition_date,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,device_id
FROM online.ml_device_day_active_status
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
join
(
SELECT partition_date,cl_id,page_name,action,params['referrer_link'] as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action in ('do_search','on_click_navbar_search')
union all
SELECT partition_date,cl_id,null as page_name,'大家都���看' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date,cl_id,page_name,'热搜词' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date,cl_id,null as page_name,'详情页高亮词' as action,null as referrer_link
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='on_click_card'
and params['card_type']='highlight_word'
)t2
on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id
group by t1.partition_date
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end
having search_entrance is not null
)t
)n
LATERAL VIEW explode(n.search_entrance) v AS search_entrance
group by day_id,device_os_type,active_type,v.search_entrance;
\ No newline at end of file
sqls/search_related/search_click_path.sql deleted 100644 → 0
View file @ 4734938d
SELECT
day_id AS `日期`
,device_os_type AS `设备类型`
,active_type AS `活跃类型`
,search_entrance AS `搜索入口`
,click_search_pv AS `点击搜索框pv`
,click_search_uv AS `点击搜索框uv`
FROM pm.tl_pm_search_click_path_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`点击搜索框pv`desc
\ No newline at end of file
sqls/search_related/search_input_type.sql deleted 100644 → 0
View file @ 4734938d
--近7天搜索方式的搜索量
SELECT regexp_replace(DATE_SUB(current_date,1) ,'-','') AS `日期`,
T4.QUERY AS `搜索词`,
T4.INPUT_TYPE AS `搜索类型`,
T4.SEARCH_PV AS `搜索量`
FROM
(SELECT T3.*
FROM (SELECT T2.*,
SUM(SEARCH_PV_ALL) OVER(PARTITION BY T2.QUERY) SEARCH_PV_SUM,
ROW_NUMBER() OVER(PARTITION BY T2.QUERY ORDER BY SEARCH_PV DESC) SEARCH_PV_GROUP_SUM,
DENSE_RANK() OVER(ORDER BY SUM(SEARCH_PV_ALL) OVER(PARTITION BY T2.QUERY) DESC,T2.QUERY) SEARCH_PV_GROUP
FROM (SELECT T1.*,
CASE
WHEN INPUT_TYPE = '全部' THEN
0
ELSE
1
END INPUT_TYPE_RANK,
CASE
WHEN INPUT_TYPE = '全部' THEN
SEARCH_PV
ELSE
0
END SEARCH_PV_ALL
FROM (SELECT QUERY,INPUT_TYPE,SUM(SEARCH_PV) AS SEARCH_PV
FROM PM.TL_PM_SEARCH_INPUT_TYPE_D T
WHERE T.PARTITION_DAY >= regexp_replace(DATE_SUB(current_date,7) ,'-','')
AND T.PARTITION_DAY<regexp_replace((current_date),'-','')
AND T.DAY_ID >= regexp_replace(DATE_SUB(current_date,7) ,'-','')
AND T.DAY_ID<regexp_replace((current_date),'-','')
GROUP BY T.QUERY,T.INPUT_TYPE) T1) T2)T3
WHERE SEARCH_PV_GROUP<=300
ORDER BY T3.SEARCH_PV_SUM DESC,
T3.QUERY,
T3.INPUT_TYPE_RANK,
T3.SEARCH_PV_GROUP_SUM)T4
\ No newline at end of file
sqls/search_related/search_result_click_rate.sql deleted 100644 → 0
View file @ 4734938d
SELECT
day_id AS `日期`
,query AS `关键词`
,search_pv AS `搜索量`
,search_uv AS `搜索uv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_pv*100/page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_10_pv*100/page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_diary_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_answer_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_topic_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_10_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_5_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_10_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_5_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,search_more_pv AS `综合tab下的搜索的点击pv`
,page_more_pv AS `搜索综合结果页浏览PV`
,more_click_pv AS `搜索综合结果页内容卡片点击PV`
,more_click_10_pv AS `搜索综合结果页前10内容卡片点击PV`
,more_click_5_diary_pv AS `搜索综合结果页前5日记卡片点击PV`
,more_click_5_answer_pv AS `搜索综合结果页前5问答卡片点击PV`
,more_click_5_topic_pv AS `搜索综合结果页前5帖子卡片点击PV`
,search_wel_pv AS `美购tab下的搜索的点击pv`
,page_wel_pv AS `搜索美购结果页浏览pv`
,wel_click_pv AS `搜索美购结果页卡片点击PV`
,wel_click_10_pv AS `搜索美购结果页前10卡片点击PV`
,wel_click_5_pv AS `搜索美购结果页前5卡片点击PV`
,search_diary_pv AS `日记tab下的搜索的点击pv`
,page_diary_pv AS `搜索日记结果页浏览pv`
,diary_click_pv AS `搜索日记结果页卡片点击PV`
,diary_click_10_pv AS `搜索日记结果页前10卡片点击PV`
,diary_click_5_pv AS `搜索日记结果页前5卡片点击PV`
FROM pm.tl_pm_search_result_click_rate_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>=regexp_replace(DATE_SUB(current_date,7) ,'-','')
and day_id<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`搜索量` desc,`关键词`
sqls/search_related_insert/search_related_insert.sql deleted 100644 → 0
View file @ 4734938d
This diff is collapsed.
tables/daily_content/create_daliy_content.sql deleted 100644 → 0
View file @ 4734938d
This diff is collapsed.
tables/daily_recommend_strategy/create_recommend_daily_data.sql deleted 100644 → 0
View file @ 4734938d
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
card_click BIGINT comment '{"chs_name":"首页卡片点击PV","description":"","etl":"","value":"","remark":""}',
card_exposure BIGINT comment '{"chs_name":"首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
total_second_click BIGINT comment '{"chs_name":"来自首页卡片的总商业二跳pv（包含页面上搜索及卡片点击）","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"来自首页卡片的单PV平均浏览时长","description":"","etl":"","value":"","remark":""}',
navbar_search BIGINT comment '{"chs_name":"来自首页卡片的搜索框+搜索按钮点击PV","description":"","etl":"","value":"","remark":""}',
highlight_word BIGINT comment '{"chs_name":"来自首页卡片的文内搜索点击PV","description":"","etl":"","value":"","remark":""}',
self_welfare_card BIGINT comment '{"chs_name":"来自首页卡片的商品卡片点击PV","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card BIGINT comment '{"chs_name":"来自首页卡片的推荐商品+查看全部商品点击pv","description":"","etl":"","value":"","remark":""}',
recommend_content_card BIGINT comment '{"chs_name":"来自首页卡片的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
recommend_special_card BIGINT comment '{"chs_name":"来自首页卡片的推荐专题点击pv","description":"","etl":"","value":"","remark":""}',
transfer_card BIGINT comment '{"chs_name":"来自首页卡片的转诊点击pv","description":"","etl":"","value":"","remark":""}',
video_consultation BIGINT comment '{"chs_name":"来自首页卡片的视频面诊点击pv","description":"","etl":"","value":"","remark":""}'
)comment '首页推荐策略日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
tables/search_related/create_search_click_path.sql deleted 100644 → 0
View file @ 4734938d
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_click_path_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
search_entrance string comment '{"chs_name":"搜索入口","description":"","etl":"","value":"","remark":""}',
click_search_pv int comment '{"chs_name":"点击搜索或搜索框pv","description":"","etl":"","value":"","remark":""}',
click_search_uv int comment '{"chs_name":"点击搜索或搜索框uv","description":"","etl":"","value":"","remark":""}'
)comment '分搜索入口的搜索点击数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
tables/search_related/create_search_input_type.sql deleted 100644 → 0
View file @ 4734938d
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_input_type_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
query string comment '{"chs_name":"搜索词","description":"","etl":"","value":"","remark":""}',
input_type string comment '{"chs_name":"搜索方式","description":"","etl":"","value":"","remark":""}',
search_pv bigint comment '{"chs_name":"搜索量","description":"","etl":"","value":"","remark":""}'
)comment '分方式搜索日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment