Skip to content

B
bi-report
Commit 8177a377 authored by 魏艺敏's avatar 魏艺敏
Browse files
Options

update codes

parent 7a4b9bf5
Expand all Show whitespace changes
Inline Side-by-side
Showing with 268 additions and 12 deletions
pm/daily_content_data/etl/daily_content_data.sql
View file @ 8177a377
......@@ -244,23 +244,23 @@ LEFT JOIN
,device_os_type
,active_type
,t4.channel
,round(sum(page_stay)/count(distinct cl_id)/60,4) as neirong_stay
,round(sum(page_stay)/count(distinct cl_id)/60,2) as neirong_stay
,round(sum(CASE WHEN page_name IN ('diary_detail','topic_detail') THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name IN ('diary_detail','topic_detail') THEN cl_id END)/60,4) AS diary_stay
count(distinct CASE WHEN page_name IN ('diary_detail','topic_detail') THEN cl_id END)/60,2) AS diary_stay
,round(sum(CASE WHEN page_name IN ('post_detail','user_post_detail','doctor_post_detail') THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name IN ('post_detail','user_post_detail','doctor_post_detail') THEN cl_id END)/60,4) AS post_stay
count(distinct CASE WHEN page_name IN ('post_detail','user_post_detail','doctor_post_detail') THEN cl_id END)/60,2) AS post_stay
,round(sum(CASE WHEN page_name ='question_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='question_detail' THEN cl_id END)/60,4) AS question_stay
count(distinct CASE WHEN page_name ='question_detail' THEN cl_id END)/60,2) AS question_stay
,round(sum(CASE WHEN page_name ='question_answer_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='question_answer_detail' THEN cl_id END)/60,4) AS question_answer_stay
count(distinct CASE WHEN page_name ='question_answer_detail' THEN cl_id END)/60,2) AS question_answer_stay
,round(sum(CASE WHEN page_name ='answer_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='answer_detail' THEN cl_id END)/60,4) AS answer_stay
count(distinct CASE WHEN page_name ='answer_detail' THEN cl_id END)/60,2) AS answer_stay
,round(sum(CASE WHEN page_name ='video_steep' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='video_steep' THEN cl_id END)/60,4) AS video_stay
count(distinct CASE WHEN page_name ='video_steep' THEN cl_id END)/60,2) AS video_stay
,round(sum(CASE WHEN page_name ='article_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='article_detail' THEN cl_id END)/60,4) AS article_stay
count(distinct CASE WHEN page_name ='article_detail' THEN cl_id END)/60,2) AS article_stay
,round(sum(CASE WHEN page_name IN ('wiki_detail','product_detail','wiki_brand','wiki_collect') THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name IN ('wiki_detail','product_detail','wiki_brand','wiki_collect') THEN cl_id END)/60,4) AS wiki_stay
count(distinct CASE WHEN page_name IN ('wiki_detail','product_detail','wiki_brand','wiki_collect') THEN cl_id END)/60,2) AS wiki_stay
FROM
(
SELECT t1.partition_date,device_os_type,active_type,channel,t2.cl_id,t2.page_name,t2.page_stay
......@@ -429,7 +429,7 @@ LEFT JOIN
,device_os_type
,active_type
,t5.channel
,round(sum(use_duration)/count(distinct cl_id)/60,4) as app_duration
,round(sum(use_duration)/count(distinct cl_id)/60,2) as app_duration
,round(avg(open_times),4) as avg_opentimes
FROM
(
......
pm/daily_recommend_strategy/job/step3.job
View file @ 8177a377
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com
\ No newline at end of file
pm/search_related/en-cn.properties 0 → 100644
View file @ 8177a377
search_click_path=分搜索入口的搜索框点击数据
search_input_type=分搜索方式近7天数据
search_result_click_rate=分搜索词转化率数据
search_related=分搜索方式及搜索词日报
\ No newline at end of file
pm/search_related/etl/create_search_click_path.sql 0 → 100644
View file @ 8177a377
--***************************************************************
--*脚本名称:
--*功能: 分搜索入口搜索日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_click_path_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
search_entrance string comment '{"chs_name":"搜索入口","description":"","etl":"","value":"","remark":""}',
click_search_pv int comment '{"chs_name":"点击搜索或搜索框pv","description":"","etl":"","value":"","remark":""}',
click_search_uv int comment '{"chs_name":"点击搜索或搜索框uv","description":"","etl":"","value":"","remark":""}'
)comment '分搜索入口的搜索点击数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/search_related/etl/create_search_input_type.sql 0 → 100644
View file @ 8177a377
--***************************************************************
--*脚本名称:
--*功能: 分搜索方式日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_input_type_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
query string comment '{"chs_name":"搜索词","description":"","etl":"","value":"","remark":""}',
input_type string comment '{"chs_name":"搜索方式","description":"","etl":"","value":"","remark":""}',
search_pv bigint comment '{"chs_name":"搜索量","description":"","etl":"","value":"","remark":""}'
)comment '分方式搜索日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/search_related/etl/create_search_result_click_rate.sql 0 → 100644
View file @ 8177a377
--***************************************************************
--*脚本名称:
--*功能: 分搜索关键词转化日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_search_result_click_rate_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
query string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
search_pv bigint comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
search_uv bigint comment '{"chs_name":"搜索入口","description":"","etl":"","value":"","remark":""}',
search_more_pv bigint comment '{"chs_name":"综合tab下的搜索的点击pv","description":"","etl":"","value":"","remark":""}',
page_more_pv bigint comment '{"chs_name":"搜索综合结果页浏览pv","description":"","etl":"","value":"","remark":""}',
more_click_pv bigint comment '{"chs_name":"搜索综合结果页内容卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_10_pv bigint comment '{"chs_name":"搜索综合结果页前10内容卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_5_diary_pv bigint comment '{"chs_name":"搜索综合结果页前5日记卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_5_answer_pv bigint comment '{"chs_name":"搜索综合结果页前5回答卡片点击PV","description":"","etl":"","value":"","remark":""}',
more_click_5_topic_pv bigint comment '{"chs_name":"搜索综合结果页前5帖子卡片点击PV","description":"","etl":"","value":"","remark":""}',
search_wel_pv bigint comment '{"chs_name":"美购tab下的搜索的点击pv","description":"","etl":"","value":"","remark":""}',
page_wel_pv bigint comment '{"chs_name":"搜索美购结果页浏览pv","description":"","etl":"","value":"","remark":""}',
wel_click_pv bigint comment '{"chs_name":"搜索美购结果页卡片点击PV","description":"","etl":"","value":"","remark":""}',
wel_click_10_pv bigint comment '{"chs_name":"搜索美购结果页前10卡片点击PV","description":"","etl":"","value":"","remark":""}',
wel_click_5_pv bigint comment '{"chs_name":"搜索美购结果页前5卡片点击PV","description":"","etl":"","value":"","remark":""}',
search_diary_pv bigint comment '{"chs_name":"日记tab下的搜索的点击pv","description":"","etl":"","value":"","remark":""}',
page_diary_pv bigint comment '{"chs_name":"搜索日记结果页浏览pv","description":"","etl":"","value":"","remark":""}',
diary_click_pv bigint comment '{"chs_name":"搜索日记结果页卡片点击PV","description":"","etl":"","value":"","remark":""}',
diary_click_10_pv bigint comment '{"chs_name":"搜索日记结果页前10卡片点击PV","description":"","etl":"","value":"","remark":""}',
diary_click_5_pv bigint comment '{"chs_name":"搜索日记结果页前5卡片点击PV","description":"","etl":"","value":"","remark":""}',
search_more_uv bigint comment '{"chs_name":"综合tab下的搜索的点击uv","description":"","etl":"","value":"","remark":""}',
page_more_uv bigint comment '{"chs_name":"搜索综合结果页浏览uv","description":"","etl":"","value":"","remark":""}',
more_click_uv bigint comment '{"chs_name":"搜索综合结果页内容卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_10_uv bigint comment '{"chs_name":"搜索综合结果页前10内容卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_5_diary_uv bigint comment '{"chs_name":"搜索综合结果页前5日记卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_5_answer_uv bigint comment '{"chs_name":"搜索综合结果页前5回答卡片点击UV","description":"","etl":"","value":"","remark":""}',
more_click_5_topic_uv bigint comment '{"chs_name":"搜索综合结果页前5帖子卡片点击UV","description":"","etl":"","value":"","remark":""}',
search_wel_uv bigint comment '{"chs_name":"美购tab下的搜索的点击uv","description":"","etl":"","value":"","remark":""}',
page_wel_uv bigint comment '{"chs_name":"搜索美购结果页浏览uv","description":"","etl":"","value":"","remark":""}',
wel_click_uv bigint comment '{"chs_name":"搜索美购结果页卡片点击UV","description":"","etl":"","value":"","remark":""}',
wel_click_10_uv bigint comment '{"chs_name":"搜索美购结果页前10卡片点击UV","description":"","etl":"","value":"","remark":""}',
wel_click_5_uv bigint comment '{"chs_name":"搜索美购结果页前5卡片点击UV","description":"","etl":"","value":"","remark":""}',
search_diary_uv bigint comment '{"chs_name":"日记tab下的搜索的点击uv","description":"","etl":"","value":"","remark":""}',
page_diary_uv bigint comment '{"chs_name":"搜索日记结果页浏览uv","description":"","etl":"","value":"","remark":""}',
diary_click_uv bigint comment '{"chs_name":"搜索日记结果页卡片点击UV","description":"","etl":"","value":"","remark":""}',
diary_click_10_uv bigint comment '{"chs_name":"搜索日记结果页前10卡片点击UV","description":"","etl":"","value":"","remark":""}',
diary_click_5_uv bigint comment '{"chs_name":"搜索日记结果页前5卡片点击UV","description":"","etl":"","value":"","remark":""}'
)comment '分搜索词日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/search_related/etl/search_related.sql 0 → 100644
View file @ 8177a377
This diff is collapsed.
pm/search_related/job/step1_1.job 0 → 100644
View file @ 8177a377
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
pm/search_related/job/step1_2.job 0 → 100644
View file @ 8177a377
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/search_related/job/step1_3.job 0 → 100644
View file @ 8177a377
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/search_related/job/step1_4.job 0 → 100644
View file @ 8177a377
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/search_related/job/step1_5.job 0 → 100644
View file @ 8177a377
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
pm/search_related/job/step2.job 0 → 100644
View file @ 8177a377
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
command=/home/bi/bi-report/lib/shell/hive search_related
\ No newline at end of file
pm/search_related/job/step3.job 0 → 100644
View file @ 8177a377
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/search_related/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,duanyingrong@igengmei.com,dengguangyu@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com
\ No newline at end of file
pm/search_related/report/search_click_path.sql 0 → 100644
View file @ 8177a377
SELECT
day_id AS `日期`
,device_os_type AS `设备类型`
,active_type AS `活跃类型`
,search_entrance AS `搜索入口`
,click_search_pv AS `点击搜索框pv`
,click_search_uv AS `点击搜索框uv`
FROM pm.tl_pm_search_click_path_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`点击搜索框pv`desc
\ No newline at end of file
pm/search_related/report/search_input_type.sql 0 → 100644
View file @ 8177a377
--近7天搜索方式的搜索量
SELECT regexp_replace(DATE_SUB(current_date,1) ,'-','') AS `日期`,
T4.QUERY AS `搜索词`,
T4.INPUT_TYPE AS `搜索类型`,
T4.SEARCH_PV AS `搜索量`
FROM
(SELECT T3.*
FROM (SELECT T2.*,
SUM(SEARCH_PV_ALL) OVER(PARTITION BY T2.QUERY) SEARCH_PV_SUM,
ROW_NUMBER() OVER(PARTITION BY T2.QUERY ORDER BY SEARCH_PV DESC) SEARCH_PV_GROUP_SUM,
DENSE_RANK() OVER(ORDER BY SUM(SEARCH_PV_ALL) OVER(PARTITION BY T2.QUERY) DESC,T2.QUERY) SEARCH_PV_GROUP
FROM (SELECT T1.*,
CASE
WHEN INPUT_TYPE = '全部' THEN
0
ELSE
1
END INPUT_TYPE_RANK,
CASE
WHEN INPUT_TYPE = '全部' THEN
SEARCH_PV
ELSE
0
END SEARCH_PV_ALL
FROM (SELECT QUERY,INPUT_TYPE,SUM(SEARCH_PV) AS SEARCH_PV
FROM PM.TL_PM_SEARCH_INPUT_TYPE_D T
WHERE T.PARTITION_DAY >= regexp_replace(DATE_SUB(current_date,7) ,'-','')
AND T.PARTITION_DAY<regexp_replace((current_date),'-','')
AND T.DAY_ID >= regexp_replace(DATE_SUB(current_date,7) ,'-','')
AND T.DAY_ID<regexp_replace((current_date),'-','')
GROUP BY T.QUERY,T.INPUT_TYPE) T1) T2)T3
WHERE SEARCH_PV_GROUP<=300
ORDER BY T3.SEARCH_PV_SUM DESC,
T3.QUERY,
T3.INPUT_TYPE_RANK,
T3.SEARCH_PV_GROUP_SUM)T4
\ No newline at end of file
pm/search_related/report/search_result_click_rate.sql 0 → 100644
View file @ 8177a377
SELECT
day_id AS `日期`
,query AS `关键词`
,search_pv AS `搜索量`
,search_uv AS `搜索uv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_pv*100/page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_10_pv*100/page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_diary_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_answer_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_topic_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_10_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_5_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_10_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_5_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,search_more_pv AS `综合tab下的搜索的点击pv`
,page_more_pv AS `搜索综合结果页浏览PV`
,more_click_pv AS `搜索综合结果页内容卡片点击PV`
,more_click_10_pv AS `搜索综合结果页前10内容卡片点击PV`
,more_click_5_diary_pv AS `搜索综合结果页前5日记卡片点击PV`
,more_click_5_answer_pv AS `搜索综合结果页前5问答卡片点击PV`
,more_click_5_topic_pv AS `搜索综合结果页前5帖子卡片点击PV`
,search_wel_pv AS `美购tab下的搜索的点击pv`
,page_wel_pv AS `搜索美购结果页浏览pv`
,wel_click_pv AS `搜索美购结果页卡片点击PV`
,wel_click_10_pv AS `搜索美购结果页前10卡片点击PV`
,wel_click_5_pv AS `搜索美购结果页前5卡片点击PV`
,search_diary_pv AS `日记tab下的搜索的点击pv`
,page_diary_pv AS `搜索日记结果页浏览pv`
,diary_click_pv AS `搜索日记结果页卡片点击PV`
,diary_click_10_pv AS `搜索日记结果页前10卡片点击PV`
,diary_click_5_pv AS `搜索日记结果页前5卡片点击PV`
FROM pm.tl_pm_search_result_click_rate_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>=regexp_replace(DATE_SUB(current_date,7) ,'-','')
and day_id<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`搜索量` desc,`关键词`
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment