Skip to content

B
bi-report
Commit fb92acfc authored by 赵建伟's avatar 赵建伟
Browse files
Options

update codes

parent 44173034
Hide whitespace changes
Inline Side-by-side
Showing with 106 additions and 116 deletions
lib/shell/hive.sh
View file @ fb92acfc
...@@ -25,8 +25,6 @@ else ...@@ -25,8 +25,6 @@ else
partition_date=`date --date="-1 day" +%Y%m%d` partition_date=`date --date="-1 day" +%Y%m%d`
fi fi
echo $partition_date
#参数替换 #参数替换
tmp_sql=`cat ${sql_path}` tmp_sql=`cat ${sql_path}`
sql=${tmp_sql//'${partition_day}'/${partition_date}} sql=${tmp_sql//'${partition_day}'/${partition_date}}
......
sqls/demo/demo.sql
View file @ fb92acfc
SET mapreduce.job.queuename=pm; SET mapreduce.job.queuename = data;
SET mapreduce.map.memory.mb=8192; SET mapreduce.map.memory.mb = 8192;
SET mapreduce.map.java.opts=-Xmx8000m; SET mapreduce.map.java.opts = -Xmx8000m;
SET mapreduce.reduce.memory.mb=8192; SET mapreduce.reduce.memory.mb = 8192;
SET mapreduce.reduce.java.opts=-Xmx8000m; SET mapreduce.reduce.java.opts = -Xmx8000m;
set hive.auto.convert.join=true; set hive.auto.convert.join = true;
SET mapred.reduce.tasks=20; SET mapred.reduce.tasks = 20;
SET role admin; SET role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar; INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = ${partition_day})
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = #partition_day) SELECT day_id as day_id
, device_os_type as device_os_type
SELECT day_id as day_id , active_type as active_type
,device_os_type as device_os_type , v.search_entrance as search_entrance
,active_type as active_type , sum(search_pv) as click_search_pv
,v.search_entrance as search_entrance , sum(search_uv) as click_search_uv
,sum(search_pv) as click_search_pv FROM (
,sum(search_uv) as click_search_uv SELECT day_id
FROM , device_os_type
( , active_type
SELECT day_id,device_os_type,active_type,array(search_entrance,'合计') as search_entrance , array(search_entrance, ''合计'') as search_entrance
,search_pv,search_uv , search_pv
FROM , search_uv
( FROM (
SELECT t1.partition_date as day_id SELECT t1.partition_date as day_id
,t1.device_os_type , t1.device_os_type
,t1.active_type , t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框' , case when action = ''on_click_navbar_search'' and page_name = '' home '' then ''首页搜索框''
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框' when action=''on_click_navbar_search'' and page_name=''welfare_home'' then ''美购首页搜索框''
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框' when action=''on_click_navbar_search'' and page_name=''category'' then ''品类聚合页搜索框''
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框' when action=''on_click_navbar_search'' and page_name=''welfare_list'' and referrer_link =''["home","category"]'' then ''来自品类聚合的美购列表页搜索框''
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框' when action=''on_click_navbar_search'' and page_name=''welfare_list'' and referrer_link =''["welfare_home"]'' then ''来自美购首页的美购列表页搜索框''
when action in ('do_search','on_click_navbar_search') when action in (''do_search'',''on_click_navbar_search'')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail' and page_name in (''diary_detail'',''topic_detail'',''post_detail'',''user_post_detail''
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框' ,''doctor_post_detail'',''question_detail'',''answer_detail'',''question_answer_detail'') then ''内容详情页搜索框''
when action ='详情页高亮词' then '详情页高亮词搜索' when action =''详情页高亮词'' then ''详情页高亮词搜索''
when action='大家都在看' then '首页feed大家都在看' when action=''大家都在看'' then ''首页feed大家都在看''
when action='热搜词' then '美购首页热搜词' else null end as search_entrance when action=''热搜词'' then ''美购首页热搜词'' else null end as search_entrance
,count(cl_id) as search_pv ,count(cl_id) as search_pv
,count(distinct cl_id) as search_uv ,count(distinct cl_id) as search_uv
FROM FROM
( (
SELECT partition_date,device_os_type SELECT partition_date, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备' , case WHEN active_type = ''4'' THEN '' 老活跃设备 ''
WHEN active_type in ('1','2') then '新增设备' END as active_type WHEN active_type in (''1'', ''2'') then '' 新增设备 '' END as active_type
,device_id , device_id
FROM online.ml_device_day_active_status FROM online.ml_device_day_active_status
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','') where partition_date = regexp_replace(DATE_SUB(current_date, 1), ''-'', '''')
AND active_type in ('1','2','4') AND active_type in (''1'', ''2'', ''4'')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3' and first_channel_source_type not in ('' yqxiu1 '', '' yqxiu2 '', '' yqxiu3 '', '' yqxiu4 '', '' yqxiu5 '', '' mxyc1 '', '' mxyc2 '', '' mxyc3 ''
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang' , '' wanpu '', '' jinshan '', '' jx '', '' maimai '', '' zhuoyi '', '' huatian '', '' suopingjingling '', '' mocha '', '' mizhe '', '' meika '', '' lamabang ''
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1' , '' js-az1 '', '' js-az2 '', '' js-az3 '', '' js-az4 '', '' js-az5 '', '' jfq-az1 '', '' jfq-az2 '', '' jfq-az3 '', '' jfq-az4 '', '' jfq-az5 '', '' toufang1 ''
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4' , '' toufang2 '', '' toufang3 '', '' toufang4 '', '' toufang5 '', '' toufang6 '', '' TF-toufang1 '', '' TF-toufang2 '', '' TF-toufang3 '', '' TF-toufang4 ''
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100' , '' TF-toufang5 '', '' tf-toufang1 '', '' tf-toufang2 '', '' tf-toufang3 '', '' tf-toufang4 '', '' tf-toufang5 '', '' benzhan '', '' promotion_aso100 ''
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ' , '' promotion_qianka '', '' promotion_xiaoyu '', '' promotion_dianru '', '' promotion_malioaso '', '' promotion_malioaso-shequ ''
,'promotion_shike','promotion_julang_jl03','promotion_zuimei') , '' promotion_shike '', '' promotion_julang_jl03 '', '' promotion_zuimei '')
AND first_channel_source_type not like 'promotion\_jf\_%' AND first_channel_source_type not like '' promotion\_jf\_%''
)t1 ) t1
join join
( (
SELECT partition_date,cl_id,page_name,action,params['referrer_link'] as referrer_link SELECT partition_date, cl_id, page_name, action, params['' referrer_link ''] as referrer_link
FROM online.bl_hdfs_maidian_updates FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','') where partition_date = regexp_replace(DATE_SUB(current_date, 1), ''-'', '''')
and action in ('do_search','on_click_navbar_search') and action in ('' do_search '', '' on_click_navbar_search '')
union all
union all SELECT partition_date, cl_id, null as page_name, '' 大家都在看 '' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
SELECT partition_date,cl_id,null as page_name,'大家都在看' as action, null as referrer_link where partition_date = regexp_replace(DATE_SUB(current_date, 1), ''-'', '''')
FROM online.bl_hdfs_maidian_updates AND action = '' on_click_card ''
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','') AND params['' in_page_pos '']='' 猜你喜欢 ''
AND action = 'on_click_card' AND params['' tab_name '']='' 精选 ''
AND params['in_page_pos']='猜你喜欢' AND params['' card_type '']='' search_word ''
AND params['tab_name']='精选' -- AND page_name='' home '' android的page_name为空
AND params['card_type']='search_word' union all
--AND page_name='home' android的page_name为空 SELECT partition_date, cl_id, page_name, '' 热搜词 '' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
union all where partition_date = regexp_replace(DATE_SUB(current_date, 1), ''-'', '''')
AND action = '' on_click_card ''
SELECT partition_date,cl_id,null as page_name,'热搜词' as action, null as referrer_link AND page_name='' welfare_home ''
FROM online.bl_hdfs_maidian_updates AND params['' card_type ''] ='' search_word ''
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','') AND params['' in_page_pos '']='' 大家都在搜 ''
AND action = 'on_click_card' union all
AND page_name='welfare_home' select partition_date, cl_id, null as page_name, '' 详情页高亮词 '' as action, null as referrer_link
AND params['card_type'] ='search_word' from online.bl_hdfs_maidian_updates
AND params['in_page_pos']='大家都在搜' where partition_date = regexp_replace(DATE_SUB(current_date, 1), ''-'', '''')
and action ='' on_click_card ''
union all and params['' card_type '']='' highlight_word ''
) t2
select partition_date,cl_id,null as page_name,'详情页高亮词' as action,null as referrer_link on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id
from online.bl_hdfs_maidian_updates group by t1.partition_date
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','') , t1.device_os_type
and action='on_click_card' , t1.active_type
and params['card_type']='highlight_word' , case when action ='' on_click_navbar_search '' and page_name='' home '' then '' 首页搜索框 ''
)t2 when action ='' on_click_navbar_search '' and page_name='' welfare_home '' then '' 美购首页搜索框 ''
on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id when action ='' on_click_navbar_search '' and page_name='' category '' then '' 品类聚合页搜索框 ''
group by t1.partition_date when action ='' on_click_navbar_search '' and page_name='' welfare_list '' and referrer_link =''["home", "category"]'' then '' 来自品类聚合的美购列表页搜索框 ''
,t1.device_os_type when action ='' on_click_navbar_search '' and page_name='' welfare_list '' and referrer_link =''["welfare_home"]'' then '' 来自美购首页的美购列表页搜索框 ''
,t1.active_type when action in ('' do_search '', '' on_click_navbar_search '')
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框' and page_name in ('' diary_detail '', '' topic_detail '', '' post_detail '', '' user_post_detail ''
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框' , '' doctor_post_detail '', '' question_detail '', '' answer_detail '', '' question_answer_detail '') then '' 内容详情页搜索框 ''
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框' when action ='' 详情页高亮词 '' then '' 详情页高亮词搜索 ''
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框' when action ='' 大家都在看 '' then '' 首页feed大家都在看 ''
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框' when action ='' 热搜词 '' then '' 美购首页热搜词 '' else null end
when action in ('do_search','on_click_navbar_search') having search_entrance is not null
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail' ) t
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框' ) n LATERAL VIEW explode(n.search_entrance) v AS search_entrance
when action ='详情页高亮词' then '详情页高亮词搜索' group by day_id, device_os_type, active_type, v.search_entrance;
when action='大家都在看' then '首页feed大家都在看' \ No newline at end of file
when action='热搜词' then '美购首页热搜词' else null end
having search_entrance is not null
)t
)n
LATERAL VIEW explode(n.search_entrance) v AS search_entrance
group by day_id,device_os_type,active_type,v.search_entrance;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment