update codes

jobs/daily_content_data/step2.job

 #step2.job
 type=command
 dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
-command=curl -X GET http://localhost:8553/api/report/execSql?name=daily_content_data_insert
+command=/home/bi/bi-report/shells/daily_content_data/daily_content_data.sh
\ No newline at end of file

shells/search_related/search_related.sh

@@ -13,7 +13,7 @@ fi
 
 sql="
 
-SET mapreduce.job.queuename=pm;
+SET mapreduce.job.queuename=data;
 SET mapreduce.map.memory.mb=8192;
 SET mapreduce.map.java.opts=-Xmx8000m;
 SET mapreduce.reduce.memory.mb=8192;
@@ -42,8 +42,8 @@ FROM
 				,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
 					  when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
 					  when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '从品类到美购列表'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '从美购首页到美购列表'
+					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
+					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
 					  when action in ('do_search','on_click_navbar_search')
 					       and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
 					       	,'doctor_post_detail','question_detail','answer_detail','question_answer_detail')  then '内容详情页搜索框'
@@ -113,8 +113,8 @@ FROM
 				,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
 					  when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
 					  when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '从品类到美购列表'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '从美购首页到美购列表'
+					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
+					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
 					  when action in ('do_search','on_click_navbar_search')
 					       and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
 					       	,'doctor_post_detail','question_detail','answer_detail','question_answer_detail')  then '内容详情页搜索框'

sqls/search_related_insert/search_click_path_insert.sql

-
-
-INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = #partition_day)
-
-
-SELECT day_id as day_id
-			,device_os_type as device_os_type
-			,active_type as active_type
-			,v.search_entrance as search_entrance
-			,sum(search_pv) as click_search_pv
-			,sum(search_uv) as click_search_uv
-FROM
-(
-	SELECT day_id,device_os_type,active_type,array(search_entrance,'合计') as search_entrance
-			,search_pv,search_uv
-	FROM
-	(
-		SELECT t1.partition_date as day_id
-				,t1.device_os_type
-				,t1.active_type
-				,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
-					  when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
-					  when action in ('do_search','on_click_navbar_search')
-					       and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
-					       	,'doctor_post_detail','question_detail','answer_detail','question_answer_detail')  then '内容详情页搜索框'
-					  when action ='详情页高亮词' then '详情页高亮词搜索'
-					  when action='大家都在看' then '首页feed大家都在看'
-					  when action='热搜词' then '美购首页热搜词' else null end as search_entrance
-				,count(cl_id) as search_pv
-				,count(distinct cl_id) as search_uv
-		FROM
-		(
-			SELECT partition_date,device_os_type
-			,case WHEN active_type = '4'  THEN '老活跃设备'
-				  WHEN active_type  in ('1','2')  then '新增设备' END as active_type
-			,device_id
-			FROM online.ml_device_day_active_status
-			where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			AND active_type in ('1','2','4')
-			and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
-			        ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
-			        ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
-			        ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
-			        ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
-			        ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
-			        ,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
-		    AND first_channel_source_type not like 'promotion\_jf\_%'
-		)t1
-		join
-		(
-			SELECT partition_date,cl_id,page_name,action,params['referrer_link'] as referrer_link
-			FROM online.bl_hdfs_maidian_updates
-			where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			and action in ('do_search','on_click_navbar_search')
-
-			union all
-
-			SELECT partition_date,cl_id,null as page_name,'大家都在看' as action, null as referrer_link
-			FROM online.bl_hdfs_maidian_updates
-			where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			AND action = 'on_click_card'
-			AND params['in_page_pos']='猜你喜欢'
-			AND params['tab_name']='精选'
-			AND params['card_type']='search_word'
-			--AND page_name='home' android的page_name为空
-
-			union all
-
-			SELECT partition_date,cl_id,page_name,'热搜词' as action, null as referrer_link
-			FROM online.bl_hdfs_maidian_updates
-			where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			AND action = 'on_click_card'
-			AND page_name='welfare_home'
-			AND params['card_type'] ='search_word'
-			AND params['in_page_pos']='大家都在搜'
-
-			union all
-
-			select 	partition_date,cl_id,null as page_name,'详情页高亮词' as action,null as referrer_link
-			from online.bl_hdfs_maidian_updates
-			where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			and action='on_click_card'
-			and params['card_type']='highlight_word'
-		)t2
-		on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id
-		group by t1.partition_date
-				,t1.device_os_type
-				,t1.active_type
-				,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
-					  when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
-					  when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
-					  when action in ('do_search','on_click_navbar_search')
-					       and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
-					       	,'doctor_post_detail','question_detail','answer_detail','question_answer_detail')  then '内容详情页搜索框'
-					  when action ='详情页高亮词' then '详情页高亮词搜索'
-					  when action='大家都在看' then '首页feed大家都在看'
-					  when action='热搜词' then '美购首页热搜词' else null end
-		having search_entrance is not null
-	)t
-)n
-LATERAL VIEW explode(n.search_entrance) v AS search_entrance
-group by day_id,device_os_type,active_type,v.search_entrance

sqls/search_related_insert/search_input_type_insert.sql

-INSERT OVERWRITE TABLE pm.tl_pm_search_input_type_d PARTITION (PARTITION_DAY = #partition_day)
-
-SELECT partition_date as day_id
-			,t1.query
-			,t2.input_type
-			,sum(t1.all_search_pv) as search_pv
-	FROM
-	(
-		SELECT
-			partition_date
-			,query
-			,array(case when input_type in ('联想','聚合模块') then 'sug点击' else input_type end,'全部') as input_type
-			,count(click.cl_id) as all_search_pv
-			,count(distinct click.cl_id) as all_search_uv
-		FROM
-		(
-			SELECT cl_id
-				,partition_date
-				,case when params['input_type'] in ('热门','related_search') then '发现' else params['input_type'] end as input_type
-				,params['query'] as query
-			FROM online.bl_hdfs_maidian_updates
-			WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			  AND action in ('do_search','search_result_click_search')
-
-			union all
-
-			SELECT cl_id
-				,partition_date
-				,'首页精选大家都在看' as input_type
-				,params['card_name'] as query
-			FROM online.bl_hdfs_maidian_updates
-			WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			  AND action = 'on_click_card'
-			  AND params['in_page_pos']='猜你喜欢'
-			  AND params['tab_name']='精选'
-			  AND params['card_type']='search_word'
-			  --AND page_name='home' android的page_name为空
-
-			union all
-
-			SELECT cl_id
-				,partition_date
-				,'美购首页热搜词' as input_type
-				,params['card_name'] as query
-			FROM online.bl_hdfs_maidian_updates
-			WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			  AND action = 'on_click_card'
-			  AND page_name='welfare_home'
-			  AND params['card_type'] ='search_word'
-			  AND params['in_page_pos']='大家都在搜'
-
-			  union all
-
-			select 	cl_id
-					,partition_date
-					,'详情页高亮词' as input_type
-					,params['card_name'] as query
-			from online.bl_hdfs_maidian_updates
-			where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-			and int(split(app_version,'\\.')[1]) >= 27
-			and action='on_click_card'
-			and params['card_type']='highlight_word'
-
-		)click
-		JOIN
-		(
-		    SELECT device_id
-		    from online.ml_device_history_detail
-		    WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
-		    and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
-				   ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
-				   ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
-				   ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
-				   ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
-				   ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
-				   ,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
-		    AND first_channel_source_type not like 'promotion\_jf\_%'
-		)dev_channel
-		  on dev_channel.device_id = click.cl_id
-		GROUP BY partition_date,query,case when input_type in ('联想','聚合模块') then 'sug点击' else input_type end
-	)t1
-	LATERAL VIEW explode(t1.input_type) t2 AS input_type
-	GROUP BY t1.partition_date,t1.query,t2.input_type
\ No newline at end of file