Merge branch 'yindanlei' into 'master'

Yindanlei

See merge request !58

pm/icon_function_day/en-cn.properties

+icon_function_day=首页icon及功能区数据
\ No newline at end of file

pm/icon_function_day/etl/create_icon_function_day.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 首页icon及功能区数据
+--*业务名称: pm
+--*输入数据:
+--*作者: yindanlei@igengmei.com
+--*更新时间:2020-09-23
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_icon_function_d
+(
+ date string comment '{"chs_name":"日期","description":"","etl":"","value":"","remark":""}',
+ grey_type string comment '{"chs_name":"灰度","description":"","etl":"","value":"","remark":""}',
+ dau BIGINT comment '{"chs_name":"dau","description":"","etl":"","value":"","remark":""}',
+ dau_733 BIGINT comment '{"chs_name":"7.33版本及以上dau","description":"","etl":"","value":"","remark":""}',
+ gengxinlv string comment '{"chs_name":"版本更新率","description":"","etl":"","value":"","remark":""}',
+ quanzhong_dau string comment '{"chs_name":"权重日活","description":"","etl":"","value":"","remark":""}',
+ ciliu string comment '{"chs_name":"次留率","description":"","etl":"","value":"","remark":""}',
+ function_click_pv_rate string comment '{"chs_name":"功能入口点击率pv","description":"","etl":"","value":"","remark":""}',
+ function_click_uv_rate string comment '{"chs_name":"功能入口点击率uv","description":"","etl":"","value":"","remark":""}',
+ icon_click_pv_rate string comment '{"chs_name":"icon点击率pv","description":"","etl":"","value":"","remark":""}',
+ icon_click_uv_rate string comment '{"chs_name":"icon点击率uv","description":"","etl":"","value":"","remark":""}',
+ refer_function_wel_pv_rate string comment '{"chs_name":"通过功能区到商详页pv占比","description":"","etl":"","value":"","remark":""}',
+ refer_function_wel_uv_rate string comment '{"chs_name":"通过功能区到商详页uv占比","description":"","etl":"","value":"","remark":""}',
+ refer_icon_wel_pv_rate string comment '{"chs_name":"通过icon到商详页pv占比","description":"","etl":"","value":"","remark":""}',
+ refer_icon_wel_uv_rate string comment '{"chs_name":"通过icon到商详页uv占比","description":"","etl":"","value":"","remark":""}',
+ click_function_pv string comment '{"chs_name":"功能入口点击pv","description":"","etl":"","value":"","remark":""}',
+ click_function_uv string comment '{"chs_name":"功能入口点击uv","description":"","etl":"","value":"","remark":""}',
+ exp_function_pv string comment '{"chs_name":"功能入口曝光pv","description":"","etl":"","value":"","remark":""}',
+ exp_function_uv string comment '{"chs_name":"功能入口曝光uv","description":"","etl":"","value":"","remark":""}',
+ click_icon_pv string comment '{"chs_name":"icon点击pv","description":"","etl":"","value":"","remark":""}',
+ click_icon_uv string comment '{"chs_name":"icon点击uv","description":"","etl":"","value":"","remark":""}',
+ exp_icon_pv string comment '{"chs_name":"icon曝光pv","description":"","etl":"","value":"","remark":""}',
+ exp_icon_uv string comment '{"chs_name":"icon曝光uv","description":"","etl":"","value":"","remark":""}',
+ wel_pv string comment '{"chs_name":"商详页浏览pv","description":"","etl":"","value":"","remark":""}',
+ wel_uv string comment '{"chs_name":"商详页浏览uv","description":"","etl":"","value":"","remark":""}',
+ wel_function_pv string comment '{"chs_name":"通过功能区到商详页pv","description":"","etl":"","value":"","remark":""}',
+ wel_function_uv string comment '{"chs_name":"通过功能区到商详页uv","description":"","etl":"","value":"","remark":""}',
+ wel_icon_pv string comment '{"chs_name":"通过icon到商详页pv","description":"","etl":"","value":"","remark":""}',
+ wel_icon_uv string comment '{"chs_name":"通过icon到商详页uv","description":"","etl":"","value":"","remark":""}',
+ neirong_icon_pv string comment '{"chs_name":"通过icon到内容页pv","description":"","etl":"","value":"","remark":""}',
+ neirong_icon_uv string comment '{"chs_name":"通过icon到内容页uv","description":"","etl":"","value":"","remark":""}',
+ all_two_pv string comment '{"chs_name":"通过icon总二跳pv","description":"","etl":"","value":"","remark":""}',
+ all_two_uv string comment '{"chs_name":"通过icon总二跳uv","description":"","etl":"","value":"","remark":""}'
+ )comment '首页icon及功能区数据'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/icon_function_day/etl/icon_function_day.sql

+SET mapreduce.job.queuename=data;
+SET mapreduce.map.memory.mb=8192;
+SET mapreduce.map.java.opts=-Xmx8000m;
+SET mapreduce.reduce.memory.mb=8192;
+SET mapreduce.reduce.java.opts=-Xmx8000m;
+set hive.auto.convert.join=true;
+SET mapred.reduce.tasks=20;
+SET role admin;
+
+ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
+CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
+CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
+
+INSERT OVERWRITE TABLE pm.tl_pm_icon_function_d PARTITION (PARTITION_DAY = ${partition_day})
+
+SELECT t1.partition_date as date
+		,t1.grey_type as grey_type
+		,count(distinct t1.device_id) as dau
+		,nvl(count(distinct t2.device_id),0) as  dau_733
+		,nvl(concat(round(count(distinct t2.device_id)/count(distinct t1.device_id)*100,2),'%'),0) as gengxinlv
+		,round(count(DISTINCT CASE WHEN active_type = '老活' AND device_os_type = 'android' AND channel_type = 'AI' THEN t2.device_id END)*0.14
+              +count(DISTINCT CASE WHEN active_type = '老活' AND device_os_type = 'android' AND channel_type = '医美' THEN t2.device_id END)*0.64
+              +count(DISTINCT CASE WHEN active_type = '新增' AND device_os_type = 'android' AND channel_type = 'AI' THEN t2.device_id END)*0.08
+              +count(DISTINCT CASE WHEN active_type = '新增' AND device_os_type = 'android' AND channel_type = '医美' THEN t2.device_id END)*0.19
+              +count(DISTINCT CASE WHEN active_type = '老活' AND device_os_type = 'ios' AND channel_type = 'AI' THEN t2.device_id END)*0.32
+              +count(DISTINCT CASE WHEN active_type = '老活' AND device_os_type = 'ios' AND channel_type = '积分墙' THEN t2.device_id END)*0.28
+              +count(DISTINCT CASE WHEN active_type = '老活' AND device_os_type = 'ios' AND channel_type = '医美' THEN t2.device_id END)*1.00
+              +count(DISTINCT CASE WHEN active_type = '新增' AND device_os_type = 'ios' AND channel_type = 'AI' THEN t2.device_id END)*0.19
+              +count(DISTINCT CASE WHEN active_type = '新增' AND device_os_type = 'ios' AND channel_type = '积分墙' THEN t2.device_id END)*0.03
+              +count(DISTINCT CASE WHEN active_type = '新增' AND device_os_type = 'ios' AND channel_type = '医美' THEN t2.device_id END)*0.57,0) as  quanzhong_dau
+		,nvl(concat(round(count(distinct t3.device_id)/count(distinct t2.device_id)*100,2),'%'),0) as ciliu
+		,nvl(concat(round(sum(t4.click_function_pv)/sum(t4.exp_function_pv)*100,2),'%'),0) as function_click_pv_rate
+		,nvl(concat(round(count(distinct case when t4.click_function_pv>0 then t4.cl_id end)/count(case when t4.exp_function_pv>0 then t4.cl_id end)*100,2),'%'),0) as function_click_uv_rate
+		,nvl(concat(round(sum(t5.click_icon_pv)/sum(t5.exp_icon_pv)*100,2),'%'),0) as icon_click_pv_rate
+		,nvl(concat(round(count(distinct case when t5.click_icon_pv>0 then t5.cl_id end)/count(case when t5.exp_icon_pv>0 then t5.cl_id end)*100,2),'%'),0) as icon_click_uv_rate
+		,nvl(concat(round(sum(t7.wel_function_pv)/sum(t7.wel_pv)*100,2),'%'),0) as refer_function_wel_pv_rate
+		,nvl(concat(round(count(case when t7.wel_function_pv>0 then t7.cl_id end)/count(case when t7.wel_pv>0 then t7.cl_id end)*100,2),'%'),0) as refer_function_wel_uv_rate
+		,nvl(concat(round(sum(t6.wel_pv)/sum(t7.wel_pv)*100,2),'%'),0) as refer_icon_wel_pv_rate
+		,nvl(concat(round(count(case when t6.wel_pv>0 then t6.cl_id end)/count(case when t7.wel_pv>0 then t7.cl_id end)*100,2),'%'),0) as refer_icon_wel_uv_rate
+		,sum(t4.click_function_pv) as click_function_pv
+		,count(distinct case when t4.click_function_pv>0 then t4.cl_id end) as click_function_uv
+		,sum(t4.exp_function_pv) as exp_function_pv
+		,count(case when t4.exp_function_pv>0 then t4.cl_id end) as exp_function_uv
+
+		,sum(t5.click_icon_pv) as click_icon_pv
+		,count(distinct case when t5.click_icon_pv>0 then t5.cl_id end) as click_icon_uv
+		,sum(t5.exp_icon_pv) as exp_icon_pv
+		,count(case when t5.exp_icon_pv>0 then t5.cl_id end) as exp_icon_uv
+
+		,nvl(sum(t7.wel_pv),0) as wel_pv
+		,nvl(count(case when t7.wel_pv>0 then t7.cl_id end),0) as wel_uv
+		,nvl(sum(t7.wel_function_pv),0) as wel_function_pv
+		,nvl(count(case when t7.wel_function_pv>0 then t7.cl_id end),0) as wel_function_uv
+
+		,nvl(sum(t6.wel_pv),0) as wel_icon_pv
+		,nvl(count(case when t6.wel_pv>0 then t6.cl_id end),0) as wel_icon_uv
+
+		,nvl(sum(t6.neirong_pv),0) as neirong_icon_pv
+		,nvl(count(case when t6.neirong_pv>0 then t6.cl_id end),0) as neirong_icon_uv
+
+		,nvl(sum(t6.all_pv),0) as all_two_pv
+		,nvl(count(case when t6.all_pv>0 then t6.cl_id end),0) as all_two_uv
+
+from
+(
+	SELECT
+	concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+	,m.device_id,device_os_type
+	,case when active_type in ('1','2') then '新增'
+          when active_type ='4' then '老活' end  as active_type
+    ,CASE WHEN is_ai_channel = 'true' THEN 'AI'
+           WHEN first_channel_source_type in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+             ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+             ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+             ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+             ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+             ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+             ,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')  then '积分墙' ELSE '医美' END as channel_type
+    ,case when (partition_date<='20200916' AND substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in(0,1,2,3,4,5,6,7,8,9))
+    			or (partition_date>='20200917' AND partition_date <='20200922' AND substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15)) then '灰度'
+    	  when (partition_date='20200916' AND substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in(10,11,12,13,14,15))
+    	  		or (partition_date='20200922' AND substr(convup(setencryption(m.device_id,'sha-1'),16,10),-2,2)%20 in(16,17,18,19)) then '其他'
+    	  WHEN partition_date>='20200923' then '灰度全量' else '非灰' end as grey_type
+	FROM online.ml_device_day_active_status m
+		LEFT JOIN
+        (SELECT code,is_ai_channel,partition_day
+         FROM DIM.DIM_AI_CHANNEL_ZP_NEW
+         WHERE partition_day>= '20200910' AND partition_day <= REGEXP_REPLACE(date_sub(current_date(),1),'-','')) tmp
+        ON  m.partition_date=tmp.partition_day AND first_channel_source_type=code
+	where partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+	AND active_type in ('1','2','4')
+)t1
+left join
+(
+	SELECT partition_date,device_id
+    FROM
+    (
+        select concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+        		,app_version_list,device_id
+        from online.ml_device_updates
+        where partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+    )a
+    LATERAL VIEW EXPLODE (app_version_list) t as app_version
+    where int(split(app_version,'\\.')[1]) >= 33
+    group by partition_date,device_id
+)t2
+on t1.partition_date=t2.partition_date and t1.device_id=t2.device_id
+left join
+(
+	SELECT device_id
+    ,concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
+    FROM online.ml_device_day_active_status
+    WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+)t3
+on t2.device_id=t3.device_id and date_add(t2.partition_date,1)=t3.partition_date
+LEFT JOIN
+(
+	SELECT t2.partition_date,t2.cl_id
+			,sum(t2.pv) as exp_function_pv
+			,sum(t3.pv) as click_function_pv
+	from
+	(--精准曝光，卡片id和session_id去重
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+				cl_id,
+				count(distinct time_str) as pv
+		from online.ml_community_precise_exposure_detail
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
+		AND is_exposure = '1'  ----精准曝光
+		AND page_name ='home'
+		AND (card_content_type ='function_entrance' or card_type='function_entrance')
+		group by partition_date,
+				cl_id
+	)t2
+	LEFT JOIN
+	(--卡片，卡片id和session_id去重
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+		      cl_id,
+		      count(distinct time_str) as pv
+		from online.bl_hdfs_maidian_updates
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND ((action='on_click_card' and params['card_type']='function_entrance') or action='on_click_function')
+		AND params['page_name'] ='home'
+		GROUP BY partition_date,
+		      cl_id
+	)t3
+	on t2.partition_date=t3.partition_date
+	    and t2.cl_id=t3.cl_id
+	group by t2.partition_date,t2.cl_id
+)t4
+on t2.partition_date =t4.partition_date and t2.device_id=t4.cl_id
+
+LEFT JOIN
+(
+	SELECT t2.partition_date,t2.cl_id
+			,sum(t2.pv) as exp_icon_pv
+			,sum(t3.pv) as click_icon_pv
+	from
+	(--精准曝光，卡片id和session_id去重
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+				cl_id,
+				count(distinct time_str) as pv
+		from online.ml_community_precise_exposure_detail
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
+		AND is_exposure = '1'  ----精准曝光
+		AND card_type = 'icon'
+    	AND page_name = 'home'
+		group by partition_date,
+				cl_id
+	)t2
+	LEFT JOIN
+	(--卡片，卡片id和session_id去重
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+		      cl_id,
+		      count(distinct time_str) as pv
+		from online.bl_hdfs_maidian_updates
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND ((action = 'on_click_card' AND params['card_type'] = 'icon' AND page_name = 'home')
+            or (action = 'home_click_section'))
+		GROUP BY partition_date,
+		      cl_id
+	)t3
+	on t2.partition_date=t3.partition_date
+	and t2.cl_id=t3.cl_id
+	group by t2.partition_date,t2.cl_id
+)t5
+on t2.partition_date =t5.partition_date and t2.device_id=t5.cl_id
+
+LEFT JOIN
+(
+	SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+			cl_id,
+			count(distinct CASE WHEN page_name='welfare_detail' THEN time_str END) as wel_pv,
+			count(distinct CASE WHEN page_name in ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','video_steep','article_detail'
+	                    ,'wiki_detail','product_detail','wiki_brand','wiki_collect','question_answer_detail') THEN time_str END) as neirong_pv,
+			count(distinct time_str) as all_pv
+	FROM online.bl_hdfs_maidian_updates
+	WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+	AND action = 'page_view'
+    AND page_name in ('welfare_detail','diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','video_steep','article_detail'
+	                    ,'wiki_detail','product_detail','wiki_brand','wiki_collect','question_answer_detail','organization_detail','expert_detail','welfare_list')
+    AND params['referrer'] = 'category'
+	group by partition_date,
+			cl_id
+)t6
+ON t2.partition_date=t6.partition_date AND t2.device_id=t6.cl_id
+
+LEFT JOIN
+(
+	SELECT t2.partition_date,t2.cl_id
+			,sum(t2.pv) as wel_pv
+			,sum(t3.pv) as wel_function_pv
+	from
+	(
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+				cl_id,
+				count(distinct time_str) as pv
+		from online.bl_hdfs_maidian_updates
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND page_name ='welfare_detail'
+		and action='page_view'
+		group by partition_date,
+				cl_id
+	)t2
+	left join
+	(
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+				cl_id,
+				count(distinct time_str) as pv
+		from online.bl_hdfs_maidian_updates
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND page_name ='welfare_detail'
+		and action='page_view'
+		and params['referrer_link'] like '%[%'
+		and json_split(params['referrer_link'])[0] in ('home','')
+		and json_split(params['referrer_link'])[1] in ('face_scan','new_sign','consultant_conversation_detail','video_face_consultation','ai_conversation_detail')
+		group by partition_date,
+				cl_id
+
+		union all
+
+		SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,
+				cl_id,
+				count(distinct time_str) as pv
+		from online.bl_hdfs_maidian_updates
+		WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+		AND page_name ='welfare_detail'
+		and action='page_view'
+		and params['referrer_link'] like '%[%'
+		and json_split(params['referrer_link'])[0] in ('home','')
+		and referrer='custom_special'
+		and params['referrer_id']='605'
+		group by partition_date,
+				cl_id
+	)t3
+	on t2.partition_date=t3.partition_date and t2.cl_id=t3.cl_id
+	group by t2.partition_date,t2.cl_id
+)t7
+on t2.partition_date =t7.partition_date and t2.device_id=t7.cl_id
+LEFT JOIN
+(
+    SELECT distinct device_id
+    FROM dim.dim_device_user_staff   --去除内网用户
+)a
+    on t2.device_id=a.device_id
+LEFT JOIN
+(
+    SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,device_id
+    FROM
+    (--找出user_id当天活跃的第一个设备id
+        SELECT user_id,partition_date,
+                if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
+          FROM online.ml_user_updates
+          WHERE partition_date>='20200910' AND partition_date<=REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+    )t1
+    JOIN
+    (  --医生账号
+        SELECT distinct user_id
+        FROM online.tl_hdfs_doctor_view
+        WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+
+        --马甲账号/模特用户
+        UNION ALL
+        SELECT user_id
+        FROM ml.ml_c_ct_ui_user_dimen_d
+        WHERE partition_day = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+        AND (is_puppet = 'true' or is_classifyuser = 'true')
+
+        UNION ALL
+        --公司内网覆盖用户
+        select distinct user_id
+        from dim.dim_device_user_staff
+
+        UNION ALL
+        --登陆过医生设备
+        SELECT distinct t1.user_id
+        FROM
+        (
+            SELECT user_id, v.device_id as device_id
+            FROM online.ml_user_history_detail
+                LATERAL VIEW EXPLODE(device_history_list) v AS device_id
+            WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+        )t1
+        JOIN
+        (
+            SELECT device_id
+            FROM online.ml_device_history_detail
+            WHERE partition_date = REGEXP_REPLACE(date_sub(current_date(),1),'-','')
+            AND is_login_doctor = '1'
+        )t2
+            ON t1.device_id = t2.device_id
+    )t2
+    on t1.user_id=t2.user_id
+    group by partition_date,device_id
+)dev
+on t2.partition_date=dev.partition_date and t2.device_id=dev.device_id
+WHERE (a.device_id is NULL or a.device_id ='')
+AND (dev.device_id is NULL or dev.device_id ='')
+group by t1.partition_date,t1.grey_type
+order by date desc
+
+
+
+
+
+
+
+
+

pm/icon_function_day/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/fangan_day/job/step1_15.job → pm/icon_function_day/job/step1_10.job

-#step1_15.job
+#step1_10.job
 type=command
 command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file

pm/fangan_day/job/step1_16.job → pm/icon_function_day/job/step1_2.job

-#step1_16.job
+#step1_2.job
 type=command
 command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive dim DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file

pm/fangan_day/job/step1_3.job → pm/icon_function_day/job/step1_3.job

 #step1_3.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file

pm/icon_function_day/job/step1_4.job

+#step1_4.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file

pm/icon_function_day/job/step1_5.job

+#step1_5.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/icon_function_day/job/step1_6.job

+#step1_6.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/icon_function_day/job/step1_7.job

+#step1_7.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file

pm/icon_function_day/job/step1_8.job

+#step1_8.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file

pm/icon_function_day/job/step1_9.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file

pm/icon_function_day/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10
+command=sh /home/bi/bi-report/lib/shell/hive icon_function_day
\ No newline at end of file

pm/icon_function_day/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/icon_function_day/zhangfenglin@igengmei.com,cp-sunyinghe@igengmei.com,duanyingrong@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file

pm/icon_function_day/report/icon_function_day.sql

+--首页icon及功能区数据
+
+SELECT
+		date AS `日期`
+		,grey_type AS `灰度`
+		,dau AS `dau`
+		,dau_733 AS `7.33版本及以上dau`
+		,gengxinlv AS `版本更新率`
+		,quanzhong_dau AS `权重日活`
+		,ciliu AS `次留率`
+		,function_click_pv_rate AS `功能入口点击率pv`
+		,function_click_uv_rate AS `功能入口点击率uv`
+		,icon_click_pv_rate AS `icon点击率pv`
+		,icon_click_uv_rate AS `icon点击率uv`
+		,refer_function_wel_pv_rate AS `通过功能区到商详页pv占比`
+		,refer_function_wel_uv_rate AS `通过功能区到商详页uv占比`
+		,refer_icon_wel_pv_rate AS `通过icon到商详页pv占比`
+		,refer_icon_wel_uv_rate AS `通过icon到商详页uv占比`
+		,click_function_pv AS `功能入口点击pv`
+		,click_function_uv AS `功能入口点击uv`
+		,exp_function_pv AS `功能入口曝光pv`
+		,exp_function_uv AS `功能入口曝光uv`
+		,click_icon_pv AS `icon点击pv`
+		,click_icon_uv AS `icon点击uv`
+		,exp_icon_pv AS `icon曝光pv`
+		,exp_icon_uv AS `icon曝光uv`
+		,wel_pv AS `商详页浏览pv`
+		,wel_uv AS `商详页浏览uv`
+		,wel_function_pv AS `通过功能区到商详页pv`
+		,wel_function_uv AS `通过功能区到商详页uv`
+		,wel_icon_pv AS `通过icon到商详页pv`
+		,wel_icon_uv AS `通过icon到商详页UV`
+		,neirong_icon_pv AS `通过icon到内容页pv`
+		,neirong_icon_uv AS `通过icon到内容页uv`
+		,all_two_pv AS `通过icon总二跳pv`
+		,all_two_uv AS `通过icon总二跳uv`
+FROM pm.tl_pm_icon_function_d
+where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')