Merge branch 'weiyimin' into 'master'

Weiyimin

See merge request !49

pm/clue_daily/report/clue_daily.sql

@@ -10,21 +10,29 @@ SELECT day_id `日期`
 		,valid_msg_num as `有效私信咨询人次-总计`
 		,clue_dev as `授权电话线索设备数-总计`
 		,clue_num as `授权电话线索人次-总计`
-		,call_dev as `拨打电话线索设备数-总计`
-		,call_num as `拨打电话线索人次-总计`
-		,valid_call_dev as `有效电话线索设备数-总计`
-		,valid_call_num as `有效电话线索人次-总计`
+		,call_dev+call_dev_d as `拨打电话线索设备数-总计`
+		,call_num+call_num_d as `拨打电话线索人次-总计`
+		,valid_call_dev+valid_call_dev_d as `有效电话线索设备数-总计`
+		,valid_call_num+valid_call_num_d as `有效电话线索人次-总计`
+		,call_dev as `当天拨打电话线索设备数-总计`
+		,call_num as `当天拨打电话线索人次-总计`
+		,valid_call_dev as `当天有效电话线索设备数-总计`
+		,valid_call_num as `当天有效电话线索人次-总计`
 
-		,msg_dev_4 `私信咨询设备数-4级以上商户`
-		,msg_num_4 `私信咨询人次-4级以上商户`
-		,valid_msg_dev_4 `有效私信咨询设备数-4级以上商户`
-		,valid_msg_num_4 as `有效私信咨询人次-4级以上商户`
-		,clue_dev_4 as `授权电话线索设备数-4级以上商户`
-		,clue_num_4 as `授权电话线索人次-4级以上商户`
-		,call_dev_4 as `拨打电话线索设备数-4级以上商户`
-		,call_num_4 as `拨打电话线索人次-4级以上商户`
-		,valid_call_dev_4 as `有效电话线索设备数-4级以上商户`
-		,valid_call_num_4 as `有效电话线索人次-4级以上商户`
+		,msg_dev_3 `私信咨询设备数-3级以上商户`
+		,msg_num_3 `私信咨询人次-3级以上商户`
+		,valid_msg_dev_3 `有效私信咨询设备数-3级以上商户`
+		,valid_msg_num_3 as `有效私信咨询人次-3级以上商户`
+		,clue_dev_3 as `授权电话线索设备数-3级以上商户`
+		,clue_num_3 as `授权电话线索人次-3级以上商户`
+		,call_dev_3+call_dev_3_d as `拨打电话线索设备数-3级以上商户`
+		,call_num_3+valid_call_num_3 as `拨打电话线索人次-3级以上商户`
+		,valid_call_dev_3+valid_call_dev_3_d as `有效电话线索设备数-3级以上商户`
+		,valid_call_num_3+valid_call_num_3_d as `有效电话线索人次-3级以上商户`
+		,call_dev_3 as `当天拨打电话线索设备数-3级以上商户`
+		,call_num_3 as `当天拨打电话线索人次-3级以上商户`
+		,valid_call_dev_3 as `当天有效电话线索设备数-3级以上商户`
+		,valid_call_num_3 as `当天有效电话线索人次-3级以上商户`
 
 		,msg_dev_kyc `私信咨询设备数-kyc`
 		,msg_num_kyc `私信咨询人次-kyc`
@@ -32,10 +40,14 @@ SELECT day_id `日期`
 		,valid_msg_num_kyc as `有效私信咨询人次-kyc`
 		,clue_dev_kyc as `授权电话线索设备数-kyc`
 		,clue_num_kyc as `授权电话线索人次-kyc`
-		,call_dev_kyc as `拨打电话线索设备数-kyc`
-		,call_num_kyc as `拨打电话线索人次-kyc`
-		,valid_call_dev_kyc as `有效电话线索设备数-kyc`
-		,valid_call_num_kyc as `有效电话线索人次-kyc`
+		,call_dev_kyc+call_dev_kyc_d as `拨打电话线索设备数-kyc`
+		,call_num_kyc+call_num_kyc_d as `拨打电话线索人次-kyc`
+		,valid_call_dev_kyc+valid_call_dev_kyc_d as `有效电话线索设备数-kyc`
+		,valid_call_num_kyc+valid_call_num_kyc_d as `有效电话线索人次-kyc`
+		,call_dev_kyc as `当天拨打电话线索设备数-kyc`
+		,call_num_kyc as `当天拨打电话线索人次-kyc`
+		,valid_call_dev_kyc as `当天有效电话线索设备数-kyc`
+		,valid_call_num_kyc as `当天有效电话线索人次-kyc`
 
 		,msg_dev_qa `私信咨询设备数-需求自测`
 		,msg_num_qa `私信咨询人次-需求自测`
@@ -43,10 +55,14 @@ SELECT day_id `日期`
 		,valid_msg_num_qa as `有效私信咨询人次-需求自测`
 		,clue_dev_qa as `授权电话线索设备数-需求自测`
 		,clue_num_qa as `授权电话线索人次-需求自测`
-		,call_dev_qa as `拨打电话线索设备数-需求自测`
-		,call_num_qa as `拨打电话线索人次-需求自测`
-		,valid_call_dev_qa as `有效电话线索设备数-需求自测`
-		,valid_call_num_qa as `有效电话线索人次-需求自测`
+		,call_dev_qa+call_dev_qa_d as `拨打电话线索设备数-需求自测`
+		,call_num_qa+call_num_qa_d as `拨打电话线索人次-需求自测`
+		,valid_call_dev_qa+valid_call_dev_qa_d as `有效电话线索设备数-需求自测`
+		,valid_call_num_qa+valid_call_num_qa_d as `有效电话线索人次-需求自测`
+		,call_dev_qa_d as `当天拨打电话线索设备数-需求自测`
+		,call_num_qa_d as `当天拨打电话线索人次-需求自测`
+		,valid_call_dev_qa_d as `当天有效电话线索设备数-需求自测`
+		,valid_call_num_qa_d as `当天有效电话线索人次-需求自测`
 FROM pm.tl_pm_userclue_d
 where partition_day in ('20200901','202001001','20201101','20201201','20210101',regexp_replace(date_sub(current_date,1),'-',''))
 order by `日期`,`系统`,`新老`,`是否灰度`

pm/meigou_detail_page/report/meigou_detail_page_dispense_pv.sql

@@ -41,7 +41,7 @@ FROM
 	SELECT t1.partition_date as `日期`
 			,t1.device_type as `设备类型`
 			,t1.active_type as `活跃类型`
-			,t2.channel as `渠道`
+			,t1.channel as `渠道`
 			,count(case when type='美购详情页' then cl_id end ) as `美购详情页浏览pv`
 			,count(case when type='加购物车' then cl_id end ) as `加购物车点击pv`
 			,count(case when type='评价列表页' then cl_id end ) as `评价列表页点击pv`
@@ -71,6 +71,7 @@ FROM
                 AND partition_date<regexp_replace((current_date),'-','')
 			    AND page_name = 'welfare_detail'
 				AND action='page_view'
+				and params['is_first']=1
 			)pv
 			LEFT JOIN
 			( -- 去掉疑似机构刷量的PV和UV
@@ -265,30 +266,37 @@ FROM
 		)click
 		JOIN
 		(
-				SELECT partition_date,device_os_type as device_type,m.device_id
-                ,case WHEN active_type = '4'  THEN '老活跃设备'
-                      WHEN active_type  in ('1','2')  then '新增设备' END as active_type
-                ,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
-                                    or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
-                                    or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
-                                    or first_channel_source_type like '%mailuo%' THEN '可疑'
-                            WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
-                                    or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
-                                    or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
-                                    THEN 'AI'  ELSE '其他' END , '合计') as channel
-                FROM online.ml_device_day_active_status m
+			select partition_date,device_type,device_id,active_type,t.channel
+             from
+             (
+                select partition_date,device_os_type as device_type,m.device_id
+                    ,case when active_type = '4'  then '老活跃设备'
+                      when active_type  in ('1','2')  then '新增设备' end as active_type
+                ,array(case when (a.device_id is not null or b.device_id is not null
+                                or first_channel_source_type like '%xinyouxingkong%'
+                                or first_channel_source_type like '%jingmeng%'
+                                or first_channel_source_type like '%longyuzhixing%'
+                                or first_channel_source_type like '%mailuo%') then '渠道可疑'
+                        when tmp.is_ai_channel='true' then 'AI' else '其他' end , '合计') as channel
+                from online.ml_device_day_active_status m
                     LEFT JOIN
-                    (SELECT col1,col2  --col1:子渠道，col2：是否属于AI，col3:标识
-                       FROM pm.tl_pm_ydl
-                       WHERE col3='0204_danlei_channel')tmp
-                    on first_channel_source_type=tmp.col1
+                    (select code,is_spam,is_ai_channel,partition_day
+                        from DIM.DIM_AI_CHANNEL_ZP_NEW
+                        where partition_day >=regexp_replace(DATE_SUB(current_date,90) ,'-','') and partition_day<regexp_replace((current_date),'-',''))tmp
+                    on first_channel_source_type=tmp.code and m.partition_date=tmp.partition_day
                     LEFT JOIN
-                    (SELECT DISTINCT device_id
-                       FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
-                       WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
+                    (select distinct device_id
+                       from al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
+                       where partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
                     ON m.device_id = a.device_id
+                    LEFT JOIN
+                    (select device_id,day_id
+                        from pm.tl_pm_channel_d
+                        where partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
+                        group by device_id,day_id)b
+                    ON m.device_id = b.device_id AND m.partition_date = b.day_id
                 where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
-    			AND partition_date < regexp_replace((current_date),'-','')
+                AND partition_date < regexp_replace((current_date),'-','')
                 AND active_type in ('1','2','4')
                 and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
                 ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
@@ -298,12 +306,13 @@ FROM
                 ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
                 ,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
                 AND first_channel_source_type not like 'promotion\_jf\_%'
+            )a
+            LATERAL VIEW explode(a.channel) t AS channel
 		)mas
 		ON click.cl_id=mas.device_id
 		  AND click.partition_date=mas.partition_date
 	)t1
-	LATERAL VIEW explode(t1.channel) t2 AS channel
-	GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
+	GROUP BY t1.partition_date,t1.device_type,t1.active_type,t1.channel
 )T1
 ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
 

pm/sign_daily/en-cn.properties

+sign_daily=签到用户质量日报
\ No newline at end of file

pm/sign_daily/etl/create_sign_daily.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 签到用户质量日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_sign_deepaction_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ user_type string comment '{"chs_name":"用户类型","description":"","etl":"","value":"","remark":""}',
+ active_num bigint comment '{"chs_name":"活跃设备数","description":"","etl":"","value":"","remark":""}',
+ wel_pv bigint comment '{"chs_name":"商品核心页浏览pv","description":"","etl":"","value":"","remark":""}',
+ wel_uv bigint comment '{"chs_name":"商品核心页浏览uv","description":"","etl":"","value":"","remark":""}',
+ clue_pv bigint comment '{"chs_name":"有效线索人次","description":"","etl":"","value":"","remark":""}',
+ clue_uv bigint comment '{"chs_name":"有效线索人次","description":"","etl":"","value":"","remark":""}'
+)comment '签到用户质量日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/sign_daily/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/sign_daily/job/step1_10.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file

pm/sign_daily/job/step1_11.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file

pm/sign_daily/job/step1_12.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file

pm/sign_daily/job/step1_13.job

+#step1_13.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/sign_daily/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/sign_daily/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/sign_daily/job/step1_4.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file

pm/sign_daily/job/step1_5.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_user_sign_v2_view
\ No newline at end of file

pm/sign_daily/job/step1_6.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file

pm/sign_daily/job/step1_7.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file

pm/sign_daily/job/step1_8.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file

pm/sign_daily/job/step1_9.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file

pm/sign_daily/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
+command=sh /home/bi/bi-report/lib/shell/hive sign_daily
\ No newline at end of file

pm/sign_daily/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/sign_daily/zhaoyang@igengmei.com,zhangwen@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file

pm/sign_daily/report/sign_daily.sql

+SELECT day_id as `日期`
+        ,round((sum(CASE WHEN device_os_type = 'ios' AND user_type='非首日签到用户' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'ios' and user_type='非首日签到用户'  THEN active_num END))/(sum(CASE WHEN device_os_type = 'ios' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'ios' THEN active_num END)),2) as `单个iOS非首日签到贡献商品核心页PV/单个iOS老贡献商品核心页PV`
+        ,round((sum(CASE WHEN device_os_type = 'ios' AND user_type='非首日签到用户' THEN clue_pv END)/sum(CASE WHEN device_os_type = 'ios' AND user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'ios'  THEN clue_pv END)/sum(CASE WHEN device_os_type = 'ios' THEN active_num END)),2) as `单个iOS非首日签到贡献有效线索人次/单个iOS老贡献有效线索人次`
+        ,round((sum(CASE WHEN device_os_type = 'android' AND user_type='非首日签到用户' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'android' and user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'android' THEN wel_pv END)/sum(CASE WHEN device_os_type = 'android' THEN active_num END)),2) as `单个android非首日签到贡献商品核心页PV/单个android老贡献商品核心页PV`
+        ,round((sum(CASE WHEN device_os_type = 'android' AND user_type='非首日签到用户' THEN clue_pv END)/sum(CASE WHEN device_os_type = 'android' AND user_type='非首日签到用户' THEN active_num END))/(sum(CASE WHEN device_os_type = 'android'  THEN clue_pv END)/sum(CASE WHEN device_os_type = 'android' THEN active_num END)),2) as `单个android非首日签到贡献有效线索人次/单个android老贡献有效线索人次`
+FROM pm.tl_pm_sign_deepaction_d
+where partition_day in ('20200901','202001001','20201101','20201201','20210101',regexp_replace(date_sub(current_date,1),'-',''))
+group by day_id
+order by `日期`