Merge branch 'master' of http://git.wanmeizhensuo.com/data/bi-report into yindanlei

.idea/misc.xml

@@ -16,5 +16,8 @@
   <component name="JavaScriptSettings">
     <option name="languageLevel" value="ES6" />
   </component>
+  <component name="ProjectPlainTextFileTypeManager">
+    <file url="file://$PROJECT_DIR$/pm/clue_daily/report/ai_qa.sql" />
+  </component>
   <component name="ProjectRootManager" version="2" languageLevel="JDK_13" default="false" project-jdk-name="Python 2.7 (shell-tools)" project-jdk-type="Python SDK" />
 </project>
\ No newline at end of file

pm/clue_daily/en-cn.properties

+clue_daily=线索日报（用户侧）
\ No newline at end of file

pm/clue_daily/etl/create_clue_daily.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 线索日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_userclue_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
+ channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
+ dau bigint comment '{"chs_name":"dau(不去除积分墙)","description":"","etl":"","value":"","remark":""}',
+ user_num bigint comment '{"chs_name":"注册用户","description":"","etl":"","value":"","remark":""}',
+ phone_user_num bigint comment '{"chs_name":"手机号注册用户","description":"","etl":"","value":"","remark":""}',
+ total_clue_dev bigint comment '{"chs_name":"总咨询设备","description":"","etl":"","value":"","remark":""}',
+ total_clue_num bigint comment '{"chs_name":"总线索","description":"","etl":"","value":"","remark":""}',
+ total_valid_dev bigint comment '{"chs_name":"总有效咨询设备","description":"","etl":"","value":"","remark":""}',
+ total_valid_num bigint comment '{"chs_name":"总有效线索","description":"","etl":"","value":"","remark":""}',
+ msg_dev bigint comment '{"chs_name":"私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
+ msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev bigint comment '{"chs_name":"有效私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num bigint comment '{"chs_name":"有效私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
+ clue_dev bigint comment '{"chs_name":"授权电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ clue_num bigint comment '{"chs_name":"授权电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+ call_dev bigint comment '{"chs_name":"当天拨打电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ call_num bigint comment '{"chs_name":"当天拨打电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev bigint comment '{"chs_name":"当天有效电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ valid_call_num bigint comment '{"chs_name":"当天有效电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+ call_dev_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ call_num_d bigint comment '{"chs_name":"非当天拨打电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_d bigint comment '{"chs_name":"非当天有效电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_d bigint comment '{"chs_name":"非当天有效电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+
+ msg_dev_3 bigint comment '{"chs_name":"私信咨询设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ msg_num_3 bigint comment '{"chs_name":"私信咨询人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev_3 bigint comment '{"chs_name":"有效私信咨询设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num_3 bigint comment '{"chs_name":"有效私信咨询人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ clue_dev_3 bigint comment '{"chs_name":"授权电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ clue_num_3 bigint comment '{"chs_name":"授权电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ call_dev_3 bigint comment '{"chs_name":"当天拨打电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ call_num_3 bigint comment '{"chs_name":"当天拨打电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_3 bigint comment '{"chs_name":"当天有效电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_3 bigint comment '{"chs_name":"当天有效电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ call_dev_3_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ call_num_3_d bigint comment '{"chs_name":"非当天拨打电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_3_d bigint comment '{"chs_name":"非当天有效电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_3_d bigint comment '{"chs_name":"非当天有效电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+
+ msg_dev_kyc bigint comment '{"chs_name":"私信咨询设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ msg_num_kyc bigint comment '{"chs_name":"私信咨询人次-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev_kyc bigint comment '{"chs_name":"有效私信咨询设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num_kyc bigint comment '{"chs_name":"有效私信咨询人次-kyc","description":"","etl":"","value":"","remark":""}',
+ clue_dev_kyc bigint comment '{"chs_name":"授权电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ clue_num_kyc bigint comment '{"chs_name":"授权电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+ call_dev_kyc bigint comment '{"chs_name":"当天拨打电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ call_num_kyc bigint comment '{"chs_name":"当天拨打电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_kyc bigint comment '{"chs_name":"当天有效电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_kyc bigint comment '{"chs_name":"当天有效电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+ call_dev_kyc_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ call_num_kyc_d bigint comment '{"chs_name":"非当天拨打电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_kyc_d bigint comment '{"chs_name":"非当天有效电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_kyc_d bigint comment '{"chs_name":"非当天有效电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+
+ msg_dev_qa bigint comment '{"chs_name":"私信咨询设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ msg_num_qa bigint comment '{"chs_name":"私信咨询人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev_qa bigint comment '{"chs_name":"有效私信咨询设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num_qa bigint comment '{"chs_name":"有效私信咨询人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ clue_dev_qa bigint comment '{"chs_name":"授权电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ clue_num_qa bigint comment '{"chs_name":"授权电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ call_dev_qa bigint comment '{"chs_name":"当天拨打电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ call_num_qa bigint comment '{"chs_name":"当天拨打电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_qa bigint comment '{"chs_name":"当天有效电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_qa bigint comment '{"chs_name":"当天有效电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ call_dev_qa_d bigint comment '{"chs_name":"非当天拨打电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ call_num_qa_d bigint comment '{"chs_name":"非当天拨打电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_qa_d bigint comment '{"chs_name":"非当天有效电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_qa_d bigint comment '{"chs_name":"非当天有效电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}'
+)comment '用户侧线索日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/clue_daily/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/clue_daily/job/step1_10.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file

pm/clue_daily/job/step1_11.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file

pm/clue_daily/job/step1_12.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file

pm/clue_daily/job/step1_13.job

+#step1_13.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/clue_daily/job/step1_14.job

+#step1_13.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file

pm/clue_daily/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/clue_daily/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/clue_daily/job/step1_4.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file

pm/clue_daily/job/step1_5.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_message_view
\ No newline at end of file

pm/clue_daily/job/step1_6.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file

pm/clue_daily/job/step1_7.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_doctor_relation_dimen_day
\ No newline at end of file

pm/clue_daily/job/step1_8.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file

pm/clue_daily/job/step1_9.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file

pm/clue_daily/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
+command=sh /home/bi/bi-report/lib/shell/hive clue_daily
\ No newline at end of file

pm/clue_daily/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/clue_daily/liudi@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file

pm/clue_daily/report/clue_daily.sql

+SELECT day_id `日期`
+		,device_os_type `系统`
+		,active_type `新老`
+		,grey_type `是否灰度`
+		,channel `渠道`
+		,dau as `dau（不去除积分墙）`
+		,user_num as `注册的用户数`
+		,phone_user_num as `手机号注册用户数`
+		,total_clue_dev as `总咨询设备（去重）`
+		,total_clue_num as `总线索`
+		,total_valid_dev+valid_call_dev_d as `总有效咨询设备（去重）`
+		,total_valid_num+valid_call_num_d as `总有效线索`
+		,nvl(concat(round((total_valid_dev+valid_call_dev_d)/dau*100,2),'%'),0) as `总有效线索设备(去重)/dau`
+		,nvl(concat(round((total_valid_num+valid_call_num_d)/dau*100,2),'%'),0) as `总有效线索/dau`
+		,nvl(concat(round((total_valid_dev+valid_call_dev_d)/total_clue_dev*100,2),'%'),0) as `总有效线索设备(去重)/总咨询设备(去重)`
+		,nvl(concat(round((total_valid_num+valid_call_num_d)/total_clue_num*100,2),'%'),0) as `总有效线索/总线索`
+		,msg_dev as `私信咨询设备数-总计`
+		,msg_num as `私信咨询人次-总计`
+		,valid_msg_dev as `有效私信咨询设备数-总计`
+		,valid_msg_num as `有效私信咨询人次-总计`
+		,nvl(concat(round(valid_msg_num/msg_num*100,2),'%'),0) as `有效私信咨询人次/私信咨询人次`
+		,clue_dev as `授权电话线索设备数-总计`
+		,clue_num as `授权电话线索人次-总计`
+		,call_dev+call_dev_d as `拨打电话线索设备数-总计`
+		,call_num+call_num_d as `拨打电话线索人次-总计`
+		,nvl(concat(round((call_num+call_num_d)/clue_num*100,2),'%'),0) as `拨打电话人次/授权电话人次`
+		,valid_call_dev+valid_call_dev_d as `有效电话线索设备数-总计`
+		,valid_call_num+valid_call_num_d as `有效电话线索人次-总计`
+		,call_dev as `当天拨打电话线索设备数-总计`
+		,call_num as `当天拨打电话线索人次-总计`
+		,valid_call_dev as `当天有效电话线索设备数-总计`
+		,valid_call_num as `当天有效电话线索人次-总计`
+
+		,msg_dev_3 `私信咨询设备数-3级以上商户`
+		,msg_num_3 `私信咨询人次-3级以上商户`
+		,nvl(concat(round(msg_num_3/msg_num*100,2),'%'),0) as `私信咨询人次-3级以上/私信咨询人次`
+		,valid_msg_dev_3 `有效私信咨询设备数-3级以上商户`
+		,valid_msg_num_3 as `有效私信咨询人次-3级以上商户`
+		,clue_dev_3 as `授权电话线索设备数-3级以上商户`
+		,clue_num_3 as `授权电话线索人次-3级以上商户`
+		,nvl(concat(round(clue_num_3/clue_num*100,2),'%'),0) as `授权电话线索-3级以上/授权电话线索`
+		,call_dev_3+call_dev_3_d as `拨打电话线索设备数-3级以上商户`
+		,call_num_3+valid_call_num_3 as `拨打电话线索人次-3级以上商户`
+		,nvl(concat(round((call_num_3+valid_call_num_3)/clue_num_3*100,2),'%'),0) as `拨打电话线索人次-3级以上/授权电话线索人次-3级以上`
+		,valid_call_dev_3+valid_call_dev_3_d as `有效电话线索设备数-3级以上商户`
+		,valid_call_num_3+valid_call_num_3_d as `有效电话线索人次-3级以上商户`
+		,nvl(concat(round((valid_call_num_3+valid_call_num_3_d)/clue_num_3*100,2),'%'),0) as `有效电话线索人次-3级以上/授权电话线索人次-3级以上`
+		,call_dev_3 as `当天拨打电话线索设备数-3级以上商户`
+		,call_num_3 as `当天拨打电话线索人次-3级以上商户`
+		,valid_call_dev_3 as `当天有效电话线索设备数-3级以上商户`
+		,valid_call_num_3 as `当天有效电话线索人次-3级以上商户`
+
+		,msg_dev_kyc `私信咨询设备数-kyc`
+		,msg_num_kyc `私信咨询人次-kyc`
+		,valid_msg_dev_kyc `有效私信咨询设备数-kyc`
+		,valid_msg_num_kyc as `有效私信咨询人次-kyc`
+		,clue_dev_kyc as `授权电话线索设备数-kyc`
+		,clue_num_kyc as `授权电话线索人次-kyc`
+		,call_dev_kyc+call_dev_kyc_d as `拨打电话线索设备数-kyc`
+		,call_num_kyc+call_num_kyc_d as `拨打电话线索人次-kyc`
+		,valid_call_dev_kyc+valid_call_dev_kyc_d as `有效电话线索设备数-kyc`
+		,valid_call_num_kyc+valid_call_num_kyc_d as `有效电话线索人次-kyc`
+		,call_dev_kyc as `当天拨打电话线索设备数-kyc`
+		,call_num_kyc as `当天拨打电话线索人次-kyc`
+		,valid_call_dev_kyc as `当天有效电话线索设备数-kyc`
+		,valid_call_num_kyc as `当天有效电话线索人次-kyc`
+
+		,msg_dev_qa `私信咨询设备数-需求自测`
+		,msg_num_qa `私信咨询人次-需求自测`
+		,valid_msg_dev_qa `有效私信咨询设备数-需求自测`
+		,valid_msg_num_qa as `有效私信咨询人次-需求自测`
+		,clue_dev_qa as `授权电话线索设备数-需求自测`
+		,clue_num_qa as `授权电话线索人次-需求自测`
+		,call_dev_qa+call_dev_qa_d as `拨打电话线索设备数-需求自测`
+		,call_num_qa+call_num_qa_d as `拨打电话线索人次-需求自测`
+		,valid_call_dev_qa+valid_call_dev_qa_d as `有效电话线索设备数-需求自测`
+		,valid_call_num_qa+valid_call_num_qa_d as `有效电话线索人次-需求自测`
+		,call_dev_qa_d as `当天拨打电话线索设备数-需求自测`
+		,call_num_qa_d as `当天拨打电话线索人次-需求自测`
+		,valid_call_dev_qa_d as `当天有效电话线索设备数-需求自测`
+		,valid_call_num_qa_d as `当天有效电话线索人次-需求自测`
+FROM pm.tl_pm_userclue_d
+where partition_day in ('20200901','202001001','20201101','20201201','20210101',regexp_replace(date_sub(current_date,1),'-',''))
+order by `日期`,`系统`,`新老`,`是否灰度`

pm/daily_content_data/etl/daily_content_data.sql

@@ -14,6 +14,8 @@ CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
 INSERT OVERWRITE TABLE pm.tl_pm_content_d PARTITION (PARTITION_DAY = ${partition_day})
 
 
+
+
 SELECT
 		T1.partition_date AS day_id
 		,T1.device_os_type AS device_os_type
@@ -614,7 +616,9 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 	        AND page_name = 'home'
 			AND action = 'on_click_card'
-			AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
+			AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+            and params['transaction_type'] not like '%ctr'
+            and params['transaction_type'] not like '%cvr'
 			AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
 
 			UNION ALL
@@ -626,7 +630,9 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 	        AND page_name = 'home'
 			AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
-			AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
+			AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+            and params['transaction_type'] not like '%ctr'
+            and params['transaction_type'] not like '%cvr'
 
 			UNION ALL
 
@@ -637,7 +643,8 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 			AND page_name = 'home'
 			AND action = 'on_click_card'
-			AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
+			AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+                  or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
 			AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
 
 			UNION ALL
@@ -649,7 +656,8 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 	        AND page_name = 'home'
 			AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
-			AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr','samecity_cvr')
+			AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+                  or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
 		)t2
 		ON t1.partition_date=t2.partition_date
 		  AND t1.device_id=t2.cl_id
@@ -1161,3 +1169,5 @@ ON T1.partition_date=T9.partition_date
   AND T1.channel=T9.channel
 ORDER BY day_id desc,device_os_type,active_type,is_ai_channel;
 
+
+

pm/daily_content_data/job/step3.job

 #step3.job
 type=command
 dependencies=step2
-command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/liudi@igengmei.com,wangxin@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
+command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/liudi@igengmei.com,wangxin@igengmei.com,dengguangyu@igengmei.com,zhaoyang@igengmei.com/weiyimin@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file

pm/daily_operation/en-cn.properties

+daily_operation=运营日报
\ No newline at end of file

pm/daily_operation/etl/create_daily_operation.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 运营日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_operation_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
+ dau bigint comment '{"chs_name":"dau(去除积分墙)","description":"","etl":"","value":"","remark":""}',
+ login_dev_num bigint comment '{"chs_name":"新增注册设备数","description":"","etl":"","value":"","remark":""}',
+ login_user_num bigint comment '{"chs_name":"新增注册账号","description":"","etl":"","value":"","remark":""}',
+ login_phone_user_num bigint comment '{"chs_name":"新增注册手机号用户数","description":"","etl":"","value":"","remark":""}',
+ retention_1 string comment '{"chs_name":"次日留存率","description":"","etl":"","value":"","remark":""}',
+ retention_7 string comment '{"chs_name":"第7日留存率","description":"","etl":"","value":"","remark":""}',
+ retention_30 string comment '{"chs_name":"第30日留存率","description":"","etl":"","value":"","remark":""}',
+ per_open_times bigint comment '{"chs_name":"单设备打开频次","description":"","etl":"","value":"","remark":""}',
+ home_feeds_ctr string comment '{"chs_name":"信息流内容ctr","description":"","etl":"","value":"","remark":""}',
+ per_use_time double comment '{"chs_name":"单设备app停留时长(M)","description":"","etl":"","value":"","remark":""}',
+ neirong_page_stay double comment '{"chs_name":"单设备内容时长(M)","description":"","etl":"","value":"","remark":""}',
+ neirong_in_app_use_time string comment '{"chs_name":"单设备内容时长/app停留时长","description":"","etl":"","value":"","remark":""}',
+ service_privacy_uv bigint comment '{"chs_name":"到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
+ service_click_uv bigint comment '{"chs_name":"同意隐私授权设备设备数","description":"","etl":"","value":"","remark":""}',
+ service_privacy_uv_in_dau string comment '{"chs_name":"到达隐私授权界面设备数/DAU","description":"","etl":"","value":"","remark":""}',
+ service_click_uv_in_dau string comment '{"chs_name":"同意隐私授权设备数/到达隐私授权界面设备数","description":"","etl":"","value":"","remark":""}',
+
+ open_push_uv bigint comment '{"chs_name":"开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
+ open_push_uv_in_dau string comment '{"chs_name":"开启推送授权设备数/DAU","description":"","etl":"","value":"","remark":""}',
+ push_receive_uv_in_open_push_uv string comment '{"chs_name":"次日接收推送设备数/当天开启推送授权设备数","description":"","etl":"","value":"","remark":""}',
+ click_uv bigint comment '{"chs_name":"点击推送的设备数","description":"","etl":"","value":"","remark":""}',
+ non_click_uv bigint comment '{"chs_name":"未点击推送的设备数","description":"","etl":"","value":"","remark":""}',
+ click_uv_in_dau string comment '{"chs_name":"点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
+ non_click_uv_in_dau string comment '{"chs_name":"未点击推送设备数/DAU","description":"","etl":"","value":"","remark":""}',
+ op_push_open_uv bigint comment '{"chs_name":"点击运营推送设备数","description":"","etl":"","value":"","remark":""}',
+ auto_push_open_uv bigint comment '{"chs_name":"点击系统推送设备数","description":"","etl":"","value":"","remark":""}',
+ personal_push_open_uv bigint comment '{"chs_name":"点击个性化推送设备数","description":"","etl":"","value":"","remark":""}',
+ sign_push_open_uv bigint comment '{"chs_name":"点击签到推送设备数","description":"","etl":"","value":"","remark":""}',
+ op_push_open_uv_in_receive string comment '{"chs_name":"点击运营推送设备/接收到运营推送设备","description":"","etl":"","value":"","remark":""}',
+ auto_push_open_uv_in_receive string comment '{"chs_name":"点击系统推送设备/接收到系统推送设备","description":"","etl":"","value":"","remark":""}',
+ personal_push_open_uv_in_receive string comment '{"chs_name":"点击个性化推送设备/接收到个性化推送设备","description":"","etl":"","value":"","remark":""}',
+ sign_push_open_uv_in_receive string comment '{"chs_name":"点击签到推送设备/接收到签到推送设备","description":"","etl":"","value":"","remark":""}',
+
+ question_num bigint comment '{"chs_name":"主动提问数","description":"","etl":"","value":"","remark":""}',
+ answer_num bigint comment '{"chs_name":"主动回答数","description":"","etl":"","value":"","remark":""}',
+ topic_num bigint comment '{"chs_name":"主动发帖数","description":"","etl":"","value":"","remark":""}',
+ reply_num bigint comment '{"chs_name":"真实评论数","description":"","etl":"","value":"","remark":""}',
+ question_num_in_dau string comment '{"chs_name":"主动提问设备数/DAU`","description":"","etl":"","value":"","remark":""}',
+ answer_num_in_dau string comment '{"chs_name":"主动回答设备数/DAU","description":"","etl":"","value":"","remark":""}',
+ topic_num_in_dau string comment '{"chs_name":"主动发帖设备数/DAU","description":"","etl":"","value":"","remark":""}',
+ reply_num_in_dau string comment '{"chs_name":"主动评论设备数/DAU","description":"","etl":"","value":"","remark":""}',
+
+ neirongUV bigint comment '{"chs_name":"内容UV","description":"","etl":"","value":"","remark":""}',
+ neirongPV bigint comment '{"chs_name":"内容PV","description":"","etl":"","value":"","remark":""}',
+ neirongUV_in_dau string comment '{"chs_name":"内容UV/DAU","description":"","etl":"","value":"","remark":""}',
+ neirongPV_in_UV double comment '{"chs_name":"内容PV/内容UV","description":"","etl":"","value":"","remark":""}',
+ result_more_UV bigint comment '{"chs_name":"搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
+ result_meigou_UV bigint comment '{"chs_name":"搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
+ result_con_UV bigint comment '{"chs_name":"搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
+ result_more_UV_in_dau string comment '{"chs_name":"搜索结果综合页UV/DAU","description":"","etl":"","value":"","remark":""}',
+ result_meigou_UV_in_dau string comment '{"chs_name":"搜索结果美购页UV/DAU","description":"","etl":"","value":"","remark":""}',
+ result_con_UV_in_dau string comment '{"chs_name":"搜索结果商业页UV/DAU","description":"","etl":"","value":"","remark":""}',
+ result_more_PV bigint comment '{"chs_name":"搜索结果综合页PV","description":"","etl":"","value":"","remark":""}',
+ result_meigou_PV bigint comment '{"chs_name":"搜索结果美购页PV","description":"","etl":"","value":"","remark":""}',
+ result_con_PV bigint comment '{"chs_name":"搜索结果商业页PV","description":"","etl":"","value":"","remark":""}',
+ result_more_PV_in_UV double comment '{"chs_name":"搜索结果综合页PV/搜索结果综合页UV","description":"","etl":"","value":"","remark":""}',
+ result_meigou_PV_in_UV double comment '{"chs_name":"搜索结果美购页PV/搜索结果美购页UV","description":"","etl":"","value":"","remark":""}',
+ result_con_PV_in_UV double comment '{"chs_name":"搜索结果商业页PV/搜索结果商业页UV","description":"","etl":"","value":"","remark":""}',
+ result_more_PV_in_dau string comment '{"chs_name":"搜索结果综合页PV/DAU","description":"","etl":"","value":"","remark":""}',
+ result_meigou_PV_in_dau string comment '{"chs_name":"搜索结果美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
+ result_con_PV_in_dau string comment '{"chs_name":"搜索结果商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
+ neirong_meigou_PV bigint comment '{"chs_name":"来自内容页的美购页PV","description":"","etl":"","value":"","remark":""}',
+ neirong_con_PV bigint comment '{"chs_name":"来自内容页的商业页PV","description":"","etl":"","value":"","remark":""}',
+ neirong_meigou_PV_in_dau string comment '{"chs_name":"来自内容页的美购页PV/DAU","description":"","etl":"","value":"","remark":""}',
+ neirong_con_PV_in_dau string comment '{"chs_name":"来自内容页的商业页PV/DAU","description":"","etl":"","value":"","remark":""}',
+ neirong_click_pv bigint comment '{"chs_name":"内容页的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
+ wel_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的商业二跳/内容pv","description":"","etl":"","value":"","remark":""}',
+ neirong_second_pv_in_neirongPV string comment '{"chs_name":"来自内容页的内容二跳/内容pv","description":"","etl":"","value":"","remark":""}'
+ )comment '运营日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_operation/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/daily_operation/job/step1_10.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_diary_view
\ No newline at end of file

pm/daily_operation/job/step1_11.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_problem_view
\ No newline at end of file

pm/daily_operation/job/step1_12.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_question_view
\ No newline at end of file

pm/daily_operation/job/step1_13.job

+#step1_13.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/daily_operation/job/step1_14.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_view
\ No newline at end of file

pm/daily_operation/job/step1_15.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file

pm/daily_operation/job/step1_16.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file

pm/daily_operation/job/step1_17.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file

pm/daily_operation/job/step1_18.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file

pm/daily_operation/job/step1_19.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file

pm/daily_operation/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/daily_operation/job/step1_20.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_topicreply_view
\ No newline at end of file

pm/daily_operation/job/step1_21.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_reply_view
\ No newline at end of file

pm/daily_operation/job/step1_22.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file

pm/daily_operation/job/step1_23.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive bl bl_et_bg_trackingpushlog_inc_d
\ No newline at end of file

pm/daily_operation/job/step1_24.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_auroracallback_view
\ No newline at end of file

pm/daily_operation/job/step1_25.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_jpushuserdevicebinding_view
\ No newline at end of file

pm/daily_operation/job/step1_26.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file

pm/daily_operation/job/step1_27.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file

pm/daily_operation/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/daily_operation/job/step1_4.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file

pm/daily_operation/job/step1_5.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive dim DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file

pm/daily_operation/job/step1_7.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml mid_ml_c_et_pe_preciseexposure_dimen_d
\ No newline at end of file

pm/daily_operation/job/step1_8.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_ck_click_dimen_d
\ No newline at end of file

pm/daily_operation/job/step1_9.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file

pm/daily_operation/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17,step1_18,step1_19,step1_20,step1_21,step1_22,step1_23,step1_24,step1_25,step1_26,step1_27
+command=sh /home/bi/bi-report/lib/shell/hive daily_operation
\ No newline at end of file

pm/daily_operation/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/daily_operation/liudi@igengmei.com,wangxin@igengmei.com,xuepengfei@igengmei.com,zhaoyang@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file

pm/daily_operation/report/daily_operation.sql

+SELECT FROM_UNIXTIME(UNIX_TIMESTAMP(day_id,'yyyymmdd'),'yyyy-mm-dd') `日期`
+		,device_os_type `系统`
+		,active_type `新老`
+		,channel `渠道`
+		,dau as `dau（去除积分墙）`
+		,nvl(login_dev_num,0) as `新增注册设备数`
+		,nvl(login_user_num,0) as `新增注册账号`
+		,nvl(login_phone_user_num,0) as `新增注册手机号用户数`
+		,nvl(retention_1,0) as `次日留存率`
+		,nvl(retention_7,0) as `第7日留存率`
+		,nvl(retention_30,0) as `第30日留存率`
+		,nvl(per_open_times,0) as `单设备打开频次`
+		,nvl(home_feeds_ctr,0) as `信息流内容ctr`
+		,nvl(per_use_time,0) as `单设备app停留时长(M)`
+		,nvl(neirong_page_stay,0) as `单设备内容时长(M)`
+		,nvl(neirong_in_app_use_time,0) as `单设备内容时长/app停留时长`
+		,nvl(service_privacy_uv,0) as `到达隐私授权界面设备数`
+		,nvl(service_click_uv,0) as `同意隐私授权设备设备数`
+		,nvl(service_privacy_uv_in_dau,0) as `到达隐私授权界面设备数/DAU`
+		,nvl(service_click_uv_in_dau,0) as `同意隐私授权设备数/到达隐私授权界面设备数`
+		,nvl(open_push_uv,0) as `开启推送授权设备数`
+		,nvl(open_push_uv_in_dau,0) as `开启推送授权设备数/DAU`
+		,nvl(push_receive_uv_in_open_push_uv,0) as `次日接收推送设备数/当天开启推送授权设备数`
+		,nvl(click_uv,0) as `点击推送的设备数`
+		,nvl(non_click_uv,0) as `未点击推送的设备数`
+		,nvl(click_uv_in_dau,0) as `点击推送设备数/DAU`
+
+		,nvl(non_click_uv_in_dau,0) `未点击推送设备数/DAU`
+		,nvl(op_push_open_uv,0) `点击运营推送设备数`
+		,nvl(auto_push_open_uv,0) as `点击系统推送设备数`
+		,nvl(personal_push_open_uv,0) `点击个性化推送设备数`
+		,nvl(sign_push_open_uv,0) as `点击签到推送设备数`
+		,nvl(op_push_open_uv_in_receive,0) as `点击运营推送设备/接收到运营推送设备`
+		,nvl(auto_push_open_uv_in_receive,0) as `点击系统推送设备/接收到系统推送设备`
+		,nvl(personal_push_open_uv_in_receive,0) as `点击个性化推送设备/接收到个性化推送设备`
+		,nvl(sign_push_open_uv_in_receive,0) as `点击签到推送设备/接收到签到推送设备`
+		,nvl(question_num,0) as `主动提问数`
+		,nvl(answer_num,0) as `主动回答数`
+		,nvl(topic_num,0) as `主动发帖数`
+		,nvl(reply_num,0) as `真实评论数`
+		,nvl(question_num_in_dau,0) as `主动提问设备数/DAU`
+		,nvl(answer_num_in_dau,0) as `主动回答设备数/DAU`
+		,nvl(topic_num_in_dau,0) as `主动发帖设备数/DAU`
+		,nvl(reply_num_in_dau,0) as `主动评论设备数/DAU`
+
+		,nvl(neirongUV,0) as `内容UV`
+		,nvl(neirongPV,0) `内容PV`
+		,nvl(neirongUV_in_dau,0) `内容UV/DAU`
+		,nvl(neirongPV_in_UV,0) `内容PV/内容UV`
+		,nvl(result_more_UV,0) as `搜索结果综合页UV`
+		,nvl(result_meigou_UV,0) as `搜索结果美购页UV`
+		,nvl(result_con_UV,0) as `搜索结果商业页UV`
+		,nvl(result_more_UV_in_dau,0) as `搜索结果综合页UV/DAU`
+		,nvl(result_meigou_UV_in_dau,0) as `搜索结果美购页UV/DAU`
+		,nvl(result_con_UV_in_dau,0) as `搜索结果商业页UV/DAU`
+		,nvl(result_more_PV,0) as `搜索结果综合页PV`
+		,nvl(result_meigou_PV,0) as `搜索结果美购页PV`
+		,nvl(result_con_PV,0) as `搜索结果商业页PV`
+		,nvl(result_more_PV_in_UV,0) as `搜索结果综合页PV/搜索结果综合页UV`
+		,nvl(result_meigou_PV_in_UV,0) as `搜索结果美购页PV/搜索结果美购页UV`
+		,nvl(result_con_PV_in_UV,0) `搜索结果商业页PV/搜索结果商业页UV`
+		,nvl(result_more_PV_in_dau,0) `搜索结果综合页PV/DAU`
+		,nvl(result_meigou_PV_in_dau,0) `搜索结果美购页PV/DAU`
+		,nvl(result_con_PV_in_dau,0) as `搜索结果商业页PV/DAU`
+		,nvl(neirong_meigou_PV,0) as `来自内容页的美购页PV`
+		,nvl(neirong_con_PV,0) as `来自内容页的商业页PV`
+		,nvl(neirong_meigou_PV_in_dau,0) as `来自内容页的美购页PV/DAU`
+		,nvl(neirong_con_PV_in_dau,0) as `来自内容页的商业页PV/DAU`
+		,nvl(neirong_click_pv,0) as `内容页的推荐内容点击pv`
+		,nvl(wel_second_pv_in_neirongPV,0) as `来自内容页的商业二跳/内容pv`
+		,nvl(neirong_second_pv_in_neirongPV,0) as `来自内容页的内容二跳/内容pv`
+FROM pm.tl_pm_operation_d
+where ((partition_day ='20200824' and day_id<='20200726')
+       or (partition_day<regexp_replace(date_sub(current_date,1),'-','') and partition_day>'20200824' and day_id=regexp_replace(date_sub(FROM_UNIXTIME(UNIX_TIMESTAMP(partition_day,'yyyymmdd'),'yyyy-mm-dd'),29),'-',''))
+       or (partition_day=regexp_replace(date_sub(current_date,1),'-','') and day_id>=regexp_replace(date_sub(current_date,30),'-','')))
+order by `日期`,`系统`,`新老`,`渠道`

pm/daily_push/etl/create_push_quality.sql

+--***************************************************************
+--*脚本名称:
+--*功能: push质量数据
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_push_quality_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ op_only_wel double comment '{"chs_name":"只通过运营push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
+ op_push_first_wel double comment '{"chs_name":"先通过运营push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
+ op_open_first_wel double comment '{"chs_name":"先主动打开再运营push-漏到核心页","description":"","etl":"","value":"","remark":""}',
+ op_only_clue double comment '{"chs_name":"只通过运营push来-漏到线索","description":"","etl":"","value":"","remark":""}',
+ op_push_first_clue double comment '{"chs_name":"先通过运营push来-漏到线索","description":"","etl":"","value":"","remark":""}',
+ op_open_first_clue double comment '{"chs_name":"先主动打开再运营push-漏到线索","description":"","etl":"","value":"","remark":""}',
+
+ auto_only_wel double comment '{"chs_name":"只通过自动化push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
+ auto_push_first_wel double comment '{"chs_name":"先通过自动化push来-漏到核心页","description":"","etl":"","value":"","remark":""}',
+ auto_open_first_wel double comment '{"chs_name":"先主动打开再自动化push-漏到核心页","description":"","etl":"","value":"","remark":""}',
+ auto_only_clue double comment '{"chs_name":"只通过自动化push来-漏到线索","description":"","etl":"","value":"","remark":""}',
+ auto_push_first_clue double comment '{"chs_name":"先通过自动化push来-漏到线索","description":"","etl":"","value":"","remark":""}',
+ auto_open_first_clue double comment '{"chs_name":"先主动打开再自动化push-漏到线索","description":"","etl":"","value":"","remark":""}',
+)comment 'push质量数据'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_push/job/step1_10.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file

pm/daily_push/job/step1_11.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ML_TRADE_ORDER_DETAIL_DAY
\ No newline at end of file

pm/daily_push/job/step1_12.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file

pm/daily_push/job/step1_13.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file

pm/daily_push/job/step1_14.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file

pm/daily_push/job/step1_15.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file

pm/daily_push/job/step1_16.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file

pm/daily_push/job/step1_17.job

+#step1_9.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/daily_push/job/step2.job

 #step2.job
 type=command
-dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17
 command=sh /home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file

pm/daily_push/job/step3.job

 #step3.job
 type=command
 dependencies=step2
-command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
+command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,weiyimin@igengmei.com,yindanlei@igengmei.com,wangyan@igengmei.com
\ No newline at end of file

pm/daily_recommend_strategy/en-cn.properties

-daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file
+daily_recommend_strategy=首页推荐策略及内容页ctr日报
+home_recommend_strategy=首页推荐策略日报
+content_ctr=内容详情页灰度ctr
\ No newline at end of file

pm/daily_recommend_strategy/etl/create_content_grey.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 首页推荐策略日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_contentpage_ctr
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
+ page_name string comment '{"chs_name":"页面名称","description":"","etl":"","value":"","remark":""}',
+ content_pv bigint comment '{"chs_name":"内容详情页pv","description":"","etl":"","value":"","remark":""}',
+ content_uv bigint comment '{"chs_name":"内容详情页uv","description":"","etl":"","value":"","remark":""}',
+ wel_exp_pv bigint comment '{"chs_name":"你可能喜欢美购卡片曝光pv","description":"","etl":"","value":"","remark":""}',
+ content_exp_pv bigint comment '{"chs_name":"你可能喜欢内容卡片曝光pv","description":"","etl":"","value":"","remark":""}',
+ wel_click_pv bigint comment '{"chs_name":"你可能喜欢美购卡片点击pv","description":"","etl":"","value":"","remark":""}',
+ content_click_pv bigint comment '{"chs_name":"你可能喜欢内容卡片点击pv","description":"","etl":"","value":"","remark":""}',
+ slide_wel_click_pv bigint comment '{"chs_name":"横划卡片点击pv","description":"","etl":"","value":"","remark":""}',
+ self_wel_click_pv bigint comment '{"chs_name":"关联卡片点击pv","description":"","etl":"","value":"","remark":""}'
+)comment '内容详情页卡片ctr'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_recommend_strategy/etl/create_home_retention.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 首页推荐策略日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
+ recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
+ card_click bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ card_exposure bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ avg_page_stay double comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ navbar_search bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ highlight_word bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ self_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ recommend_welfare_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ recommend_content_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ recommend_special_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ transfer_card bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}',
+ video_consultation bigint comment '{"chs_name":"","description":"","etl":"","value":"","remark":""}'
+
+)comment '内容日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_recommend_strategy/job/step3.job

 #step3.job
 type=command
 dependencies=step2
-command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
+command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,zhaoyang@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/weiyimin@igengmei.com
\ No newline at end of file

pm/daily_recommend_strategy/report/content_ctr.sql

+SELECT
+    day_id as `日期`,
+    device_os_type as `设备类型`,
+    active_type as `活跃类型`,
+    grey_type as `灰度`,
+    page_name as `页面`,
+    NVL(CONCAT(ROUND(wel_click_pv/wel_exp_pv*100,2),'%'),0) as `内容页你可能喜欢美购卡片点击PV/该类卡片曝光PV`,
+    NVL(CONCAT(ROUND(content_click_pv/content_exp_pv*100,2),'%'),0) as `内容页你可能喜欢内容卡片点击PV/该类卡片曝光PV`,
+    content_pv as `内容详情页pv`,
+    content_uv as `内容详情页uv`,
+    wel_exp_pv as `你可能喜欢美购卡片曝光pv`,
+    content_exp_pv as `你可能喜欢内容卡片曝光pv`,
+    wel_click_pv as `你可能喜欢美购卡片点击pv`,
+    content_click_pv as `你可能喜欢内容卡片点击pv`,
+    slide_wel_click_pv as `横划卡片点击pv`,
+    self_wel_click_pv as `关联卡片点击pv`
+FROM pm.tl_pm_contentpage_ctr
+WHERE partition_day>='20200730' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+order by `日期` desc,`设备类型`,`活跃类型`,`灰度`,`页面`
\ No newline at end of file

pm/daily_recommend_strategy/report/daily_recommend_strategy.sql → pm/daily_recommend_strategy/report/home_recommend_strategy.sql

@@ -4,21 +4,20 @@ SELECT
     active_type as `活跃类型`,
     card_content_type as `卡片类型`,
     recommend_type as `推荐类型`,
-    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
     NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
-    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
     card_click as `首页卡片点击PV`,
     card_exposure as `首页卡片精准曝光PV`,
-    (navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)as `有效二跳pv`,
+    (navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)as `有效二跳pv`,
     avg_page_stay as `来自I的单PV平均浏览时长`,
     navbar_search as `来自I的搜索框+搜索按钮点击PV`,
     highlight_word as `来自I的文内搜索点击PV`,
     self_welfare_card as `来自I的商品卡片点击PV`,
     recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
     recommend_content_card as `来自I的推荐内容点击pv`,
-    '未配置' as `来自I的推荐专题点击pv`,
-    '未上线' as `来自I的转诊点击pv`,
-    '未上线' as `来自I的视频面诊点击pv`
-FROM pm.tl_pm_recommend_strategy_d
-WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+--    '未配置' as `来自I的推荐专题点击pv`,
+    transfer_card as `来自I的转诊点击pv`,
+    video_consultation as `来自I的视频面诊点击pv`FROM pm.tl_pm_recommend_strategy_d
+WHERE partition_day>='20200824' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
 order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file

pm/kyc_q2/en-cn.properties

+kyc_q2=用户兴趣引导日报及ai自测日报
+ai_daily=AI自测漏斗数据
+ai_qa=AI自测问答选项
+kyc_qa_daily=用户兴趣引导日报

pm/kyc_q2/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/kyc_q2/job/step1_10.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_zx_api_ai_qa_answer
\ No newline at end of file

pm/kyc_q2/job/step1_11.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_zx_api_ai_qa_question
\ No newline at end of file

pm/kyc_q2/job/step1_12.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_userextra_view
\ No newline at end of file

pm/kyc_q2/job/step1_13.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_message_view
\ No newline at end of file

pm/kyc_q2/job/step1_14.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_conversationuserstatus_view
\ No newline at end of file

pm/kyc_q2/job/step1_15.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_message_detail
\ No newline at end of file

pm/kyc_q2/job/step1_16.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_doctor_relation_dimen_day
\ No newline at end of file

pm/kyc_q2/job/step1_17.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file

pm/kyc_q2/job/step1_18.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file

pm/kyc_q2/job/step1_19.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file

pm/kyc_q2/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/kyc_q2/job/step1_20.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file

pm/kyc_q2/job/step1_21.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file

pm/kyc_q2/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/kyc_q2/job/step1_4.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_zx_api_ai_qa_question_answer
\ No newline at end of file