Merge branch 'weiyimin' into 'master'

push codes

See merge request !43

.idea/misc.xml

@@ -16,5 +16,8 @@
   <component name="JavaScriptSettings">
     <option name="languageLevel" value="ES6" />
   </component>
+  <component name="ProjectPlainTextFileTypeManager">
+    <file url="file://$PROJECT_DIR$/pm/clue_daily/report/ai_qa.sql" />
+  </component>
   <component name="ProjectRootManager" version="2" languageLevel="JDK_13" default="false" project-jdk-name="Python 2.7 (shell-tools)" project-jdk-type="Python SDK" />
 </project>
\ No newline at end of file

pm/clue_daily/en-cn.properties

+clue_daily=线索日报（用户侧）
\ No newline at end of file

pm/clue_daily/etl/create_clue_daily.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 线索日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_userclue_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ grey_type string comment '{"chs_name":"灰度类型","description":"","etl":"","value":"","remark":""}',
+ dau bigint comment '{"chs_name":"dau(不去除积分墙)","description":"","etl":"","value":"","remark":""}',
+ user_num bigint comment '{"chs_name":"注册的用户数","description":"","etl":"","value":"","remark":""}',
+ msg_dev bigint comment '{"chs_name":"私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
+ msg_num bigint comment '{"chs_name":"私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev bigint comment '{"chs_name":"有效私信咨询设备数-总计","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num bigint comment '{"chs_name":"有效私信咨询人次-总计","description":"","etl":"","value":"","remark":""}',
+ clue_dev bigint comment '{"chs_name":"授权电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ clue_num bigint comment '{"chs_name":"授权电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+ call_dev bigint comment '{"chs_name":"拨打电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ call_num bigint comment '{"chs_name":"拨打电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev bigint comment '{"chs_name":"有效电话线索设备数-总计","description":"","etl":"","value":"","remark":""}',
+ valid_call_num bigint comment '{"chs_name":"有效电话线索人次-总计","description":"","etl":"","value":"","remark":""}',
+
+ msg_dev_4 bigint comment '{"chs_name":"私信咨询设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ msg_num_4 bigint comment '{"chs_name":"私信咨询人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev_4 bigint comment '{"chs_name":"有效私信咨询设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num_4 bigint comment '{"chs_name":"有效私信咨询人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ clue_dev_4 bigint comment '{"chs_name":"授权电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ clue_num_4 bigint comment '{"chs_name":"授权电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ call_dev_4 bigint comment '{"chs_name":"拨打电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ call_num_4 bigint comment '{"chs_name":"拨打电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_4 bigint comment '{"chs_name":"有效电话线索设备数-4级以上商户","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_4 bigint comment '{"chs_name":"有效电话线索人次-4级以上商户","description":"","etl":"","value":"","remark":""}',
+
+ msg_dev_kyc bigint comment '{"chs_name":"私信咨询设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ msg_num_kyc bigint comment '{"chs_name":"私信咨询人次-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev_kyc bigint comment '{"chs_name":"有效私信咨询设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num_kyc bigint comment '{"chs_name":"有效私信咨询人次-kyc","description":"","etl":"","value":"","remark":""}',
+ clue_dev_kyc bigint comment '{"chs_name":"授权电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ clue_num_kyc bigint comment '{"chs_name":"授权电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+ call_dev_kyc bigint comment '{"chs_name":"拨打电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ call_num_kyc bigint comment '{"chs_name":"拨打电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_kyc bigint comment '{"chs_name":"有效电话线索设备数-kyc","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_kyc bigint comment '{"chs_name":"有效电话线索人次-kyc","description":"","etl":"","value":"","remark":""}',
+
+ msg_dev_qa bigint comment '{"chs_name":"私信咨询设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ msg_num_qa bigint comment '{"chs_name":"私信咨询人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_msg_dev_qa bigint comment '{"chs_name":"有效私信咨询设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_msg_num_qa bigint comment '{"chs_name":"有效私信咨询人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ clue_dev_qa bigint comment '{"chs_name":"授权电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ clue_num_qa bigint comment '{"chs_name":"授权电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ call_dev_qa bigint comment '{"chs_name":"拨打电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ call_num_qa bigint comment '{"chs_name":"拨打电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_call_dev_qa bigint comment '{"chs_name":"有效电话线索设备数-需求自测","description":"","etl":"","value":"","remark":""}',
+ valid_call_num_qa bigint comment '{"chs_name":"有效电话线索人次-需求自测","description":"","etl":"","value":"","remark":""}'
+
+)comment '用户侧线索日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/clue_daily/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/clue_daily/job/step1_10.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_virtual_phone_binding
\ No newline at end of file

pm/clue_daily/job/step1_11.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task_phone_binding
\ No newline at end of file

pm/clue_daily/job/step1_12.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_lead_task
\ No newline at end of file

pm/clue_daily/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/clue_daily/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/clue_daily/job/step1_4.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/clue_daily/job/step1_5.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_message_view
\ No newline at end of file

pm/clue_daily/job/step1_6.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_et_msg_conversation_dimen_inc_d
\ No newline at end of file

pm/clue_daily/job/step1_7.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_doctor_relation_dimen_day
\ No newline at end of file

pm/clue_daily/job/step1_8.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_call_detail
\ No newline at end of file

pm/clue_daily/job/step1_9.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_gm_sl_ali_virtual_phone_binding
\ No newline at end of file

pm/clue_daily/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12
+command=sh /home/bi/bi-report/lib/shell/hive clue_daily
\ No newline at end of file

pm/clue_daily/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/clue_daily/weiyimin@igengmei.com/weiyimin@qq.com
\ No newline at end of file

pm/clue_daily/report/clue_daily.sql

+SELECT mas.partition_date `日期`
+		,device_os_type `系统`
+		,active_type `新老`
+		,grey_type `是否灰度`
+		,dau as `dau（不去除积分墙）`
+		,user_num as `注册的用户数`
+		,msg_dev as `私信咨询设备数-总计`
+		,msg_num as `私信咨询人次-总计`
+		,valid_msg_dev as `有效私信咨询设备数-总计`
+		,valid_msg_num as `有效私信咨询人次-总计`
+		,clue_dev as `授权电话线索设备数-总计`
+		,clue_num as `授权电话线索人次-总计`
+		,call_dev as `拨打电话线索设备数-总计`
+		,call_num as `拨打电话线索人次-总计`
+		,valid_call_dev as `有效电话线索设备数-总计`
+		,valid_call_num as `有效电话线索人次-总计`
+
+		,msg_dev_4 `私信咨询设备数-4级以上商户`
+		,msg_num_4 `私信咨询人次-4级以上商户`
+		,valid_msg_dev_4 `有效私信咨询设备数-4级以上商户`
+		,valid_msg_num_4 as `有效私信咨询人次-4级以上商户`
+		,clue_dev_4 as `授权电话线索设备数-4级以上商户`
+		,clue_num_4 as `授权电话线索人次-4级以上商户`
+		,call_dev_4 as `拨打电话线索设备数-4级以上商户`
+		,call_num_4 as `拨打电话线索人次-4级以上商户`
+		,valid_call_dev_4 as `有效电话线索设备数-4级以上商户`
+		,valid_call_num_4 as `有效电话线索人次-4级以上商户`
+
+		,msg_dev_kyc `私信咨询设备数-kyc`
+		,msg_num_kyc `私信咨询人次-kyc`
+		,valid_msg_dev_kyc `有效私信咨询设备数-kyc`
+		,valid_msg_num_kyc as `有效私信咨询人次-kyc`
+		,clue_dev_kyc as `授权电话线索设备数-kyc`
+		,clue_num_kyc as `授权电话线索人次-kyc`
+		,call_dev_kyc as `拨打电话线索设备数-kyc`
+		,call_num_kyc as `拨打电话线索人次-kyc`
+		,valid_call_dev_kyc as `有效电话线索设备数-kyc`
+		,valid_call_num_kyc as `有效电话线索人次-kyc`
+
+		,msg_dev_qa `私信咨询设备数-需求自测`
+		,msg_num_qa `私信咨询人次-需求自测`
+		,valid_msg_dev_qa `有效私信咨询设备数-需求自测`
+		,valid_msg_num_qa as `有效私信咨询人次-需求自测`
+		,clue_dev_qa as `授权电话线索设备数-需求自测`
+		,clue_num_qa as `授权电话线索人次-需求自测`
+		,call_dev_qa as `拨打电话线索设备数-需求自测`
+		,call_num_qa as `拨打电话线索人次-需求自测`
+		,valid_call_dev_qa as `有效电话线索设备数-需求自测`
+		,valid_call_num_qa as `有效电话线索人次-需求自测`
+FROM pm.tl_pm_userclue_d
+where partition_day=regexp_replace(date_sub(current_date,1),'-','')
\ No newline at end of file