update codes

lib/java/bi-report-service/.idea/workspace.xml

@@ -2,14 +2,29 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="">
-      <change afterPath="$PROJECT_DIR$/../../shell/hive" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/en-cn.properties" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/en-cn.properties" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/create_daliy_content.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/create_daily_content_data.sql" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/daily_content_data_insert.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/daily_content_data.sql" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step2.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step2.job" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step3.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step3.job" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step4.job" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/report/daily_content_data.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_content_data/report/select_daily_content_data.sql" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/en-cn.properties" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/en-cn.properties" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/create_daily_content_data.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/etl/create_daily_recommend_strategy.sql" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/etl/daily_content_data.sql" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/daily_content_data.zip" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_1.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_1.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_10.job" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_11.job" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_12.job" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_13.job" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_2.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_2.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_3.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_3.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_4.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_4.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_5.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_5.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_6.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_6.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_7.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_7.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_8.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_8.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step1_9.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step1_9.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step2.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step2.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/job/step3.job" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/job/step3.job" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/readme.txt" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/readme.txt" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../pm/daily_content_data/report/select_daily_content_data.sql" beforeDir="false" afterPath="$PROJECT_DIR$/../../../pm/daily_recommend_strategy/report/select_daily_recommend_strategy.sql" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../../../readme.txt" beforeDir="false" afterPath="$PROJECT_DIR$/../../../readme.txt" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -64,6 +79,7 @@
     <option name="showLibraryContents" value="true" />
   </component>
   <component name="PropertiesComponent">
+    <property name="ASKED_ADD_EXTERNAL_FILES" value="true" />
     <property name="RequestMappingsPanelOrder0" value="0" />
     <property name="RequestMappingsPanelOrder1" value="1" />
     <property name="RequestMappingsPanelWidth0" value="75" />
@@ -188,13 +204,16 @@
       <workItem from="1591668349091" duration="1455000" />
       <workItem from="1592214013610" duration="2379000" />
       <workItem from="1592568026557" duration="772000" />
-      <workItem from="1593835574199" duration="1924000" />
+      <workItem from="1593835574199" duration="3629000" />
     </task>
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
     <option name="version" value="1" />
   </component>
+  <component name="VcsManagerConfiguration">
+    <option name="ADD_EXTERNAL_FILES_SILENTLY" value="true" />
+  </component>
   <component name="WindowStateProjectService">
     <state x="773" y="303" key="#com.intellij.ide.util.MemberChooser" timestamp="1593835959317">
       <screen x="0" y="22" width="1920" height="1129" />

pm/daily_content_data/en-cn.properties

-select_daily_content_data=内容日报-新
\ No newline at end of file

pm/daily_content_data/job/step1_10.job

-#step1_10.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_view
\ No newline at end of file

pm/daily_content_data/job/step1_11.job

-#step1_11.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_topicreply_view
\ No newline at end of file

pm/daily_content_data/job/step1_12.job

-#step1_12.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_answer_reply_view
\ No newline at end of file

pm/daily_content_data/job/step1_13.job

-#step1_13.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file

pm/daily_content_data/job/step1_4.job

-#step1_4.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/daily_content_data/job/step1_7.job

-#step1_7.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_diary_view
\ No newline at end of file

pm/daily_content_data/report/select_daily_content_data.sql

---内容日报
-
-SELECT
-		day_id AS `日期`
-		,device_os_type AS `设备类型`
-		,active_type AS `活跃类型`
-		,is_ai_channel AS `是否AI渠道`
-		,dau AS `DAU`
-		,content_uv AS `内容详情页UV`
-		,content_pv AS `内容详情页PV`
-		,per_content_uv AS `内容UV/DAU`
-		,per_content_pv AS `内容PV/内容UV`
-		,retention_1 AS `内容用户APP次留`
-		,retention_7 AS `内容用户APP7留`
-		,retention_30 AS `内容用户APP30留`
-		,avg_app_duration AS `内容用户单设备App时长(m)`
-		,avg_content_stay AS `内容用户单设备内容时长(m)`
-		,avg_open_times AS `内容用户单设备打开次数`
-		,search_related_stay AS `内容用户搜索相关页面单设备页面时长(m)`
-		,welfare_stay AS `内容用户美购详情页单设备页面时长(m)`
-		,content_question_stay AS `内容用户问题详情页单设备页面时长(m)`
-		,ai_related_stay AS `内容用户AI相关页面单设备页面时长(m)`
-		,content_diary_stay AS `内容用户日记详情页单设备页面时长(m)`
-		,home_stay AS `内容用户首页单设备页面时长(m)`
-		,conv_related_stay AS `内容用户咨询相关页面单设备页面时长(m)`
-		,recommend_rate AS `首页feeds推荐进入内容UV/内容UV`
-		,per_recommend_pv AS `首页feeds推荐进入内容PV/UV`
-		,feeds_rate AS `首页feeds非推荐进入内容UV/内容UV`
-		,per_feeds_pv AS `首页feeds非推荐进入内容PV/UV`
-		,search_rate AS `搜索进入内容UV/内容UV`
-		,per_search_pv AS `搜索进入内容PV/UV`
-		,zone_rate AS `内容聚合页进入内容UV/内容UV`
-		,per_zone_pv AS `内容聚合页进入内容PV/UV`
-		,content_rate AS `内容详情页推荐板块进入内容UV/内容UV`
-		,per_from_content_pv AS `内容详情页推荐板块进入内容PV/UV`
-		,blank_rate AS `无来源页面(大多数为push)进入内容UV/内容UV`
-		,per_blank_pv AS `无来源页面(大多数为push)进入内容PV/UV`
-		,comment_rate AS `评论列表页进入内容UV/内容UV`
-		,per_comment_pv AS `评论列表页进入内容PV/UV`
-		,org_rate AS `医生医院主页进入内容UV/内容UV`
-		,per_org_pv AS `医生医院主页进入内容PV/UV`
-		,category_rate AS `品类聚合页进入内容UV/内容UV`
-		,per_category_pv AS `品类聚合页进入内容PV/UV`
-		,my_diary_rate AS `我的日记页进入内容UV/内容UV`
-		,per_my_diary_pv AS `我的日记页进入内容PV/UV`
-		,ai_rate AS `AI报告页进入内容UV/内容UV`
-		,per_ai_pv AS `AI报告页进入内容PV/UV`
-		,diary_uv AS `日记UV`
-		,diary_pv AS `日记PV`
-		,diary_rate AS `日记UV/内容UV`
-		,per_diary_pv AS `日记PV/日记UV`
-		,diary_stay AS `日记单设备时长(m)`
-		,post_uv AS `帖子UV`
-		,post_pv AS `帖子PV`
-		,post_rate AS `帖子UV/内容UV`
-		,per_post_pv AS `帖子PV/帖子UV`
-		,post_stay AS `帖子单设备时长(m)`
-		,question_uv AS `问题UV`
-		,question_pv AS `问题PV`
-		,question_rate AS `问题UV/内容UV`
-		,per_question_pv AS `问题PV/问题UV`
-		,question_stay AS `问题单设备时长(m)`
-		,question_answer_uv AS `问答UV`
-		,question_answer_pv AS `问答PV`
-		,question_answer_rate AS `问答UV/内容UV`
-		,per_question_answer_pv AS `问答PV/问答UV`
-		,question_answer_stay AS `问答单设备时长(m)`
-		,answer_uv AS `回答UV`
-		,answer_pv AS `回答PV`
-		,answer_rate AS `回答UV/内容UV`
-		,per_answer_pv AS `回答PV/回答UV`
-		,answer_stay AS `回答单设备时长(m)`
-		,video_uv AS `视频UV`
-		,video_pv AS `视频PV`
-		,video_rate AS `视频UV/内容UV`
-		,per_video_pv AS `视频PV/视频UV`
-		,video_stay AS `视频单设备时长(m)`
-		,wiki_uv AS `百科UV`
-		,wiki_pv AS `百科PV`
-		,wiki_rate AS `百科UV/内容UV`
-		,per_wiki_pv AS `百科PV/百科UV`
-		,wiki_stay AS `百科单设备时长(m)`
-		,article_uv AS `专栏UV`
-		,article_pv AS `专栏PV`
-		,article_rate AS `专栏UV/内容UV`
-		,per_article_pv AS `专栏PV/专栏UV`
-		,article_stay AS `专栏单设备时长(m)`
-FROM pm.tl_pm_content_d
-where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','');

pm/daily_recommend_strategy/en-cn.properties

+daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file

pm/daily_content_data/job/step1_2.job → pm/daily_recommend_strategy/job/step1_2.job

 #step1_2.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file

pm/daily_content_data/job/step1_3.job → pm/daily_recommend_strategy/job/step1_3.job

 #step1_3.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_updates
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/daily_recommend_strategy/job/step1_4.job

+#step1_4.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/daily_content_data/job/step1_5.job → pm/daily_recommend_strategy/job/step1_5.job

 #step1_5.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file

pm/daily_content_data/job/step1_6.job → pm/daily_recommend_strategy/job/step1_6.job

 #step1_6.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_doctor_view
\ No newline at end of file

pm/daily_recommend_strategy/job/step1_7.job

+#step1_7.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_c_ct_ui_user_dimen_d
\ No newline at end of file

pm/daily_content_data/job/step1_8.job → pm/daily_recommend_strategy/job/step1_8.job

 #step1_8.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_problem_view
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_history_detail
\ No newline at end of file

pm/daily_content_data/job/step1_9.job → pm/daily_recommend_strategy/job/step1_9.job

 #step1_9.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_question_view
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file

pm/daily_content_data/job/step2.job → pm/daily_recommend_strategy/job/step2.job

 #step2.job
 type=command
-dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
-command=/home/bi/bi-report/lib/shell/hive daily_content_data
\ No newline at end of file
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
+command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file

pm/daily_content_data/job/step3.job → pm/daily_recommend_strategy/job/step3.job

 #step3.job
 type=command
 dependencies=step2
-command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file
+command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file

pm/daily_recommend_strategy/report/daily_recommend_strategy.sql

+SELECT
+    day_id as `日期`,
+    device_os_type as `设备类型`,
+    active_type as `活跃类型`,
+    card_content_type as `卡片类型`,
+    recommend_type as `推荐类型`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
+    card_click as `首页卡片点击PV`,
+    card_exposure as `首页卡片精准曝光PV`,
+    (navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
+    avg_page_stay as `来自I的单PV平均浏览时长`,
+    navbar_search as `来自I的搜索框+搜索按钮点击PV`,
+    highlight_word as `来自I的文内搜索点击PV`,
+    self_welfare_card as `来自I的商品卡片点击PV`,
+    recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
+    recommend_content_card as `来自I的推荐内容点击pv`,
+    '未配置' as `来自I的推荐专题点击pv`,
+    '未上线' as `来自I的转诊点击pv`,
+    '未上线' as `来自I的视频面诊点击pv`
+FROM pm.tl_pm_recommend_strategy_d
+WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file

readme.txt

@@ -34,3 +34,10 @@ BI report project init.
     4.优化邮件内容，wps打开去掉  样式格式化【metabase中包含内容简略视图，而且有metabaselogo】  邮件内容格式化 python实现  ok
     5.一个附件多个sheet  或者  一封邮件  多个附件【metabase默认支持指定多个问题、多个附件】 python发送邮件实现，一个文件多个sheet实现起来相对比较难  发送邮件时需要指定附件名称(带账期)  ok
     6.excel文件内容格式化   优先级比较低 【指定每个字段的最大长度】  使用python实现，尝试  ok
+
+
+
+规范约定：
+    1.job文件的命名
+        必须以job1_01的方式命名，以便在azkaban中可以有序查看
+