Merge branch 'weiyimin' into 'master'

Weiyimin

See merge request !76

pm/clear_content/en-cn.properties

-clear_content=首页内容清洗
-clear_content_answer=首页内容清洗-回答
-clear_content_diary=首页内容清洗-日记
\ No newline at end of file

pm/clear_content/job/step1_10.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_operation_updates
\ No newline at end of file

pm/clear_content/job/step1_11.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_vote_view
\ No newline at end of file

pm/clear_content/job/step1_12.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_vote_view
\ No newline at end of file

pm/clear_content/job/step1_13.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_favor_view
\ No newline at end of file

pm/clear_content/job/step1_14.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_favor_view
\ No newline at end of file

pm/clear_content/job/step1_2.job

-#step1_2.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_mp_api_tractate_tag_v3
\ No newline at end of file

pm/clear_content/job/step1_3.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive tl tl_mp_api_tractate_tag_v3
\ No newline at end of file

pm/clear_content/job/step1_4.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_3_0_view
\ No newline at end of file

pm/clear_content/job/step1_5.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file

pm/clear_content/job/step1_7.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ML_D_CT_UI_USERCLEAN_DIMEN_D
\ No newline at end of file

pm/clear_content/job/step1_9.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_reply_view
\ No newline at end of file

pm/clear_content/job/step2.job

-#step2.job
-type=command
-dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14
-command=curl -X GET http://localhost:8553/api/report/email/clear_content/weiyimin@igengmei.com/weiyimin@qq.com
\ No newline at end of file

pm/daily_content/en-cn.properties

+daily_content=内容日报-简化版
+home_content-detail=首页内容数据-分日明细
+home_content_by_month=首页内容数据-月均
+ai_content_detail=ai内容数据-分日明细
+ai_content_by_month=ai内容数据-月均
\ No newline at end of file

pm/daily_content/etl/create_daily_content.sql

+--***************************************************************
+--*脚本名称:
+--*功能: 内容日报-简化版-for 思璟
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_content_v2
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ channel string comment '{"chs_name":"渠道","description":"","etl":"","value":"","remark":""}',
+ dau BIGINT comment '{"chs_name":"日活","description":"","etl":"","value":"","remark":""}',
+ retention_rate_2 string comment '{"chs_name":"次留率","description":"","etl":"","value":"","remark":""}',
+ retention_rate_3 string comment '{"chs_name":"第3日留存率","description":"","etl":"","value":"","remark":""}',
+ retention_rate_7 string comment '{"chs_name":"第7日留存率","description":"","etl":"","value":"","remark":""}',
+ home_card_click_uv BIGINT comment '{"chs_name":"首页卡片点击uv","description":"","etl":"","value":"","remark":""}',
+ home_card_click_pv BIGINT comment '{"chs_name":"首页卡片点击pv","description":"","etl":"","value":"","remark":""}',
+ home_good_click BIGINT comment '{"chs_name":"来源于首页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}',
+ page_stay_per_device BIGINT comment '{"chs_name":"单设备内容浏览时长(m)","description":"","etl":"","value":"","remark":""}',
+ ai_report_card_click_uv BIGINT comment '{"chs_name":"ai结果页内容卡片点击uv","description":"","etl":"","value":"","remark":""}',
+ ai_report_card_click_pv BIGINT comment '{"chs_name":"ai结果页内容卡片点击pv","description":"","etl":"","value":"","remark":""}',
+ ai_report_good_click BIGINT comment '{"chs_name":"来源于结果页的good click（浏览时长>=20s的内容）","description":"","etl":"","value":"","remark":""}',
+)comment '内容日报-简化版'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/clear_content/job/step1_1.job → pm/daily_content/job/step1_1.job

 #step1_1.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tractate_view
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/clear_content/job/step1_8.job → pm/daily_content/job/step1_2.job

-#step1_3.job
+#step1_2.job
 type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_user_updates
\ No newline at end of file
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/daily_content/job/step1_4.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive DIM DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file

pm/daily_content/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4
+command=curl -X GET http://localhost:8553/api/report/email/daily_content/weiyimin@igengmei.com/hanyingyue@igengmei.com
\ No newline at end of file

pm/daily_content/report/ai_content_by_month.sql

+SELECT
+        month AS `日期`
+        ,device_os_type AS `系统`
+        ,active_type AS `活跃`
+        ,channel as `渠道`
+        ,dau AS `DAU`
+        ,ai_report_card_click_uv AS `ai结果页内容卡片点击uv`
+        ,ai_report_card_click_pv AS `ai结果页内容卡片点击pv`
+        ,ai_report_good_click AS `来源于ai结果页的good click`
+        ,CONCAT(ROUND(ai_report_card_click_uv/dau*100,2),'%') AS `ai结果页内容卡片点击uv/DAU`
+        ,CONCAT(ROUND(ai_report_card_click_pv/dau*100,2),'%') AS `ai结果页内容卡片点击pv/DAU`
+        ,CONCAT(ROUND(ai_report_good_click/dau*100,2),'%') AS `来源于ai结果页的good click/DAU`
+FROM
+(
+    select substr(day_id,1,6) as month
+            ,device_os_type
+            ,active_type
+            ,channel
+            ,round(avg(dau),0) as dau
+            ,round(avg(ai_report_card_click_uv),0) as ai_report_card_click_uv
+            ,round(avg(ai_report_card_click_pv),0) as ai_report_card_click_pv
+            ,round(avg(ai_report_good_click),0) as ai_report_good_click
+    FROM pm.tl_pm_content_v2
+    where partition_day>='20201018' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
+    group by substr(day_id,1,6),device_os_type,active_type,channel
+)t1
+order by `日期`,`系统`,`活跃`,`渠道`
\ No newline at end of file

pm/daily_content/report/ai_content_detail.sql

+SELECT
+        day_id AS `日期`
+        ,device_os_type AS `系统`
+        ,active_type AS `活跃`
+        ,channel as `渠道`
+        ,dau AS `DAU`
+        ,ai_report_card_click_uv AS `ai结果页内容卡片点击uv`
+        ,ai_report_card_click_pv AS `ai结果页内容卡片点击pv`
+        ,ai_report_good_click AS `来源于ai结果页的good click`
+        ,NVL(CONCAT(ROUND(ai_report_card_click_uv/dau*100,2),'%'),0) AS `ai结果页内容卡片点击uv/DAU`
+        ,NVL(CONCAT(ROUND(ai_report_card_click_pv/dau*100,2),'%'),0) AS `ai结果页内容卡片点击pv/DAU`
+        ,NVL(CONCAT(ROUND(ai_report_good_click/dau*100,2),'%'),0) AS `来源于ai结果页的good click/DAU`
+FROM pm.tl_pm_content_v2
+where partition_day>='20201018' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
+order by `日期`,`系统`,`活跃`,`渠道`
\ No newline at end of file

pm/daily_content/report/home_content_by_month.sql

+SELECT
+        month AS `日期`
+        ,device_os_type AS `系统`
+        ,active_type AS `活跃`
+        ,channel as `渠道`
+        ,dau AS `DAU`
+        ,retention_rate_2 AS `次留率(%)`
+        ,retention_rate_3 AS `第3日留存率(%)`
+        ,retention_rate_7 AS `第7日留存率(%)`
+        ,home_card_click_uv AS `首页卡片点击uv`
+        ,home_card_click_pv AS `首页卡片点击pv`
+        ,home_good_click AS `来源于首页的good click`
+        ,CONCAT(ROUND(home_card_click_uv/dau*100,2),'%') AS `首页卡片点击uv/DAU`
+        ,CONCAT(ROUND(home_card_click_pv/dau*100,2),'%') AS `首页卡片点击pv/DAU`
+        ,CONCAT(ROUND(home_good_click/dau*100,2),'%') AS `来源于首页的good click/DAU`
+        ,page_stay_per_device AS `单设备内容浏览时长(m)`
+FROM
+(
+    select substr(day_id,1,6) as month
+            ,device_os_type
+            ,active_type
+            ,channel
+            ,round(avg(dau),0) as dau
+            ,round(avg(retention_rate_2),2) as retention_rate_2
+            ,round(avg(retention_rate_3),2) as retention_rate_3
+            ,round(avg(retention_rate_7),2) as retention_rate_7
+            ,round(avg(home_card_click_uv),0) as home_card_click_uv
+            ,round(avg(home_card_click_pv),0) as home_card_click_pv
+            ,round(avg(home_good_click),0) as home_good_click
+            ,round(avg(page_stay_per_device),2) as page_stay_per_device
+    FROM pm.tl_pm_content_v2
+    where partition_day>='20201018' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
+    group by substr(day_id,1,6),device_os_type,active_type,channel
+)t1
+order by `日期`,`系统`,`活跃`,`渠道`
\ No newline at end of file

pm/daily_content/report/home_content_detail.sql

+SELECT
+        day_id AS `日期`
+        ,device_os_type AS `系统`
+        ,active_type AS `活跃`
+        ,channel as `渠道`
+        ,dau AS `DAU`
+        ,retention_rate_2 AS `次留率(%)`
+        ,retention_rate_3 AS `第3日留存率(%)`
+        ,retention_rate_7 AS `第7日留存率(%)`
+        ,home_card_click_uv AS `首页卡片点击uv`
+        ,home_card_click_pv AS `首页卡片点击pv`
+        ,home_good_click AS `来源于首页的good click`
+        ,NVL(CONCAT(ROUND(home_card_click_uv/dau*100,2),'%'),0) AS `首页卡片点击uv/DAU`
+        ,NVL(CONCAT(ROUND(home_card_click_pv/dau*100,2),'%'),0) AS `首页卡片点击pv/DAU`
+        ,NVL(CONCAT(ROUND(home_good_click/dau*100,2),'%'),0) AS `来源于首页的good click/DAU`
+        ,NVL(page_stay_per_device,0) AS `单设备内容浏览时长(m)`
+FROM pm.tl_pm_content_v2
+where partition_day>='20201018' and partition_day <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
+order by `日期`,`系统`,`活跃`,`渠道`
\ No newline at end of file