push codes

pm/daily_push/en-cn.properties

+daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file

pm/daily_push/etl/create_push_click.sql

+--***************************************************************
+--*脚本名称:
+--*功能: push日报
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ dau BIGINT comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
+ is_open_uv BIGINT comment '{"chs_name":"push开启设备数","description":"","etl":"","value":"","remark":""}',
+ push_open_uv BIGINT comment '{"chs_name":"push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ push_open_pv BIGINT comment '{"chs_name":"push拉起人次","description":"","etl":"","value":"","remark":""}',
+ push_open_2_uv BIGINT comment '{"chs_name":"push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ push_open_2_pv BIGINT comment '{"chs_name":"push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ op_push_open_uv BIGINT comment '{"chs_name":"运营push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ op_push_open_pv BIGINT comment '{"chs_name":"运营push拉起人次","description":"","etl":"","value":"","remark":""}',
+ op_push_open_2_uv BIGINT comment '{"chs_name":"运营push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ op_push_open_2_pv BIGINT comment '{"chs_name":"运营push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ auto_push_open_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ auto_push_open_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起人次","description":"","etl":"","value":"","remark":""}',
+ auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ will_push_open_uv BIGINT comment '{"chs_name":"意愿探索push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ will_push_open_pv BIGINT comment '{"chs_name":"意愿探索push拉起人次","description":"","etl":"","value":"","remark":""}',
+ will_push_open_2_uv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ will_push_open_2_pv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ sign_push_open_uv BIGINT comment '{"chs_name":"签到push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ sign_push_open_pv BIGINT comment '{"chs_name":"签到push拉起人次","description":"","etl":"","value":"","remark":""}',
+ sign_push_open_2_uv BIGINT comment '{"chs_name":"签到push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ sign_push_open_2_pv BIGINT comment '{"chs_name":"签到push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ personal_push_open_uv BIGINT comment '{"chs_name":"个性化push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ personal_push_open_pv BIGINT comment '{"chs_name":"个性化push拉起人次","description":"","etl":"","value":"","remark":""}',
+ personal_push_open_2_uv BIGINT comment '{"chs_name":"个性化push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ personal_push_open_2_pv BIGINT comment '{"chs_name":"个性化push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ ctr_push_open_uv BIGINT comment '{"chs_name":"ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ ctr_push_open_pv BIGINT comment '{"chs_name":"ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
+ ctr_push_open_2_uv BIGINT comment '{"chs_name":"ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ ctr_push_open_2_pv BIGINT comment '{"chs_name":"ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ non_ctr_push_open_uv BIGINT comment '{"chs_name":"非ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ non_ctr_push_open_pv BIGINT comment '{"chs_name":"非ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
+ non_ctr_push_open_2_uv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ non_ctr_push_open_2_pv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ AI_push_open_uv BIGINT comment '{"chs_name":"AI_push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ AI_push_open_pv BIGINT comment '{"chs_name":"AI_push拉起人次","description":"","etl":"","value":"","remark":""}',
+ AI_push_open_2_uv BIGINT comment '{"chs_name":"AI_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ AI_push_open_2_pv BIGINT comment '{"chs_name":"AI_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
+ orignal_auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}',
+ orignal_auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}',
+ orginal_auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
+ orginal_auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
+)comment 'push日报'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_push/etl/create_push_receive.sql

+--***************************************************************
+--*脚本名称:
+--*功能: push接收数据
+--*业务名称: pm
+--*输入数据:
+--*作者: weiyimin@igengmei.com
+--*更新时间:
+--***************************************************************
+
+--设置全局变量&UDF
+SET mapreduce.job.queuename=data;
+
+--使用bl数据库
+USE pm;
+
+--创建BL层内部表
+CREATE TABLE IF NOT EXISTS pm.tl_pm_push_receive_d
+(
+ day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
+ device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
+ active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
+ push_type BIGINT comment '{"chs_name":"push类型","description":"","etl":"","value":"","remark":""}',
+ received_dev_num BIGINT comment '{"chs_name":"push接收设备数","description":"","etl":"","value":"","remark":""}',
+ received_num BIGINT comment '{"chs_name":"push接收人次","description":"","etl":"","value":"","remark":""}',
+ click_dev_num BIGINT comment '{"chs_name":"push点击设备数","description":"","etl":"","value":"","remark":""}',
+ click_num BIGINT comment '{"chs_name":"push点击人次","description":"","etl":"","value":"","remark":""}'
+)comment 'push接收数据'
+PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\t'
+COLLECTION ITEMS TERMINATED BY '\002'
+MAP KEYS TERMINATED BY '\003'
+LINES TERMINATED BY '\n'
+STORED AS TEXTFILE;
\ No newline at end of file

pm/daily_push/job/step1_1.job

+#step1_1.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file

pm/daily_push/job/step1_2.job

+#step1_2.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file

pm/daily_push/job/step1_3.job

+#step1_3.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file

pm/daily_push/job/step1_4.job

+#step1_4.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file

pm/daily_push/job/step1_5.job

+#step1_5.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_pushtask2_view
\ No newline at end of file

pm/daily_push/job/step2.job

+#step2.job
+type=command
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
+command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file

pm/daily_push/job/step3.job

+#step3.job
+type=command
+dependencies=step2
+command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com,zhaojianwei@igengmei.com
\ No newline at end of file

pm/daily_push/report/daily_push.sql

+SELECT
+    day_id as `日期`,
+    device_os_type as `设备类型`,
+    active_type as `活跃类型`,
+    card_content_type as `卡片类型`,
+    recommend_type as `推荐类型`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
+    card_click as `首页卡片点击PV`,
+    card_exposure as `首页卡片精准曝光PV`,
+    (navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
+    avg_page_stay as `来自I的单PV平均浏览时长`,
+    navbar_search as `来自I的搜索框+搜索按钮点击PV`,
+    highlight_word as `来自I的文内搜索点击PV`,
+    self_welfare_card as `来自I的商品卡片点击PV`,
+    recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
+    recommend_content_card as `来自I的推荐内容点击pv`,
+    '未配置' as `来自I的推荐专题点击pv`,
+    '未上线' as `来自I的转诊点击pv`,
+    '未上线' as `来自I的视频面诊点击pv`
+FROM pm.tl_pm_recommend_strategy_d
+WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file

pm/daily_push/report/personal_push_click_rate.sql

+SELECT
+    day_id as `日期`,
+    device_os_type as `设备类型`,
+    active_type as `活跃类型`,
+    card_content_type as `卡片类型`,
+    recommend_type as `推荐类型`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
+    NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
+    card_click as `首页卡片点击PV`,
+    card_exposure as `首页卡片精准曝光PV`,
+    (navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card/2)as `有效二跳pv`,
+    avg_page_stay as `来自I的单PV平均浏览时长`,
+    navbar_search as `来自I的搜索框+搜索按钮点击PV`,
+    highlight_word as `来自I的文内搜索点击PV`,
+    self_welfare_card as `来自I的商品卡片点击PV`,
+    recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
+    recommend_content_card as `来自I的推荐内容点击pv`,
+    '未配置' as `来自I的推荐专题点击pv`,
+    '未上线' as `来自I的转诊点击pv`,
+    '未上线' as `来自I的视频面诊点击pv`
+FROM pm.tl_pm_recommend_strategy_d
+WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
+order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file

pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql

@@ -37,8 +37,7 @@ FROM
                    WHEN active_type  IN ('1','2')  THEN '新增' END AS active_type
              ,device_id
         FROM online.ml_device_day_active_status
-        WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-          AND partition_date<regexp_replace((current_date),'-','')
+        WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
           AND active_type IN ('1','2','4')
           AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
             ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
@@ -71,8 +70,7 @@ FROM
                   card_id,
                   app_session_id
            from online.ml_community_precise_exposure_detail
-           WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-           AND partition_date<regexp_replace((current_date),'-','')
+           WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
            AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
            AND is_exposure = '1'  ----精准曝光
            AND page_name ='home'
@@ -115,8 +113,7 @@ FROM
                   params['card_id'] as card_id,
                   app_session_id
            from online.bl_hdfs_maidian_updates
-           WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-           AND partition_date<regexp_replace((current_date),'-','')
+           WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
            AND action='on_click_card'
            AND params['page_name'] ='home'
            AND params['tab_name'] = '精选'
@@ -148,8 +145,7 @@ FROM
                        when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
                   sum(page_stay) as page_stay
           FROM online.bl_hdfs_maidian_updates
-          WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-          AND partition_date<regexp_replace((current_date),'-','')
+          WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
           AND action='page_view'
           AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
           AND referrer='home'
@@ -171,8 +167,7 @@ FROM
                        when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
                  count(1) as navbar_pv
           FROM online.bl_hdfs_maidian_updates
-          WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-          AND partition_date<regexp_replace((current_date),'-','')
+          WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
           AND action in ('on_click_navbar_search','do_search')
           AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
           AND (referrer='home' or
@@ -195,8 +190,7 @@ FROM
                        when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
                  count(1) as highlight_pv
           FROM online.bl_hdfs_maidian_updates
-          WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-          AND partition_date<regexp_replace((current_date),'-','')
+          WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
           AND action='on_click_card'
           and params['card_type']='highlight_word'
           AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
@@ -223,8 +217,7 @@ FROM
                          when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
                     count(1) as pv
             FROM online.bl_hdfs_maidian_updates
-            WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-            AND partition_date<regexp_replace((current_date),'-','')
+            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             AND (get_json_object(params['extra_param'], '$.type')='交互栏'
                    or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
                    or params['in_page_pos']='top' or params['in_page_pos']='bottom')
@@ -256,8 +249,7 @@ FROM
                          when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
                     count(1) as service_pv
             FROM online.bl_hdfs_maidian_updates
-            WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-            AND partition_date<regexp_replace((current_date),'-','')
+            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             AND (action='on_click_card'and  params['card_content_type']='service'
                    or action='on_click_button' and params['button_name']='unfold')
             AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
@@ -286,8 +278,7 @@ FROM
                          when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
                     count(1) as service_pv
             FROM online.bl_hdfs_maidian_updates
-            WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-            AND partition_date<regexp_replace((current_date),'-','')
+            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
             AND action='on_click_card'
             and params['card_content_type'] in ('qa','diary','user_post','answer')
             AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
@@ -325,8 +316,7 @@ FROM
           SELECT user_id,partition_date,
                   if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
             FROM online.ml_user_updates
-            WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
-            AND partition_date<regexp_replace((current_date),'-','')
+            WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
         )t1
                 JOIN
             (  --医生账号