Merge branch 'yindanlei' into 'master'

Yindanlei

See merge request !40

pm/fangan_day/etl/create_fangan_day.sql

@@ -22,7 +22,9 @@ CREATE TABLE IF NOT EXISTS pm.tl_pm_fangan_d
  new_DAU BIGINT comment '{"chs_name":"新增DAU","description":"","etl":"","value":"","remark":""}',
  old_DAU BIGINT comment '{"chs_name":"老活DAU","description":"","etl":"","value":"","remark":""}',
  quanzhong_DAU BIGINT comment '{"chs_name":"权重日活","description":"","etl":"","value":"","remark":""}',
- liucun DOUBLE comment '{"chs_name":"次日留存率(%)","description":"","etl":"","value":"","remark":""}',
+ liucun_1 DOUBLE comment '{"chs_name":"次日留存率(%)","description":"","etl":"","value":"","remark":""}',
+ liucun_3 DOUBLE comment '{"chs_name":"3日留存率(%)","description":"","etl":"","value":"","remark":""}',
+ liucun_7 DOUBLE comment '{"chs_name":"7日留存率(%)","description":"","etl":"","value":"","remark":""}',
  wel_pv BIGINT comment '{"chs_name":"商详PV","description":"","etl":"","value":"","remark":""}',
  wel_uv BIGINT comment '{"chs_name":"商详UV","description":"","etl":"","value":"","remark":""}',
  pay_num BIGINT comment '{"chs_name":"支付订单数","description":"","etl":"","value":"","remark":""}',

pm/fangan_day/etl/fangan_day.sql

@@ -26,7 +26,9 @@ SELECT t1.partition_date as day_id
       +count(DISTINCT CASE WHEN device_type = '老活' AND device_os_type = 'ios' AND channel_type = '医美' THEN t1.device_id END)*1.00
       +count(DISTINCT CASE WHEN device_type = '新增' AND device_os_type = 'ios' AND channel_type = 'AI' THEN t1.device_id END)*0.21
       +count(DISTINCT CASE WHEN device_type = '新增' AND device_os_type = 'ios' AND channel_type = '医美' THEN t1.device_id END)*0.21,0) as quanzhong_DAU
-,round(sum(t1.rent_num)/count(DISTINCT t1.device_id)*100,2) as liucun
+,round(sum(t1.rent1_num)/count(DISTINCT t1.device_id)*100,2) as liucun_1
+,round(sum(t1.rent3_num)/count(DISTINCT t1.device_id)*100,2) as liucun_3
+,round(sum(t1.rent7_num)/count(DISTINCT t1.device_id)*100,2) as liucun_7
 ,sum(wel_pv) as wel_pv
 ,count(DISTINCT CASE WHEN wel_pv is not NULL and wel_pv <> 0 THEN t3.cl_id END) as wel_uv
 ,sum(pay_num) as pay_num
@@ -35,7 +37,10 @@ SELECT t1.partition_date as day_id
 ,count(DISTINCT t6.device_id) as true_valid_dev_num
 ,sum(t6.clue_num) as true_valid_pv
 FROM
-(   SELECT t1.partition_date,t1.device_id,device_type,device_os_type,channel_type,grey_type,count(DISTINCT CASE WHEN date_add(t1.partition_date,1) = t.partition_date THEN t.device_id END) as rent_num
+(   SELECT t1.partition_date,t1.device_id,device_type,device_os_type,channel_type,grey_type
+        ,count(DISTINCT CASE WHEN date_add(t1.partition_date,1) = t.partition_date THEN t.device_id END) as rent1_num
+        ,count(DISTINCT CASE WHEN date_add(t1.partition_date,3) = t.partition_date THEN t.device_id END) as rent3_num
+        ,count(DISTINCT CASE WHEN date_add(t1.partition_date,7) = t.partition_date THEN t.device_id END) as rent7_num
     FROM
     (---dau数据
         SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date,device_id,device_os_type
@@ -45,17 +50,13 @@ FROM
                          or (partition_date = '20200724' AND substr(convup(sha1(device_id),16,10),-1) in ('0','1','2','3','4'))
                          or (partition_date >= '20200728' AND substr(convup(sha1(device_id),16,10),-1) in ('0','1','2','3','4')) THEN '测试组A（方案库）'
                 else '测试组B（无方案库）' END as grey_type
-            ,CASE WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
-                         or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
-                         or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%') or (first_channel_source_type like 'aiqiyiand%')
-                                or (first_channel_source_type like 'bilibili%') or (first_channel_source_type like 'kuaishou%')))
-                         THEN 'AI'  ELSE '医美' END as channel_type
+            ,CASE WHEN is_ai_channel = 'true' THEN 'AI'  ELSE '医美' END as channel_type
         FROM online.ml_device_day_active_status
             LEFT JOIN
-            (SELECT col1,col2  --col1:子渠道，col2：是否属于AI，col3:标识
-               FROM pm.tl_pm_ydl
-               WHERE col3='0204_danlei_channel')tmp
-            on first_channel_source_type=tmp.col1
+            (SELECT code,is_ai_channel,partition_day
+             FROM DIM.DIM_AI_CHANNEL_ZP_NEW
+             WHERE partition_day>= '20200724' AND partition_day <=regexp_replace(date_sub(current_date(),1),'-','')) tmp
+            ON  m.partition_date=tmp.partition_day AND first_channel_source_type=code
         where partition_date>='20200724' AND partition_date<=regexp_replace(date_sub(current_date(),1),'-','')
         AND active_type in ('1','2','4')
     )t1

pm/fangan_day/job/step1_16.job

+#step1_16.job
+type=command
+command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive dim DIM_AI_CHANNEL_ZP_NEW
\ No newline at end of file

pm/fangan_day/job/step2.job

 #step2.job
 type=command
-dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16
 command=sh /home/bi/bi-report/lib/shell/hive fangan_day
\ No newline at end of file

pm/fangan_day/job/step3.job

 #step3.job
 type=command
 dependencies=step2
-command=curl -X GET http://localhost:8553/api/report/email/fangan_day/dailiang@igengmei.com/cp-sunyinghe@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
+command=curl -X GET http://localhost:8553/api/report/email/fangan_day/dailiang@igengmei.com/cp-sunyinghe@igengmei.com,yindanlei@igengmei.com,pengqiang@igengmei.com
\ No newline at end of file

pm/fangan_day/report/fangan_day.sql

@@ -7,7 +7,9 @@ SELECT
 		,new_DAU AS `新增DAU`
 		,old_DAU AS `老活DAU`
 		,quanzhong_DAU AS `权重日活`
-		,liucun AS `次日留存率(%)`
+		,liucun_1 AS `次日留存率(%)`
+		,liucun_3 AS `3日留存率(%)`
+		,liucun_7 AS `7日留存率(%)`
 		,wel_pv AS `商详PV`
 		,wel_uv AS `商详UV`
 		,pay_num AS `支付订单数`

pm/search_daily_data/etl/search_daily_data.sql

@@ -28,6 +28,7 @@ SELECT
     ,if(all_search_uv <> 0 ,concat(cast((all_search_pv/all_search_uv) as decimal(18,2)),'') , '-') as all_search_pv/all_search_uv
     ,if(search_home_pv <> 0 ,concat(cast((all_search_pv/search_home_pv)*100 as decimal(18,2)),'%') , '-') as all_search_pv/search_home_pv
     ,if(sug_out_pv_726 <> 0 ,concat(cast((sug_search_pv_726/sug_out_pv_726)*100 as decimal(18,2)),'%') , '-') as sug_search_pv_726/sug_out_pv_726
+    ,NVL(referrer_search_hexin_pv,0) as referrer_search_hexin_pv
     ,if(all_search_uv <> 0 ,concat(cast((referrer_search_hexin_pv/all_search_uv) as decimal(18,2)),'') , '-') as referrer_search_hexin_pv/all_search_uv
     ,if(all_search_uv <> 0 ,concat(cast((referrer_search_neirong_pv/all_search_uv) as decimal(18,2)),'') , '-') as referrer_search_neirong_pv/all_search_uv
     ,if(referrer_search_hexin_pv <> 0 ,concat(cast((search_hexin_two_pv/referrer_search_hexin_pv)*100 as decimal(18,2)),'%') , '-') as search_hexin_two_pv/referrer_search_hexin_pv