Merge branch 'weiyimin' into 'master'

Weiyimin

See merge request !53

pm/daily_content_data/etl/daily_content_data.sql

@@ -614,7 +614,8 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 	        AND page_name = 'home'
 			AND action = 'on_click_card'
-			AND params['transaction_type'] not in ('-1','ctr','smr','cvr','ctcvr','hotspot','pgc','newdata','samecity_ctr','samecity_cvr','hotspot_feed','samecity_ctcvr','fmctr')
+			AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+            and params['transaction_type'] not like '%ctr' or params['transaction_type'] like '%cvr')
 			AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
 
 			UNION ALL
@@ -626,7 +627,8 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 	        AND page_name = 'home'
 			AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
-			AND params['transaction_type'] not in ('-1','ctr','smr','cvr','ctcvr','hotspot','pgc','newdata','samecity_ctr','samecity_cvr','hotspot_feed','samecity_ctcvr','fmctr')
+			AND params['transaction_type'] not in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+            and params['transaction_type'] not like '%ctr' or params['transaction_type'] like '%cvr')
 
 			UNION ALL
 
@@ -637,7 +639,8 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 			AND page_name = 'home'
 			AND action = 'on_click_card'
-			AND params['transaction_type'] in ('-1','ctr','smr','cvr','ctcvr','hotspot','pgc','newdata','samecity_ctr','samecity_cvr','hotspot_feed','samecity_ctcvr','fmctr')
+			AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+                  or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr'))
 			AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
 
 			UNION ALL
@@ -649,7 +652,8 @@ LEFT JOIN
 			AND partition_date < regexp_replace((current_date),'-','')
 	        AND page_name = 'home'
 			AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
-			AND params['transaction_type'] in ('-1','ctr','smr','cvr','ctcvr','hotspot','pgc','newdata','samecity_ctr','samecity_cvr','hotspot_feed','samecity_ctcvr','fmctr')
+			AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+                  or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr'))
 		)t2
 		ON t1.partition_date=t2.partition_date
 		  AND t1.device_id=t2.cl_id

pm/daily_operation/etl/daily_operation.sql

@@ -13,6 +13,70 @@ ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
 CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
 CREATE TEMPORARY FUNCTION setencryption AS 'com.gmei.hive.common.udf.UDFStringSetEncryption';
 
+INSERT OVERWRITE TABLE pm.tl_pm_channel_d PARTITION (PARTITION_DAY = ${partition_day})
+
+ SELECT a.partition_date as day_id
+     ,a.device_id as device_id
+     ,device_os_type,active_type,first_channel_source_type,first_city
+ FROM
+ (
+     SELECT
+     partition_date,device_os_type
+     ,device_id,first_channel_source_type,first_city
+     ,case WHEN active_type = '4'  THEN '老活'
+           WHEN active_type  in ('1','2')  then '新增' END as active_type
+     FROM online.ml_device_day_active_status
+     where partition_date >= '20200101'
+     AND partition_date < regexp_replace(current_date ,'-','')
+     AND active_type in ('1','2','4')
+     and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
+     ,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
+     ,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
+     ,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
+     ,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
+     ,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
+     ,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
+     AND first_channel_source_type not like 'promotion\_jf\_%'
+ )a
+ LEFT JOIN
+ (
+     SELECT partition_date,cl_id
+     from online.bl_hdfs_maidian_updates
+     where partition_date >= '20200101'
+     AND partition_date < regexp_replace(current_date ,'-','')
+     AND action = 'popup_view'
+     AND params['popup_name']='service_privacy'
+     GROUP BY partition_date,cl_id
+ )b
+     ON a.device_id = b.cl_id
+     AND a.partition_date = b.partition_date
+
+ LEFT JOIN
+ (
+     SELECT partition_date,cl_id
+     from online.bl_hdfs_maidian_updates
+     where partition_date >= '20200101'
+     AND partition_date < regexp_replace(current_date ,'-','')
+     AND action = 'page_view'
+     GROUP BY partition_date,cl_id
+ )c
+     ON a.device_id = c.cl_id
+     AND a.partition_date = c.partition_date
+ LEFT JOIN
+ (
+     SELECT partition_day,device_id
+     FROM ml.ml_c_et_ck_click_dimen_d
+     where partition_day >= '20200101'
+     AND partition_day < regexp_replace(current_date ,'-','')
+     GROUP BY partition_day,device_id
+ )d
+     on a.device_id = d.device_id
+     AND a.partition_date = d.partition_day
+     WHERE b.cl_id is NOT NULL
+     AND c.cl_id is NULL
+     AND d.device_id is NULL
+ GROUP BY a.partition_date,a.device_id,device_os_type,active_type,first_channel_source_type,first_city;
+
 
 
 INSERT OVERWRITE TABLE pm.tl_pm_operation_d PARTITION (PARTITION_DAY = ${partition_day})
@@ -445,8 +509,8 @@ LEFT JOIN
 			AND page_code='home'
 			AND is_exposure='1'
 			AND tab_code='精选'
-			AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
-                or transaction_type like '%ctr' or transaction_type like '%cvr')
+			AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
+                or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
 			AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
 			AND card_content_type in ('user_post','diary','qa','answer')
 		)t1
@@ -525,7 +589,7 @@ LEFT JOIN
 			AND action='on_click_card'
 			AND page_code ='home'
 		    AND tab_code = '精选'
-		    AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
+		    AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
                 or transaction_type like '%ctr' or transaction_type like '%cvr')
 		    AND card_content_type in ('diary','user_post','qa','answer')
 		)t1
@@ -1122,7 +1186,7 @@ LEFT JOIN
 	    (
 	      SELECT partition_date,registration_id,platform,mesid
 	      FROM online.tl_hdfs_auroracallback_view--推送push的回调表
-	      where partition_day>=regexp_replace(date_sub(current_date,30),'-','') and partition_day<=regexp_replace(date_sub(current_date,1),'-','')
+	      where partition_date>=regexp_replace(date_sub(current_date,30),'-','') and partition_date<=regexp_replace(date_sub(current_date,1),'-','')
 	      --and notification_state='true'--用户开启通知
 	      and callback_type = '0'--送达回执
 	      and platform='android'

pm/daily_operation/job/step1_6.job

-#step1_3.job
-type=command
-command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_channel_d
\ No newline at end of file

pm/daily_operation/job/step2.job

 #step2.job
 type=command
-dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17,step1_18,step1_19,step1_20,step1_21,step1_22,step1_23,step1_24,step1_25,step1_26,step1_27
+dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13,step1_14,step1_15,step1_16,step1_17,step1_18,step1_19,step1_20,step1_21,step1_22,step1_23,step1_24,step1_25,step1_26,step1_27
 command=sh /home/bi/bi-report/lib/shell/hive daily_operation
\ No newline at end of file

pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql

@@ -68,7 +68,12 @@ FROM
                        WHEN transaction_type in ('-1','smr') THEN array('smr','合计')
                        when transaction_type in ('pgc','hotspot') then array('热点卡片','合计')
                        when transaction_type in ('newdata') then array('保量卡片')
-                       when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计') end AS recommend_type,
+                       when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')
+                       when transaction_type in ('aistragegy') then array('新用户AI帖优先')
+                       when transaction_type in ('excestragegy') then array('新用户精华帖优先')
+                       when transaction_type in ('fixedstragegy') then array(' 新氧新用户策略一')
+                       when transaction_type in ('fixedstragegy_video') then array('新氧新用户策略二') end
+                       AS recommend_type,
                   card_id,
                   app_session_id
            from online.ml_community_precise_exposure_detail
@@ -77,7 +82,7 @@ FROM
            AND is_exposure = '1'  ----精准曝光
            AND page_name ='home'
            AND tab_name = '精选'
-           AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
+           AND (transaction_type in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
                 or transaction_type like '%ctr' or transaction_type like '%cvr')
            AND card_content_type in ('qa','diary','user_post','answer','special_pool')
            group by partition_date,
@@ -90,7 +95,11 @@ FROM
                        WHEN transaction_type in ('-1','smr') THEN array('smr','合计')
                        when transaction_type in ('pgc','hotspot') then array('热点卡片','合计')
                        when transaction_type in ('newdata') then array('保量卡片')
-                       when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')end,
+                       when transaction_type in ('hotspot_feed') then array('hotspot_feed','合计')
+                       when transaction_type in ('aistragegy') then array('新用户AI帖优先')
+                       when transaction_type in ('excestragegy') then array('新用户精华帖优先')
+                       when transaction_type in ('fixedstragegy') then array(' 新氧新用户策略一')
+                       when transaction_type in ('fixedstragegy_video') then array('新氧新用户策略二') end,
                   card_id,
                   app_session_id
         )a
@@ -98,7 +107,7 @@ FROM
         group by partition_date,card_content_type,cl_id,v.recommend_type,card_id
       )t2
     on t1.device_id=t2.cl_id and t1.partition_date=t2.partition_date
-        LEFT JOIN
+    LEFT JOIN
     (--卡片，卡片id和session_id去重
         SELECT  partition_date,
                 card_content_type,
@@ -118,7 +127,12 @@ FROM
                        WHEN params['transaction_type'] in ('-1','smr') THEN array('smr','合计')
                        when params['transaction_type'] in ('pgc','hotspot') then array('热点卡片','合计')
                        when params['transaction_type'] in ('newdata') then array('保量卡片')
-                       when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')end AS recommend_type,
+                       when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')
+                       when params['transaction_type'] in ('aistragegy') then array('新用户AI帖优先')
+                       when params['transaction_type'] in ('excestragegy') then array('新用户精华帖优先')
+                       when params['transaction_type'] in ('fixedstragegy') then array(' 新氧新用户策略一')
+                       when params['transaction_type'] in ('fixedstragegy_video') then array('新氧新用户策略二')
+                       end AS recommend_type,
                   params['card_id'] as card_id,
                   app_session_id
            from online.bl_hdfs_maidian_updates
@@ -126,7 +140,7 @@ FROM
            AND action='on_click_card'
            AND params['page_name'] ='home'
            AND params['tab_name'] = '精选'
-           AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed')
+           AND (params['transaction_type'] in ('-1','smr','hotspot','pgc','newdata','hotspot_feed','aistragegy','excestragegy','fixedstragegy','fixedstragegy_video')
                 or params['transaction_type'] like '%ctr' or params['transaction_type'] like '%cvr')
            AND params['card_content_type'] in ('qa','diary','user_post','answer','special_pool')
            GROUP BY partition_date,
@@ -139,7 +153,11 @@ FROM
                        WHEN params['transaction_type'] in ('-1','smr') THEN array('smr','合计')
                        when params['transaction_type'] in ('pgc','hotspot') then array('热点卡片','合计')
                        when params['transaction_type'] in ('newdata') then array('保量卡片')
-                       when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')end,
+                       when params['transaction_type'] in ('hotspot_feed') then array('hotspot_feed','合计')
+                       when params['transaction_type'] in ('aistragegy') then array('新用户AI帖优先')
+                       when params['transaction_type'] in ('excestragegy') then array('新用户精华帖优先')
+                       when params['transaction_type'] in ('fixedstragegy') then array(' 新氧新用户策略一')
+                       when params['transaction_type'] in ('fixedstragegy_video') then array('新氧新用户策略二') end,
                   params['card_id'],
                   app_session_id
         )a
@@ -558,7 +576,7 @@ FROM
 	partition_date,device_id,device_os_type
 	,case WHEN active_type = '4'  THEN '老活跃设备'
     		      WHEN active_type  in ('1','2')  then '新增设备' END as active_type
-    ,CASE WHEN substr(convup(setencryption(device_id,'sha-1'),16,10),-1) in ('0','1','2','3','4') THEN '灰度' ELSE '非灰' END AS grey_type
+    , '灰度' AS grey_type
 	FROM online.ml_device_day_active_status
 	where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
 	AND active_type in ('1','2','4')