update

task/recommend_strategy_d.py

@@ -28,9 +28,10 @@ from pyspark.sql.functions import lit
 import pytispark.pytispark as pti
 
 db = pymysql.connect(host='172.16.40.158', port=4000, user='st_user', passwd='aqpuBLYzEV7tML5RPsN1pntUzFy',
-                         db='jerry_prod')
+                     db='jerry_prod')
 cursor = db.cursor()
 
+
 def con_sql(sql):
     # 从数据库的表里获取数据
 
@@ -78,17 +79,11 @@ spark.sql("CREATE TEMPORARY FUNCTION json_map AS 'brickhouse.udf.json.JsonMapUDF
 spark.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJsonFormatCheck'")
 spark.sql("CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDFArryMerge'")
 
-ungrey_select_sql = """SELECT *,NVL(ROUND(wel_click_pv/wel_exp_pv,6),0) as meigou_ctr,
-    NVL(ROUND(content_click_pv/content_exp_pv,6),0) as neirong_ctr FROM pm.tl_pm_contentpage_ctr where grey_type = '非灰'"""
-ungrey_df = spark.sql(ungrey_select_sql)
-ungrey_df.createOrReplaceTempView("ungrey")
-
-grey_select_sql = """SELECT *,NVL(ROUND(wel_click_pv/wel_exp_pv,6),0) as grey_meigou_ctr,
-    NVL(ROUND(content_click_pv/content_exp_pv,6),0) as grey_neirong_ctr FROM pm.tl_pm_contentpage_ctr where grey_type = '灰度'"""
-grey_df = spark.sql(grey_select_sql)
-grey_df.createOrReplaceTempView("grey")
+grey_select_sql = """SELECT *,NVL(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_exposure,5),0) as recommend_ctr,
+    NVL(ROUND(card_click/card_exposure,5),0) as click_ctr,
+    NVL(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2+transfer_card+video_consultation)/card_click,5),0) as second_jump_ctr, FROM pm.tl_pm_recommend_strategy_d"""
+device_df = spark.sql(grey_select_sql)
 
-sql = """SELECT *,ungrey.meigou_ctr,ungrey.neirong_ctr from grey"""
 device_df.show(1, False)
 sql_res = device_df.collect()
 print("-----------------------------------------------------------------------------")
@@ -97,30 +92,39 @@ for res in sql_res:
     day_id = res.day_id
     device_os_type = res.device_os_type
     active_type = res.active_type
-    grey_type = res.grey_type
-    page_name = res.page_name
-    content_pv = res.content_pv
-    content_uv = res.content_uv
-    wel_exp_pv = res.wel_exp_pv
-    content_exp_pv = res.content_exp_pv
-    meigou_ctr=res.meigou_ctr
-    neirong_ctr=res.neirong_ctr
+    card_content_type = res.card_content_type
+    recommend_type = res.recommend_type
+    card_click = res.card_click
+    card_exposure = res.card_exposure
+    avg_page_stay = res.avg_page_stay
+    navbar_search = res.navbar_search
+    highlight_word = res.highlight_word
+    self_welfare_card = res.self_welfare_card
 
-    wel_click_pv = res.wel_click_pv
-    content_click_pv = res.content_click_pv
-    slide_wel_click_pv = res.slide_wel_click_pv
-    self_wel_click_pv = res.self_wel_click_pv
-    partition_day = res.PARTITION_DAY
-    pid = hashlib.md5((day_id + device_os_type + active_type + grey_type + page_name).encode("utf8")).hexdigest()
-    instert_sql = """replace into conent_detail_page_grayscale_ctr(
-    day_id,device_os_type,active_type,grey_type,page_name,content_pv,content_uv,wel_exp_pv,
-    content_exp_pv,wel_click_pv,content_click_pv,slide_wel_click_pv,self_wel_click_pv,partition_day,pid,meigou_ctr,neirong_ctr
-    ) VALUES('{day_id}','{device_os_type}','{active_type}','{grey_type}','{page_name}',{content_pv},{content_uv},
-    {wel_exp_pv},{content_exp_pv},{wel_click_pv},{content_click_pv},{slide_wel_click_pv},{self_wel_click_pv},'{partition_day}','{pid}',{meigou_ctr},{neirong_ctr});""".format(
-        day_id=day_id,device_os_type=device_os_type,active_type=active_type,grey_type=grey_type,page_name=page_name,
-        content_pv=content_pv,content_uv=content_uv,wel_exp_pv=wel_exp_pv,content_exp_pv=content_exp_pv,wel_click_pv=wel_click_pv,
-        content_click_pv=content_click_pv,slide_wel_click_pv=slide_wel_click_pv,self_wel_click_pv=self_wel_click_pv,meigou_ctr=meigou_ctr,neirong_ctr=neirong_ctr,
-        partition_day=partition_day, pid=pid
+    recommend_welfare_card = res.recommend_welfare_card
+    recommend_content_card = res.recommend_content_card
+    recommend_special_card = res.recommend_special_card
+    transfer_card = res.transfer_card
+    video_consultation = res.video_consultation
+    partition_day = res.partition_day
+    recommend_ctr = res.recommend_ctr
+    second_jump_ctr = res.second_jump_ctr
+    click_ctr = res.click_ctr
+    pid = hashlib.md5((day_id + device_os_type + active_type + card_content_type + recommend_type).encode("utf8")).hexdigest()
+    instert_sql = """replace into recommend_strategy_d(
+    day_id,device_os_type,active_type,card_content_type,recommend_type,card_click,card_exposure,avg_page_stay,navbar_search,
+    highlight_word,self_welfare_card,recommend_welfare_card,recommend_content_card,recommend_special_card,transfer_card,video_consultation,
+    partition_day,pid,recommend_ctr,second_jump_ctr,click_ctr
+    ) VALUES('{day_id}','{device_os_type}','{active_type}','{card_content_type}','{recommend_type}',{card_click},{card_exposure},
+    {avg_page_stay},{navbar_search},{self_welfare_card},{recommend_welfare_card},{recommend_content_card},{recommend_special_card},
+    {transfer_card},{video_consultation},'{partition_day}','{pid}',{recommend_ctr},{second_jump_ctr},{click_ctr});""".format(
+        day_id=day_id, device_os_type=device_os_type, active_type=active_type, card_content_type=card_content_type,
+        card_click=card_click, recommend_type=recommend_type, card_exposure=card_exposure, avg_page_stay=avg_page_stay,
+        navbar_search=navbar_search, self_welfare_card=self_welfare_card, recommend_welfare_card=recommend_welfare_card,
+        recommend_content_card=recommend_content_card, recommend_special_card=recommend_special_card,
+        transfer_card=transfer_card,
+        video_consultation=video_consultation, partition_day=partition_day, pid=pid, recommend_ctr=recommend_ctr,
+        second_jump_ctr=second_jump_ctr, click_ctr=click_ctr
     )
     print(instert_sql)
     # cursor.execute("set names 'UTF8'")
@@ -128,4 +132,4 @@ for res in sql_res:
     db.commit()
     print(res)
 # cursor.executemany()
-db.close()
\ No newline at end of file
+db.close()