降配

task/home_feed_recommend_strategy_d.py

@@ -197,61 +197,36 @@ spark_df = spark.sql(HIVE_SQL)
 spark_df.show(100,False)
 
 pandas_df = spark_df.toPandas()
+pandas_df.fillna(0,inplace=True)
+pandas_df['卡片ctr'] = pandas_df['卡片点击pv']/pandas_df['卡片曝光pv']
 print(pandas_df)
 
-# for res in sql_res:
-#     # print(res)
-#     day_id = res.day_id
-#     device_os_type = res.device_os_type
-#     active_type = res.active_type
-#     card_content_type = res.card_content_type
-#     recommend_type = res.recommend_type
-#     card_click = res.card_click
-#     card_exposure = res.card_exposure
-#     avg_page_stay = res.avg_page_stay
-#     navbar_search = res.navbar_search
-#     highlight_word = res.highlight_word
-#     self_welfare_card = res.self_welfare_card
-#     page_pv_20 = res.page_pv_20
-#     page_pv_20_percent = res.page_pv_20_percent
-#
-#
-#     recommend_welfare_card = res.recommend_welfare_card
-#     recommend_content_card = res.recommend_content_card
-#     if not recommend_content_card:
-#         recommend_content_card = 0
-#     recommend_special_card = res.recommend_special_card
-#     if not recommend_special_card:
-#         recommend_special_card = 0
-#     transfer_card = res.transfer_card
-#     video_consultation = res.video_consultation
-#     partition_day = res.partition_day
-#     recommend_ctr = res.recommend_ctr
-#     second_jump_ctr = res.second_jump_ctr
-#     click_ctr = res.click_ctr
-#     pid = hashlib.md5((day_id + device_os_type + active_type + card_content_type + recommend_type).encode("utf8")).hexdigest()
-#     instert_sql = """replace into recommend_strategy_d(
-#     day_id,device_os_type,active_type,card_content_type,recommend_type,card_click,card_exposure,avg_page_stay,navbar_search,
-#     highlight_word,self_welfare_card,recommend_welfare_card,recommend_content_card,recommend_special_card,transfer_card,video_consultation,
-#     partition_day,pid,recommend_ctr,second_jump_ctr,click_ctr,page_pv_20_percent
-#     ) VALUES('{day_id}','{device_os_type}','{active_type}','{card_content_type}','{recommend_type}',{card_click},{card_exposure},
-#     {avg_page_stay},{navbar_search},{highlight_word},{self_welfare_card},{recommend_welfare_card},{recommend_content_card},{recommend_special_card},
-#     {transfer_card},{video_consultation},'{partition_day}','{pid}',{recommend_ctr},{second_jump_ctr},{click_ctr},{page_pv_20_percent});""".format(
-#         day_id=day_id, device_os_type=device_os_type, active_type=active_type, card_content_type=card_content_type,
-#         card_click=card_click, recommend_type=recommend_type, card_exposure=card_exposure, avg_page_stay=avg_page_stay,
-#         navbar_search=navbar_search, self_welfare_card=self_welfare_card, recommend_welfare_card=recommend_welfare_card,
-#         recommend_content_card=recommend_content_card, recommend_special_card=recommend_special_card,page_pv_20_percent=page_pv_20_percent,
-#         transfer_card=transfer_card,
-#         video_consultation=video_consultation, partition_day=partition_day, pid=pid, recommend_ctr=recommend_ctr,
-#         second_jump_ctr=second_jump_ctr, click_ctr=click_ctr,highlight_word=highlight_word
-#     )
-#     print(instert_sql)
-#     # cursor.execute("set names 'UTF8'")
-#     res = cursor.execute(instert_sql)
-#     db.commit()
-#     print(res)
-# # cursor.executemany()
-# db.close()
+for row in pandas_df.iterrows():
+    partition_date = row[1]['日期']
+    device_os_type = row[1]['设备类型']
+    active_type = row[1]['活跃类型']
+    card_content_type = row[1]['卡片类型']
+    is_cpc = row[1]['是否cpc']
+    exp_pv = row[1]['卡片曝光pv']
+    click_pv = row[1]['卡片点击pv']
+    ctr = row[1]['卡片ctr']
+    pid = hashlib.md5((partition_date + device_os_type + active_type + card_content_type + is_cpc).encode("utf8")).hexdigest()
+    instert_sql = """replace into home_feed_recommend_strategy_d(
+    day_id,device_os_type,active_type,card_content_type,recommend_type,card_click,card_exposure,avg_page_stay,navbar_search,
+    highlight_word,self_welfare_card,recommend_welfare_card,recommend_content_card,recommend_special_card,transfer_card,video_consultation,
+    partition_day,pid,recommend_ctr,second_jump_ctr,click_ctr,page_pv_20_percent
+    ) VALUES('{partition_date}','{device_os_type}','{active_type}','{card_content_type}','{is_cpc}',{exp_pv},{click_pv},
+    {ctr},'{pid}');""".format(
+        partition_date=partition_date, device_os_type=device_os_type, active_type=active_type, card_content_type=card_content_type,
+        is_cpc=is_cpc, exp_pv=exp_pv, click_pv=click_pv, ctr=ctr,pid=pid
+    )
+    print(instert_sql)
+    # cursor.execute("set names 'UTF8'")
+    res = cursor.execute(instert_sql)
+    db.commit()
+    print(res)
+# cursor.executemany()
+db.close()