降配

task/home_feed_recommend_strategy_d.py

@@ -193,7 +193,70 @@ group by partition_date
 	  ,e.is_cpc
 """.format(START_PARTITION_DATE,START_PARTITION_DATE,START_PARTITION_DATE)
 
-spark.sql(HIVE_SQL).show(100,False)
+spark_df = spark.sql(HIVE_SQL)
+spark_df.show(100,False)
+
+pandas_df = spark_df.toPandas()
+print(pandas_df)
+
+# for res in sql_res:
+#     # print(res)
+#     day_id = res.day_id
+#     device_os_type = res.device_os_type
+#     active_type = res.active_type
+#     card_content_type = res.card_content_type
+#     recommend_type = res.recommend_type
+#     card_click = res.card_click
+#     card_exposure = res.card_exposure
+#     avg_page_stay = res.avg_page_stay
+#     navbar_search = res.navbar_search
+#     highlight_word = res.highlight_word
+#     self_welfare_card = res.self_welfare_card
+#     page_pv_20 = res.page_pv_20
+#     page_pv_20_percent = res.page_pv_20_percent
+#
+#
+#     recommend_welfare_card = res.recommend_welfare_card
+#     recommend_content_card = res.recommend_content_card
+#     if not recommend_content_card:
+#         recommend_content_card = 0
+#     recommend_special_card = res.recommend_special_card
+#     if not recommend_special_card:
+#         recommend_special_card = 0
+#     transfer_card = res.transfer_card
+#     video_consultation = res.video_consultation
+#     partition_day = res.partition_day
+#     recommend_ctr = res.recommend_ctr
+#     second_jump_ctr = res.second_jump_ctr
+#     click_ctr = res.click_ctr
+#     pid = hashlib.md5((day_id + device_os_type + active_type + card_content_type + recommend_type).encode("utf8")).hexdigest()
+#     instert_sql = """replace into recommend_strategy_d(
+#     day_id,device_os_type,active_type,card_content_type,recommend_type,card_click,card_exposure,avg_page_stay,navbar_search,
+#     highlight_word,self_welfare_card,recommend_welfare_card,recommend_content_card,recommend_special_card,transfer_card,video_consultation,
+#     partition_day,pid,recommend_ctr,second_jump_ctr,click_ctr,page_pv_20_percent
+#     ) VALUES('{day_id}','{device_os_type}','{active_type}','{card_content_type}','{recommend_type}',{card_click},{card_exposure},
+#     {avg_page_stay},{navbar_search},{highlight_word},{self_welfare_card},{recommend_welfare_card},{recommend_content_card},{recommend_special_card},
+#     {transfer_card},{video_consultation},'{partition_day}','{pid}',{recommend_ctr},{second_jump_ctr},{click_ctr},{page_pv_20_percent});""".format(
+#         day_id=day_id, device_os_type=device_os_type, active_type=active_type, card_content_type=card_content_type,
+#         card_click=card_click, recommend_type=recommend_type, card_exposure=card_exposure, avg_page_stay=avg_page_stay,
+#         navbar_search=navbar_search, self_welfare_card=self_welfare_card, recommend_welfare_card=recommend_welfare_card,
+#         recommend_content_card=recommend_content_card, recommend_special_card=recommend_special_card,page_pv_20_percent=page_pv_20_percent,
+#         transfer_card=transfer_card,
+#         video_consultation=video_consultation, partition_day=partition_day, pid=pid, recommend_ctr=recommend_ctr,
+#         second_jump_ctr=second_jump_ctr, click_ctr=click_ctr,highlight_word=highlight_word
+#     )
+#     print(instert_sql)
+#     # cursor.execute("set names 'UTF8'")
+#     res = cursor.execute(instert_sql)
+#     db.commit()
+#     print(res)
+# # cursor.executemany()
+# db.close()
+
+
+
+
+
 
 # t = 1
 # day_num = 0 - t