reduce collect()

task/search_strategy_d.py

@@ -71,7 +71,7 @@ spark.sql("CREATE TEMPORARY FUNCTION is_json AS 'com.gmei.hive.common.udf.UDFJso
 spark.sql("CREATE TEMPORARY FUNCTION arrayMerge AS 'com.gmei.hive.common.udf.UDFArryMerge'")
 
 task_list = []
-task_days = 5
+task_days = 2
 
 for t in range(0, task_days):
     day_num = 0 - t
@@ -151,8 +151,8 @@ for t in range(0, task_days):
     spam_pv_df = spark.sql(sql_spam_pv_device_id)
     spam_pv_df.createOrReplaceTempView("spam_pv")
     spam_pv_df.show(1)
-    sql_res = spam_pv_df.collect()
-    spam_pv_df.cache()
+    # sql_res = spam_pv_df.collect()
+    # spam_pv_df.cache()
     # for res in sql_res:
     #     print(res)