write date to csv

word_vector/tractate.py

@@ -157,9 +157,12 @@ if __name__ == "__main__":
     spark = get_spark("test")
     click_df = get_tracate_click_data(spark, get_ndays_before_no_minus(30), get_ndays_before_no_minus(1))
     click_df.show(5, False)
+
     res_dict = get_device_click_tractate_ids_dict(click_df)
+
     with open(os.path.join(data_dir, "click_tractate_ids.csv"), "w") as f:
-        for (k, v) in res_dict:
-            f.write(k + "|" + ",".join(v))
+        for (k, v) in res_dict.items():
+            if v:
+                f.write("{}|{}\n".format(k, ",".join([str(x) for x in v])))
 
 # spark-submit --master yarn --deploy-mode client --queue root.strategy --driver-memory 16g --executor-memory 1g --executor-cores 1 --num-executors 70 --conf spark.default.parallelism=100 --conf spark.storage.memoryFraction=0.5 --conf spark.shuffle.memoryFraction=0.3 --conf spark.locality.wait=0 --jars /srv/apps/tispark-core-2.1-SNAPSHOT-jar-with-dependencies.jar,/srv/apps/spark-connector_2.11-1.9.0-rc2.jar,/srv/apps/mysql-connector-java-5.1.38.jar /srv/apps/strategy_embedding/word_vector/tractate.py