update

eda/smart_rank/gyz_test.py

@@ -99,12 +99,14 @@ device_ids_lst_rdd = spark.sparkContext.parallelize(device_info)
 result = device_ids_lst_rdd.repartition(100).map(lambda x: get_user_service_portrait(x, all_word_tags, all_tag_tag_type, all_3tag_2tag, all_tags_name, size=None, pay_time=pay_time))
 print(result.take(10))
 result1 = result.map(
-    lambda x: (x[0], x[1], x[2])
+    lambda x: (str(x[0]), str(x[1]), str(x[2]))
 )
-path = "hdfs:///strategy/esmm/"
-spark.createDataFrame(result1).toDF("device", "search_words", "user_portrait").repartition(1).write.format("csv").save(path=path + "portrait/", mode="overwrite")
+spark.createDataFrame(result1).toDF("device", "search_words", "user_portrait").coalesce(1).write.format('com.databricks.spark.csv').save("~/test_df.csv", header='true')
 
-result.saveAsTextFile("~/test_df.csv")
+
+# path = "hdfs:///strategy/esmm/"
+# spark.createDataFrame(result1).toDF("device", "search_words", "user_portrait").repartition(1).write.format("csv").save(path=path + "portrait/", mode="overwrite")
+# result.saveAsTextFile("~/test_df.csv")
 # df = result.toDF()
 # df.show()
 # result.write.format('csv').save("~/test_df.csv")