Commit 472de2ab authored by 张彦钊's avatar 张彦钊

change test file

parent 2061b704
......@@ -309,7 +309,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
native_pre = spark.createDataFrame(rdd.filter(lambda x:x[0] == 0).map(lambda x:(x[3],x[4],x[5])))\
.toDF("city","uid","cid_id")
print("native csv")
native_pre.repartion(1).write.format('com.databricks.spark.csv').save(path + "native/", header='true')
native_pre.repartition(1).write.format('com.databricks.spark.csv').save(path + "native/", header='true')
spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
.map(lambda x: (x[1],x[2],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13],x[14],x[15],x[16]))) \
.toDF("y","z","app_list", "level2_list", "level3_list","tag1_list", "tag2_list", "tag3_list", "tag4_list",
......@@ -322,7 +322,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
nearby_pre = spark.createDataFrame(rdd.filter(lambda x: x[0] == 1).map(lambda x: (x[3], x[4], x[5]))) \
.toDF("city", "uid", "cid_id")
print("nearby csv")
nearby_pre.repartion(1).write.format('com.databricks.spark.csv').save(path + "nearby/", header='true')
nearby_pre.repartition(1).write.format('com.databricks.spark.csv').save(path + "nearby/", header='true')
spark.createDataFrame(rdd.filter(lambda x: x[0] == 1)
.map(
......@@ -376,7 +376,7 @@ if __name__ == '__main__':
"where e.stat_date >= '2019-06-10'"
df = spark.sql(sql)
df.repartion(1).write.format('com.databricks.spark.csv').save(path + "native/a.csv", header='true')
df.repartition(1).write.format('com.databricks.spark.csv').save(path + "native/a.csv", header='true')
spark.stop()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment