change test file

tensnsorflow/feature_test.py

@@ -276,7 +276,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
           "left join jerry_test.order_tag ot on e.device_id = ot.device_id " \
           "left join jerry_test.sixin_tag sixin on e.device_id = sixin.device_id " \
           "left join jerry_test.cart_tag cart on e.device_id = cart.device_id " \
-          "left join jerry_test.knowledge k on feat.level2 = k.level2_id"
+          "left join jerry_test.knowledge k on feat.level2 = k.level2_id limit 6000"
 
     features = ["ucity_id", "ccity_name", "device_type", "manufacturer",
                 "channel", "top", "time", "hospital_id",
@@ -312,18 +312,18 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
 
     rdd.persist(storageLevel= StorageLevel.MEMORY_ONLY_SER)
 
-    native_pre = spark.createDataFrame(rdd.filter(lambda x:x[0] == 0).map(lambda x:(x[3],x[4],x[5])))\
-        .toDF("city","uid","cid_id")
-    print("native csv")
-    native_pre.toPandas().to_csv(local_path + "native.csv", header=True)
-    spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
-                          .map(lambda x: (x[1],x[2],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13],x[14],x[15],x[16],x[17]))) \
-        .toDF("y","z","app_list", "level2_list", "level3_list","tag1_list", "tag2_list", "tag3_list", "tag4_list",
-              "tag5_list", "tag6_list", "tag7_list", "ids","number").repartition(1).write.format("tfrecords") \
-            .save(path=path+"native/", mode="overwrite")
-    print("native tfrecord done")
-    h = time.time()
-    print((h-f)/60)
+    # native_pre = spark.createDataFrame(rdd.filter(lambda x:x[0] == 0).map(lambda x:(x[3],x[4],x[5])))\
+    #     .toDF("city","uid","cid_id")
+    # print("native csv")
+    # native_pre.toPandas().to_csv(local_path + "native.csv", header=True)
+    # spark.createDataFrame(rdd.filter(lambda x: x[0] == 0)
+    #                       .map(lambda x: (x[1],x[2],x[6],x[7],x[8],x[9],x[10],x[11],x[12],x[13],x[14],x[15],x[16],x[17]))) \
+    #     .toDF("y","z","app_list", "level2_list", "level3_list","tag1_list", "tag2_list", "tag3_list", "tag4_list",
+    #           "tag5_list", "tag6_list", "tag7_list", "ids","number").repartition(1).write.format("tfrecords") \
+    #         .save(path=path+"native/", mode="overwrite")
+    # print("native tfrecord done")
+    # h = time.time()
+    # print((h-f)/60)
 
     nearby_pre = spark.createDataFrame(rdd.filter(lambda x: x[0] == 1).map(lambda x: (x[3], x[4], x[5]))) \
         .toDF("city", "uid", "cid_id")
@@ -334,7 +334,7 @@ def get_predict(date,value_map,app_list_map,leve2_map,leve3_map):
                           .map(
         lambda x: (x[1], x[2], x[6], x[7], x[8], x[9], x[10], x[11], x[12], x[13], x[14], x[15], x[16],x[17]))) \
         .toDF("y", "z", "app_list", "level2_list", "level3_list", "tag1_list", "tag2_list", "tag3_list", "tag4_list",
-              "tag5_list", "tag6_list", "tag7_list", "ids","number").repartition(1).write.format("tfrecords") \
+              "tag5_list", "tag6_list", "tag7_list", "ids","number").repartition(100).write.format("tfrecords") \
         .save(path=path + "nearby/", mode="overwrite")
     print("nearby tfrecord done")