dist predict test

eda/esmm/Model_pipline/dist_predict.py

@@ -55,7 +55,6 @@ def input_fn(filenames, batch_size=32, num_epochs=1, perform_shuffle=False):
 
     return batch_features, batch_labels
 
-
 def model_fn(features, labels, mode, params):
     """Bulid Model function f(x) for Estimator."""
     #------hyperparameters----
@@ -268,14 +267,17 @@ if __name__ == "__main__":
 
         rdd_te_files = spark.sparkContext.parallelize(te_files)
         print("-"*100)
-        indices = rdd_te_files.repartition(40).map(lambda x: main(x))
-        print(indices.take(1))
+        # indices = rdd_te_files.repartition(40).map(lambda x: main(x))
+        # print(indices.take(1))
         print("dist predict nearby")
 
-        te_result_dataframe = spark.createDataFrame(indices.flatMap(lambda x: x.split(";")).map(
-            lambda l: Row(uid=l.split(":")[0],city=l.split(":")[1],cid_id=l.split(":")[2],ctcvr=l.split(":")[3])))
+        test = main(te_files[0])
+        print(test[:50])
 
-        te_result_dataframe.show()
+        # te_result_dataframe = spark.createDataFrame(indices.flatMap(lambda x: x.split(";")).map(
+        #     lambda l: Row(uid=l.split(":")[0],city=l.split(":")[1],cid_id=l.split(":")[2],ctcvr=l.split(":")[3])))
+        #
+        # te_result_dataframe.show()
 
         # print("nearby rdd data")
         # te_result_dataframe.show()