test features

eda/esmm/Model_pipline/dist_predict.py

@@ -176,9 +176,6 @@ def main(te_file):
     #     indices.append([prob['pctr'], prob['pcvr'], prob['pctcvr']])
     # return indices
 
-def test_map(x):
-    return x * x
-
 
 if __name__ == "__main__":
     b = time.time()
@@ -195,31 +192,25 @@ if __name__ == "__main__":
     spark.sparkContext.setLogLevel("WARN")
 
     path = "hdfs://172.16.32.4:8020/strategy/esmm/"
-    # df = spark.read.format("tfrecords").load(path+"nearby/part-r-00000")
-    # df.show()
-
-    # name = spark.sparkContext.parallelize([1,2,3,4,5])
-    #
-    # test = name.repartition(5).map(lambda x: test_map(x))
-    # print(test)
-    # print(test.collect())
-    #
+    df = spark.read.format("tfrecords").load(path+"nearby/part-r-00000")
+    df.show()
+
     # tf.logging.set_verbosity(tf.logging.INFO)
-    #
-    # te_files = [path + "nearby/part-r-00000"]
-    te_files = []
-    for i in range(0,10):
-        te_files.append([path + "native/part-r-0000" + str(i)])
-    for i in range(10,100):
-        te_files.append([path + "native/part-r-000" + str(i)])
-    # main(te_files)
-    # te_files = [[path+"nearby/part-r-00000"],[path+"native/part-r-00000"]]
-    rdd_te_files = spark.sparkContext.parallelize(te_files)
-    print("-"*100)
-    print(rdd_te_files.collect())
-    print("-" * 100)
-    indices = rdd_te_files.repartition(100).map(lambda x: main(x))
-    print(indices.take(1))
+
+    # te_files = []
+    # for i in range(0,10):
+    #     te_files.append([path + "native/part-r-0000" + str(i)])
+    # for i in range(10,100):
+    #     te_files.append([path + "native/part-r-000" + str(i)])
+    # # main(te_files)
+    # # te_files = [[path+"nearby/part-r-00000"],[path+"native/part-r-00000"]]
+    # rdd_te_files = spark.sparkContext.parallelize(te_files)
+    # print("-"*100)
+    # print(rdd_te_files.collect())
+    # print("-" * 100)
+    # indices = rdd_te_files.repartition(100).map(lambda x: main(x))
+    # print(indices.take(1))
+
 
     print("耗时(秒)：")
     print((time.time()-b))