bug fix

eda/esmm/Model_pipline/dist_predict.py

@@ -9,10 +9,6 @@ import time
 from pyspark import StorageLevel
 import os
 
-# os.environ["PYSPARK_PYTHON"] = "/opt/anaconda3/envs/esmm/bin/python"
-os.environ["LD_LIBRARY_PATH"] = "/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server:/opt/hadoop/lib/native"
-# os.environ["PYSPARK_PYTHON"] = "/opt/anaconda3/envs/esmm/bin/python"
-
 def model_fn(features, labels, mode, params):
     """Bulid Model function f(x) for Estimator."""
     #------hyperparameters----
@@ -200,12 +196,13 @@ if __name__ == "__main__":
     print(test.collect())
 
     tf.logging.set_verbosity(tf.logging.INFO)
-    te_files = [["hdfs://172.16.32.4:8020/strategy/esmm/nearby/part-r-00000"],["hdfs://172.16.32.4:8020/strategy/esmm/native/part-r-00000"]]
-    rdd_te_files = spark.sparkContext.parallelize(te_files)
-    indices = rdd_te_files.repartition(2).map(lambda x: main(x))
-    print(indices.collect())
+    te_files = [[path+"nearby/part-r-00000"],[path+"native/part-r-00000"]]
+
+    # rdd_te_files = spark.sparkContext.parallelize(te_files)
+    # indices = rdd_te_files.repartition(2).map(lambda x: main(x))
+    # print(indices.collect())
 
-    # main(te_files)
+    main(te_files)
 
     b = time.time()
     print("耗时(分钟)：")