模型调试

spark/featureEngSk.py

@@ -45,10 +45,10 @@ DATA_PATH_TRAIN = "/data/files/service_feature_{}_train.csv".format(VERSION)
 
 
 def getRedisConn():
-    # pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
-    # conn = redis.Redis(connection_pool=pool)
+    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
     # conn = redis.Redis(host="172.16.50.145", port=6379, password="XfkMCCdWDIU%ls$h",db=0)
-    conn = redis.Redis(host="172.18.51.10", port=6379,db=0) #test
+    # conn = redis.Redis(host="172.18.51.10", port=6379,db=0) #test
     return conn
 
 def parseTags(tags,i):
@@ -1043,8 +1043,8 @@ if __name__ == '__main__':
     output_file = "/strategy/train_samples"
     train_samples.write.mode("overwrite").options(header="false", sep='|').csv(output_file)
 
-    output_file = "file:///home/gmuser/test_samples"
-    output_file = "/strategy/test_samples"
+    output_file = "file:///home/gmuser/eval_samples"
+    output_file = "/strategy/eval_samples"
     test_samples.write.mode("overwrite").options(header="false", sep='|').csv(output_file)
 
     print("训练数据写入 耗时s:{}".format(time.time() - write_time_start))

train/down_vocab.py

@@ -4,10 +4,10 @@ import os
 import json
 
 def getRedisConn():
-    # pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
-    # conn = redis.Redis(connection_pool=pool)
+    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
     # conn = redis.Redis(host="172.16.50.145", port=6379, password="XfkMCCdWDIU%ls$h",db=0)
-    conn = redis.Redis(host="172.18.51.10", port=6379, db=0, decode_responses = True) #test
+    # conn = redis.Redis(host="172.18.51.10", port=6379, db=0, decode_responses = True) #test
     return conn
 
 if len(sys.argv) == 2:
@@ -41,5 +41,5 @@ for vocab_key in vocab_keys:
         f.write('\n'.join(texts))
 
 os.system("hdfs dfs -getmerge /strategy/train_samples {save_dir}train_samples.csv".format(save_dir = save_dir))
-os.system("hdfs dfs -getmerge /strategy/test_samples {save_dir}test_samples.csv".format(save_dir = save_dir))
+os.system("hdfs dfs -getmerge /strategy/eval_samples {save_dir}eval_samples.csv".format(save_dir = save_dir))