Merge branch 'master' of http://git.wanmeizhensuo.com/ML/ffm-baseline

eda/esmm/Model_pipline/submit.sh

@@ -4,6 +4,7 @@ PYTHON_PATH=/srv/envs/esmm/bin/python
 MODEL_PATH=/srv/apps/ffm-baseline_git/eda/esmm/Model_pipline
 LOCAL_PATH=/home/gmuser/esmm
 HDFS_PATH=hdfs://172.16.32.4:8020/strategy/esmm
+CLASSPATH=hadoop classpath --glob
 
 echo "rm model file"
 rm -r ${LOCAL_PATH}/model_ckpt/DeepCvrMTL/20*
@@ -11,12 +12,3 @@ rm -r ${LOCAL_PATH}/model_ckpt/DeepCvrMTL/20*
 echo "train..."
 CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --task_type=train
 
-echo "infer native..."
-CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH}/native --hdfs_dir=${HDFS_PATH}/native --task_type=infer
-
-
-echo "infer nearby..."
-CLASSPATH="$(hadoop classpath --glob)" ${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH}/nearby --hdfs_dir=${HDFS_PATH}/nearby --task_type=infer
-
-echo "sort and 2sql"
-${PYTHON_PATH} ${MODEL_PATH}/to_database.py
\ No newline at end of file

eda/esmm/Model_pipline/train.py

@@ -12,6 +12,8 @@ from datetime import date, timedelta
 import tensorflow as tf
 import subprocess
 import time
+import glob
+import random
 
 #################### CMD Arguments ####################
 FLAGS = tf.app.flags.FLAGS
@@ -297,13 +299,8 @@ def main(_):
         FLAGS.dt_dir = (date.today() + timedelta(-1)).strftime('%Y%m%d')
     FLAGS.model_dir = FLAGS.model_dir + FLAGS.dt_dir
     #FLAGS.data_dir  = FLAGS.data_dir + FLAGS.dt_dir
-    import glob
-    import random
-    tr_files = glob.glob("/home/gmuser/test/part*")
-    random.shuffle(tr_files)
-    print("tr_files:", tr_files)
 
-    # tr_files = get_filename("tr")
+    tr_files = get_filename("tr")
     va_files = get_filename("va")
     te_files = ["%s/part-r-00000" % FLAGS.hdfs_dir]
 
@@ -369,7 +366,10 @@ def get_filename(dir_in):
 if __name__ == "__main__":
     b = time.time()
     a = "export CLASSPATH='$(hadoop classpath --glob)'"
+
     os.system(a)
+    print("环境")
+    print(os.popen('hadoop classpath --glob').read())
     path = "hdfs://172.16.32.4:8020/strategy/esmm/"
     tf.logging.set_verbosity(tf.logging.INFO)
     tf.app.run()