change data dir to /home/gaoyazhe/

eda/esmm/Model_pipline/sort_and_2sql.py

@@ -28,7 +28,7 @@ def main():
         dct["city"].append(i[1])
         dct["cid_id"].append(i[2])
 
-    df1 = pd.read_csv("/srv/apps/ffm-baseline/eda/esmm/data/native/pred.txt",sep='\t',header=None,names=["ctr","cvr"])
+    df1 = pd.read_csv("/home/gaoyazhe/data/native/pred.txt",sep='\t',header=None,names=["ctr","cvr"])
     df2 = pd.DataFrame(dct)
     df2["ctr"],df2["cvr"] = df1["ctr"],df1["cvr"]
     df3 = df2.groupby(by=["uid","city"]).apply(lambda x: x.sort_values(by="cvr",ascending=False)).reset_index(drop=True).groupby(by=["uid","city"]).agg({'cid_id':set_join}).reset_index(drop=False)
@@ -46,7 +46,7 @@ def main():
         dct["city"].append(i[1])
         dct["cid_id"].append(i[2])
 
-    df1 = pd.read_csv("/srv/apps/ffm-baseline/eda/esmm/data/nearby/pred.txt",sep='\t',header=None,names=["ctr","cvr"])
+    df1 = pd.read_csv("/home/gaoyazhe/data/nearby/pred.txt",sep='\t',header=None,names=["ctr","cvr"])
     df2 = pd.DataFrame(dct)
     df2["ctr"],df2["cvr"] = df1["ctr"],df1["cvr"]
     df4 = df2.groupby(by=["uid","city"]).apply(lambda x: x.sort_values(by="cvr",ascending=False)).reset_index(drop=True).groupby(by=["uid","city"]).agg({'cid_id':set_join}).reset_index(drop=False)

eda/esmm/Model_pipline/submit.sh

@@ -6,50 +6,50 @@ currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
 echo $currentTimeStamp
 
 echo "rm leave tfrecord"
-rm /srv/apps/ffm-baseline/eda/esmm/data/tr/*
-rm /srv/apps/ffm-baseline/eda/esmm/data/va/*
-rm /srv/apps/ffm-baseline/eda/esmm/data/native/*
-rm /srv/apps/ffm-baseline/eda/esmm/data/nearby/*
+rm /home/gaoyazhe/data/tr/*
+rm /home/gaoyazhe/data/va/*
+rm /home/gaoyazhe/data/native/*
+rm /home/gaoyazhe/data/nearby/*
 
 echo "mysql to csv"
-mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_train" > /srv/apps/ffm-baseline/eda/esmm/data/tr.csv
-mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_cv" > /srv/apps/ffm-baseline/eda/esmm/data/va.csv
-mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_infer_native" > /srv/apps/ffm-baseline/eda/esmm/data/native.csv
-mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_infer_nearby" > /srv/apps/ffm-baseline/eda/esmm/data/nearby.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_train" > /home/gaoyazhe/data/tr.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_cv" > /home/gaoyazhe/data/va.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_infer_native" > /home/gaoyazhe/data/native.csv
+mysql -u root -p3SYz54LS9#^9sBvC -h 10.66.157.22 -P 4000 -D jerry_test -e "select number,data from esmm_data2ffm_infer_nearby" > /home/gaoyazhe/data/nearby.csv
 
 echo "split data"
-split -l $((`wc -l < /srv/apps/ffm-baseline/eda/esmm/data/tr.csv`/15)) /srv/apps/ffm-baseline/eda/esmm/data/tr.csv -d -a 4 /srv/apps/ffm-baseline/eda/esmm/data/tr/tr_ --additional-suffix=.csv
-split -l $((`wc -l < /srv/apps/ffm-baseline/eda/esmm/data/va.csv`/5)) /srv/apps/ffm-baseline/eda/esmm/data/va.csv -d -a 4 /srv/apps/ffm-baseline/eda/esmm/data/va/va_ --additional-suffix=.csv
-split -l $((`wc -l < /srv/apps/ffm-baseline/eda/esmm/data/native.csv`/5)) /srv/apps/ffm-baseline/eda/esmm/data/native.csv -d -a 4 /srv/apps/ffm-baseline/eda/esmm/data/native/native_ --additional-suffix=.csv
-split -l $((`wc -l < /srv/apps/ffm-baseline/eda/esmm/data/nearby.csv`/5)) /srv/apps/ffm-baseline/eda/esmm/data/nearby.csv -d -a 4 /srv/apps/ffm-baseline/eda/esmm/data/nearby/nearby_ --additional-suffix=.csv
+split -l $((`wc -l < /home/gaoyazhe/data/tr.csv`/15)) /home/gaoyazhe/data/tr.csv -d -a 4 /home/gaoyazhe/data/tr/tr_ --additional-suffix=.csv
+split -l $((`wc -l < /home/gaoyazhe/data/va.csv`/5)) /home/gaoyazhe/data/va.csv -d -a 4 /home/gaoyazhe/data/va/va_ --additional-suffix=.csv
+split -l $((`wc -l < /home/gaoyazhe/data/native.csv`/5)) /home/gaoyazhe/data/native.csv -d -a 4 /home/gaoyazhe/data/native/native_ --additional-suffix=.csv
+split -l $((`wc -l < /home/gaoyazhe/data/nearby.csv`/5)) /home/gaoyazhe/data/nearby.csv -d -a 4 /home/gaoyazhe/data/nearby/nearby_ --additional-suffix=.csv
 
 echo "csv to tfrecord"
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/srv/apps/ffm-baseline/eda/esmm/data/tr/ --output_dir=/srv/apps/ffm-baseline/eda/esmm/data/tr/
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/srv/apps/ffm-baseline/eda/esmm/data/va/ --output_dir=/srv/apps/ffm-baseline/eda/esmm/data/va/
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/srv/apps/ffm-baseline/eda/esmm/data/native/ --output_dir=/srv/apps/ffm-baseline/eda/esmm/data/native/
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/srv/apps/ffm-baseline/eda/esmm/data/nearby/ --output_dir=/srv/apps/ffm-baseline/eda/esmm/data/nearby/
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/home/gaoyazhe/data/tr/ --output_dir=/home/gaoyazhe/data/tr/
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/home/gaoyazhe/data/va/ --output_dir=/home/gaoyazhe/data/va/
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/home/gaoyazhe/data/native/ --output_dir=/home/gaoyazhe/data/native/
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Feature_pipline/get_tfrecord.py --input_dir=/home/gaoyazhe/data/nearby/ --output_dir=/home/gaoyazhe/data/nearby/
 
-cat /srv/apps/ffm-baseline/eda/esmm/data/tr/*.tfrecord > /srv/apps/ffm-baseline/eda/esmm/data/tr/tr.tfrecord
-cat /srv/apps/ffm-baseline/eda/esmm/data/va/*.tfrecord > /srv/apps/ffm-baseline/eda/esmm/data/va/va.tfrecord
-cat /srv/apps/ffm-baseline/eda/esmm/data/native/*.tfrecord > /srv/apps/ffm-baseline/eda/esmm/data/native/native.tfrecord
-cat /srv/apps/ffm-baseline/eda/esmm/data/nearby/*.tfrecord > /srv/apps/ffm-baseline/eda/esmm/data/nearby/nearby.tfrecord
+cat /home/gaoyazhe/data/tr/*.tfrecord > /home/gaoyazhe/data/tr/tr.tfrecord
+cat /home/gaoyazhe/data/va/*.tfrecord > /home/gaoyazhe/data/va/va.tfrecord
+cat /home/gaoyazhe/data/native/*.tfrecord > /home/gaoyazhe/data/native/native.tfrecord
+cat /home/gaoyazhe/data/nearby/*.tfrecord > /home/gaoyazhe/data/nearby/nearby.tfrecord
 
-rm /srv/apps/ffm-baseline/eda/esmm/data/tr/tr_*
-rm /srv/apps/ffm-baseline/eda/esmm/data/va/va_*
-rm /srv/apps/ffm-baseline/eda/esmm/data/native/native_*
-rm /srv/apps/ffm-baseline/eda/esmm/data/nearby/nearby_*
+rm /home/gaoyazhe/data/tr/tr_*
+rm /home/gaoyazhe/data/va/va_*
+rm /home/gaoyazhe/data/native/native_*
+rm /home/gaoyazhe/data/nearby/nearby_*
 
 echo "train..."
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=/srv/apps/ffm-baseline/eda/esmm/Model_pipline/model_ckpt/DeepCvrMTL/ --data_dir="/srv/apps/ffm-baseline/eda/esmm/data" --task_type="train" > /srv/apps/ffm-baseline/eda/esmm/Model_pipline/train.log
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=/srv/apps/ffm-baseline/eda/esmm/Model_pipline/model_ckpt/DeepCvrMTL/ --data_dir="/home/gaoyazhe/data" --task_type="train" > /srv/apps/ffm-baseline/eda/esmm/Model_pipline/train.log
 
 /home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/send_mail.py
 
 echo "infer native..."
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=/srv/apps/ffm-baseline/eda/esmm/Model_pipline/model_ckpt/DeepCvrMTL/ --data_dir="/srv/apps/ffm-baseline/eda/esmm/data/native" --task_type="infer"
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=/srv/apps/ffm-baseline/eda/esmm/Model_pipline/model_ckpt/DeepCvrMTL/ --data_dir="/home/gaoyazhe/data/native" --task_type="infer"
 
 
 echo "infer nearby..."
-/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=/srv/apps/ffm-baseline/eda/esmm/Model_pipline/model_ckpt/DeepCvrMTL/ --data_dir="/srv/apps/ffm-baseline/eda/esmm/data/nearby" --task_type="infer"
+/home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/DeepCvrMTL.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=354332 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=/srv/apps/ffm-baseline/eda/esmm/Model_pipline/model_ckpt/DeepCvrMTL/ --data_dir="/home/gaoyazhe/data/nearby" --task_type="infer"
 
 echo "sort and 2sql"
 /home/gaoyazhe/miniconda3/bin/python /srv/apps/ffm-baseline/eda/esmm/Model_pipline/sort_and_2sql.py