add sh filw

tensnsorflow/pipe.sh

+#! /bin/bash
+
+PYTHON_PATH=/home/gaoyazhe/miniconda3/bin/python
+MODEL_PATH=/srv/apps/ffm-baseline/tensnsorflow
+DATA_PATH=/home/gmuser/esmm_data
+OLD_PATH=/srv/apps/ffm-baseline/eda/esmm
+
+echo "rm leave tfrecord"
+rm ${DATA_PATH}/tr/*
+rm ${DATA_PATH}/va/*
+rm ${DATA_PATH}/native/*
+rm ${DATA_PATH}/nearby/*
+rm -r ${DATA_PATH}/model_ckpt/DeepCvrMTL/201*
+
+echo "data"
+${PYTHON_PATH} ${MODEL_PATH}/feature_engineering.py > ${DATA_PATH}/infer.log
+
+echo "csv to tfrecord"
+${PYTHON_PATH} ${MODEL_PATH}/test.py --input_dir=${DATA_PATH}/tr/ --output_dir=${DATA_PATH}/tr/
+${PYTHON_PATH} ${MODEL_PATH}/test.py --input_dir=${DATA_PATH}/va/ --output_dir=${DATA_PATH}/va/
+${PYTHON_PATH} ${MODEL_PATH}/test.py --input_dir=${DATA_PATH}/native/ --output_dir=${DATA_PATH}/native/
+${PYTHON_PATH} ${MODEL_PATH}/test.py --input_dir=${DATA_PATH}/nearby/ --output_dir=${DATA_PATH}/nearby/
+
+cat ${DATA_PATH}/tr/*.tfrecord > ${DATA_PATH}/tr/tr.tfrecord
+cat ${DATA_PATH}/va/*.tfrecord > ${DATA_PATH}/va/va.tfrecord
+cat ${DATA_PATH}/native/*.tfrecord > ${DATA_PATH}/native/native.tfrecord
+cat ${DATA_PATH}/nearby/*.tfrecord > ${DATA_PATH}/nearby/nearby.tfrecord
+
+rm ${DATA_PATH}/tr/tr_*
+rm ${DATA_PATH}/va/va_*
+rm ${DATA_PATH}/native/native_*
+rm ${DATA_PATH}/nearby/nearby_*
+
+
+echo "train..."
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=10 --feature_size=2000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH} --task_type=train
+
+
+echo "infer native..."
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=10 --feature_size=2000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/native --task_type=infer > ${DATA_PATH}/infer.log
+
+
+echo "infer nearby..."
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128 --dropout=0.8,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=10 --feature_size=2000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/nearby --task_type=infer > ${DATA_PATH}/infer.log
+
+echo "sort and 2sql"
+${PYTHON_PATH} ${OLD_PATH}/Model_pipline/sort_and_2sql.py
+
+echo "infer and sort and 2sql time"
+current=$(date "+%Y-%m-%d %H:%M:%S")
+timeStamp=$(date -d "$current" +%s)
+currentTimeStamp=$((timeStamp*1000+`date "+%N"`/1000000))
+echo $current
+