Merge branch 'master' of http://git.wanmeizhensuo.com/ML/ffm-baseline

eda/esmm/Model_pipline/feature_engineering.py

@@ -76,6 +76,17 @@ def con_sql(db,sql):
     return df
 
 
+def get_pre_number():
+    db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    sql = "select count(*) from esmm_pre_data"
+    cursor = db.cursor()
+    cursor.execute(sql)
+    result = cursor.fetchone()[0]
+    print("预测集数量：")
+    print(result)
+    db.close()
+
+
 def feature_engineer():
     apps_number, app_list_map, level2_number, leve2_map, level3_number, leve3_map = get_map()
     unique_values = []
@@ -221,9 +232,11 @@ def feature_engineer():
     print("train tfrecord done")
     print((h - f) / 60)
 
-    print("样本总量：")
+    print("训练集样本总量：")
     print(rdd.count())
 
+    get_pre_number()
+
     test = rdd.filter(lambda x: x[0] == validate_date).map(
         lambda x: (x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9],
                    x[10], x[11], x[12], x[13]))

eda/esmm/Model_pipline/submit.sh

@@ -16,17 +16,17 @@ rm -r ${LOCAL_PATH}/model_ckpt/DeepCvrMTL/20*
 b=`date +%Y%m%d`
 
 echo "train..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --hdfs_dir=${HDFS_PATH}/native --task_type=train > "/home/gmuser/esmm/log/train_$b.log"
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH} --hdfs_dir=${HDFS_PATH}/native --task_type=train
 
 echo "infer native..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH}/native --hdfs_dir=${HDFS_PATH}/native --task_type=infer > "/home/gmuser/esmm/log/native_$b.log"
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH}/native --hdfs_dir=${HDFS_PATH}/native --task_type=infer
 
 
 echo "infer nearby..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH}/nearby --hdfs_dir=${HDFS_PATH}/nearby --task_type=infer > "/home/gmuser/esmm/log/nearby_$b.log"
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.5 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.3,0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=10000 --field_size=15 --feature_size=600000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${LOCAL_PATH}/model_ckpt/DeepCvrMTL/ --local_dir=${LOCAL_PATH}/nearby --hdfs_dir=${HDFS_PATH}/nearby --task_type=infer
 
 echo "sort and 2sql"
-${PYTHON_PATH} ${MODEL_PATH}/to_database.py > "/home/gmuser/esmm/log/insert_$b.log"
+${PYTHON_PATH} ${MODEL_PATH}/to_database.py
 
 echo "delete files"
 rm /home/gmuser/esmm/*.csv

tensnsorflow/multi.py

@@ -157,8 +157,20 @@ def get_hdfs(dir_in):
             a.append(tmp)
     return a
 
+
+def get_pre_number():
+    db = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+    sql = "select count(*) from esmm_pre_data"
+    cursor = db.cursor()
+    cursor.execute(sql)
+    result = cursor.fetchone()[0]
+    print("预测集数量：")
+    print(result)
+    db.close()
+
+
 if __name__ == '__main__':
-    print("hello")
+   # get_pre()
     # sparkConf = SparkConf().set("spark.hive.mapred.supports.subdirectories", "true") \
     #     .set("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive", "true") \
     #     .set("spark.tispark.plan.allow_index_double_read", "false") \