删除device id

tensnsorflow/feature_engineering.py

-
-
 import pandas as pd
 import pymysql
 import datetime
@@ -25,7 +23,7 @@ def get_data():
     validate_date = con_sql(db, sql)[0].values.tolist()[0]
     print("validate_date:" + validate_date)
     temp = datetime.datetime.strptime(validate_date, "%Y-%m-%d")
-    start = (temp - datetime.timedelta(days=30)).strftime("%Y-%m-%d")
+    start = (temp - datetime.timedelta(days=60)).strftime("%Y-%m-%d")
     print(start)
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \
@@ -47,12 +45,12 @@ def get_data():
     print("after")
     df = df.drop_duplicates()
     df = df.drop_duplicates(["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                             "channel", "top", "l1","l2", "time", "stat_date","device_id"])
+                             "channel", "top", "l1","l2", "time", "stat_date"])
     print(df.shape)
 
     unique_values = []
     features = ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "time", "stat_date","device_id"]
+                "channel", "top", "time", "stat_date"]
     for i in features:
         df[i] = df[i].astype("str")
         df[i] = df[i].fillna("lost")
@@ -73,10 +71,11 @@ def get_data():
     temp = list(range(1,len(unique_values)+1))
     value_map = dict(zip(unique_values,temp))
 
+    df = df.drop("device_id", axis=1)
     train = df[df["stat_date"] != validate_date+"stat_date"]
     test = df[df["stat_date"] == validate_date+"stat_date"]
     for i in ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "l1", "time", "stat_date","l2","device_id"]:
+                "channel", "top", "l1", "time", "stat_date","l2"]:
         train[i] = train[i].map(value_map)
         test[i] = test[i].map(value_map)
 
@@ -109,8 +108,7 @@ def get_predict(date,value_map):
           "from esmm_pre_data e left join user_feature u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id " \
           "left join cid_level2 cl on e.cid_id = cl.cid " \
-          "left join cid_time_cut cut on e.cid_id = cut.cid " \
-          "where e.device_id = '358035085192742'"
+          "left join cid_time_cut cut on e.cid_id = cut.cid where e.device_id = '358035085192742'"
     df = con_sql(db, sql)
     df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name",
                             6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "l1",11:"l2",
@@ -120,12 +118,10 @@ def get_predict(date,value_map):
 
     print("predict shape")
     print(df.shape)
-
     df["uid"] = df["device_id"]
     df["city"] = df["ucity_id"]
-
     features = ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "time", "stat_date","device_id"]
+                "channel", "top", "time", "stat_date"]
     for i in features:
         df[i] = df[i].astype("str")
         df[i] = df[i].fillna("lost")
@@ -141,9 +137,8 @@ def get_predict(date,value_map):
     nearby_pre = df[df["label"] == 1]
     nearby_pre = nearby_pre.drop("label", axis=1)
 
-
     for i in ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "l1", "time", "stat_date","l2","device_id"]:
+                "channel", "top", "l1", "time", "stat_date","l2"]:
         native_pre[i] = native_pre[i].map(value_map)
         # TODO 没有覆盖到的类别会处理成na，暂时用0填充，后续完善一下
         native_pre[i] = native_pre[i].fillna(0)
@@ -152,6 +147,7 @@ def get_predict(date,value_map):
         # TODO 没有覆盖到的类别会处理成na，暂时用0填充，后续完善一下
         nearby_pre[i] = nearby_pre[i].fillna(0)
 
+
     print("native")
     print(native_pre.shape)
     print(native_pre.head())
@@ -171,4 +167,3 @@ if __name__ == '__main__':
     get_predict(date, value)
 
 
-

tensnsorflow/pipe.sh

@@ -32,15 +32,15 @@ rm ${DATA_PATH}/nearby/nearby_*
 
 
 echo "train..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128,64,32 --dropout=0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=2 --embedding_size=16 --batch_size=1024 --field_size=12 --feature_size=260100 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH} --task_type=train
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128,64,32 --dropout=0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=2 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=1460 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH} --task_type=train
 
 
 echo "infer native..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128,64,32 --dropout=0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=12 --feature_size=260100 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/native --task_type=infer > ${DATA_PATH}/infer.log
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128,64,32 --dropout=0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=1460 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/native --task_type=infer > ${DATA_PATH}/infer.log
 
 
 echo "infer nearby..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128,64,32 --dropout=0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=12 --feature_size=260100 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/nearby --task_type=infer > ${DATA_PATH}/infer.log
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.3 --learning_rate=0.0001 --deep_layers=256,128,64,32 --dropout=0.3,0.3,0.3,0.3 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=1460 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/nearby --task_type=infer > ${DATA_PATH}/infer.log
 
 echo "sort and 2sql"
 ${PYTHON_PATH} ${MODEL_PATH}/sort_to_sql.py

tensnsorflow/test.py

@@ -29,7 +29,7 @@ def gen_tfrecords(in_file):
 
     for i in range(df.shape[0]):
         feats = ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "l1", "time", "stat_date","l2","device_id"]
+                "channel", "top", "l1", "time", "stat_date","l2"]
         id = np.array([])
         for j in feats:
             id = np.append(id,df[j][i])

tensnsorflow/train.py

@@ -53,7 +53,7 @@ def input_fn(filenames, batch_size=32, num_epochs=1, perform_shuffle=False):
         features = {
             "y": tf.FixedLenFeature([], tf.float32),
             "z": tf.FixedLenFeature([], tf.float32),
-            "ids": tf.FixedLenFeature([12], tf.int64)
+            "ids": tf.FixedLenFeature([11], tf.int64)
         }
 
         parsed = tf.parse_single_example(record, features)
@@ -100,18 +100,6 @@ def model_fn(features, labels, mode, params):
 
     feat_ids = features['ids']
 
-    # ucity_id    = features['ucity_id']
-    # clevel1_id = features['clevel1_id']
-    # ccity_name = features['ccity_name']
-    # device_type = features['device_type']
-    # manufacturer = features['manufacturer']
-    # channel = features['channel']
-    # top = features['top']
-    # level2_ids = features['level2_ids']
-    # time = features['time']
-    # stat_date = features['stat_date']
-
-
     if FLAGS.task_type != "infer":
         y = labels['y']
         z = labels['z']
@@ -351,20 +339,6 @@ def main(_):
                     fo.write("%f\t%f\t%f\n" % (prob['pctr'], prob['pcvr'], prob['pctcvr']))
     elif FLAGS.task_type == 'export':
         print("Not Implemented, Do It Yourself!")
-        #feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
-
-        #feature_spec = {
-        #    'feat_ids': tf.FixedLenFeature(dtype=tf.int64, shape=[None, FLAGS.field_size]),
-        #    'feat_vals': tf.FixedLenFeature(dtype=tf.float32, shape=[None, FLAGS.field_size])
-        #}
-        #serving_input_receiver_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
-
-        #feature_spec = {
-        #    'feat_ids': tf.placeholder(dtype=tf.int64, shape=[None, FLAGS.field_size], name='feat_ids'),
-        #    'feat_vals': tf.placeholder(dtype=tf.float32, shape=[None, FLAGS.field_size], name='feat_vals')
-        #}
-        #serving_input_receiver_fn = tf.estimator.export.build_raw_serving_input_receiver_fn(feature_spec)
-        #Estimator.export_savedmodel(FLAGS.servable_model_dir, serving_input_receiver_fn)
 
 if __name__ == "__main__":
     tf.logging.set_verbosity(tf.logging.INFO)