增加日记二级标签特征

tensnsorflow/es/feature.py

@@ -44,7 +44,6 @@ def get_data():
           "u.device_type,u.manufacturer,u.channel,c.top,e.device_id,cut.time,dl.app_list " \
           "from {} e left join user_feature u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id " \
-          "left join cid_level2 cl on e.cid_id = cl.cid " \
           "left join cid_time_cut cut on e.cid_id = cut.cid " \
           "left join device_app_list dl on e.device_id = dl.device_id " \
           "left join diary_feat feat on e.cid_id = feat.diary_id " \
@@ -52,8 +51,8 @@ def get_data():
     df = con_sql(db, sql)
     # print(df.shape)
     df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id", 4: "clevel2_id", 5: "ccity_name",
-                            6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "l1",11: "l2",
-                            12: "device_id", 13: "time",14:"app_list"})
+                            6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "device_id",
+                            11: "time",12:"app_list"})
     print("esmm data ok")
     # print(df.head(2)
     print("before")
@@ -61,7 +60,7 @@ def get_data():
     print("after")
     df = df.drop_duplicates()
     df = df.drop_duplicates(["ucity_id", "clevel2_id", "ccity_name", "device_type", "manufacturer",
-                             "channel", "top", "l1","l2", "time", "stat_date","app_list"])
+                             "channel", "top", "time", "stat_date","app_list"])
 
     app_list_number,app_list_map = multi_hot(df,"app_list",1)
     level2_number,level2_map = multi_hot(df,"clevel2_id",1+app_list_number)
@@ -84,16 +83,6 @@ def get_data():
         # 下面这行代码是为了区分不同的列中有相同的值
         df[i] = df[i] + i
         unique_values.extend(list(df[i].unique()))
-    for i in ["l1","l2"]:
-        df[i] = df[i].astype("str")
-        df[i] = df[i].fillna("lost")
-        # l1和l2中的值与top类别是一个类别
-        df[i] = df[i]+"top"
-        unique_values.extend(list(df[i].unique()))
-
-    print("features:")
-    print(len(unique_values))
-    print(df.head(2))
 
     temp = list(range(1+app_list_number+level2_number, 1 + app_list_number+level2_number + len(unique_values)))
     value_map = dict(zip(unique_values,temp))
@@ -102,7 +91,7 @@ def get_data():
     train = df[df["stat_date"] != validate_date+"stat_date"]
     test = df[df["stat_date"] == validate_date+"stat_date"]
     for i in ["ucity_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "l1", "time", "stat_date","l2"]:
+                "channel", "top", "time", "stat_date"]:
         train[i] = train[i].map(value_map)
         test[i] = test[i].map(value_map)
 
@@ -114,7 +103,7 @@ def get_data():
     write_csv(train, "tr",100000)
     write_csv(test, "va",80000)
 
-    return validate_date,value_map,app_list_map
+    return validate_date,value_map,app_list_map,level2_map
 
 
 def app_list_func(x,l):
@@ -136,47 +125,44 @@ def write_csv(df,name,n):
         temp.to_csv(path + name+ "/{}_{}.csv".format(name,i), index=False)
 
 
-def get_predict(date,value_map,app_list_map):
+def get_predict(date,value_map,app_list_map,level2_map):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
-    sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \
-          "u.device_type,u.manufacturer,u.channel,c.top,cl.l1,cl.l2,e.device_id,e.cid_id,cut.time,dl.app_list " \
+    sql = "select e.y,e.z,e.label,e.ucity_id,feat.level2_ids,e.ccity_name," \
+          "u.device_type,u.manufacturer,u.channel,c.top,e.device_id,e.cid_id,cut.time,dl.app_list " \
           "from esmm_pre_data e left join user_feature u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id " \
-          "left join cid_level2 cl on e.cid_id = cl.cid " \
           "left join cid_time_cut cut on e.cid_id = cut.cid " \
-          "left join device_app_list dl on e.device_id = dl.device_id limit 6"
+          "left join device_app_list dl on e.device_id = dl.device_id " \
+          "left join diary_feat feat on e.cid_id = feat.diary_id limit 6"
     df = con_sql(db, sql)
-    df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name",
-                            6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "l1",11:"l2",
-                            12: "device_id", 13: "cid_id", 14: "time",15:"app_list"})
+    df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel2_id", 5: "ccity_name",
+                            6: "device_type", 7: "manufacturer", 8: "channel", 9: "top",
+                            10: "device_id", 11: "cid_id", 12: "time",13:"app_list"})
 
     df["stat_date"] = date
     df["app_list"] = df["app_list"].fillna("lost_na")
     df["app_list"] = df["app_list"].apply(app_list_func,args=(app_list_map,))
+    df["clevel2_id"] = df["clevel2_id"].fillna("lost_na")
+    df["clevel2_id"] = df["clevel2_id"].apply(app_list_func, args=(level2_map,))
 
     print("predict shape")
     print(df.shape)
     df["uid"] = df["device_id"]
     df["city"] = df["ucity_id"]
-    features = ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
+    features = ["ucity_id", "ccity_name", "device_type", "manufacturer",
                 "channel", "top", "time", "stat_date"]
     for i in features:
         df[i] = df[i].astype("str")
         df[i] = df[i].fillna("lost")
         df[i] = df[i] + i
-    for i in ["l1","l2"]:
-        df[i] = df[i].astype("str")
-        df[i] = df[i].fillna("lost")
-        # l1和l2中的值与top类别是一个类别
-        df[i] = df[i]+"top"
 
     native_pre = df[df["label"] == 0]
     native_pre = native_pre.drop("label", axis=1)
     nearby_pre = df[df["label"] == 1]
     nearby_pre = nearby_pre.drop("label", axis=1)
 
-    for i in ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "l1", "time", "stat_date","l2"]:
+    for i in ["ucity_id", "ccity_name", "device_type", "manufacturer",
+                "channel", "top", "time", "stat_date"]:
         native_pre[i] = native_pre[i].map(value_map)
         # TODO 没有覆盖到的类别会处理成na，暂时用0填充，后续完善一下
         native_pre[i] = native_pre[i].fillna(0)
@@ -202,6 +188,6 @@ def get_predict(date,value_map,app_list_map):
 if __name__ == '__main__':
     train_data_set = "esmm_train_data"
     path = "/data/esmm/"
-    date,value,app_list = get_data()
-    get_predict(date, value,app_list)
+    date,value,app_list,level2 = get_data()
+    get_predict(date, value,app_list,level2)
 

tensnsorflow/es/pipeline.sh

@@ -32,15 +32,15 @@ rm ${DATA_PATH}/nearby/nearby_*
 
 
 echo "train..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.9 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.5,0.5,0.5,0.5,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=200000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH} --task_type=train
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.9 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.5,0.5,0.5,0.5,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=8 --feature_size=300000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH} --task_type=train
 
 
 echo "infer native..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.9 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.5,0.5,0.5,0.5,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=200000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/native --task_type=infer > ${DATA_PATH}/infer.log
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.9 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.5,0.5,0.5,0.5,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=8 --feature_size=300000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/native --task_type=infer > ${DATA_PATH}/infer.log
 
 
 echo "infer nearby..."
-${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.9 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.5,0.5,0.5,0.5,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=11 --feature_size=200000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/nearby --task_type=infer > ${DATA_PATH}/infer.log
+${PYTHON_PATH} ${MODEL_PATH}/train.py --ctr_task_wgt=0.9 --learning_rate=0.0001 --deep_layers=512,256,128,64,32 --dropout=0.5,0.5,0.5,0.5,0.5 --optimizer=Adam --num_epochs=1 --embedding_size=16 --batch_size=1024 --field_size=8 --feature_size=300000 --l2_reg=0.005 --log_steps=100 --num_threads=36 --model_dir=${DATA_PATH}/model_ckpt/DeepCvrMTL/ --data_dir=${DATA_PATH}/nearby --task_type=infer > ${DATA_PATH}/infer.log
 
 echo "sort and 2sql"
 ${PYTHON_PATH} ${MODEL_PATH}/to_database.py

tensnsorflow/es/to_tfrecord.py

@@ -28,17 +28,19 @@ def gen_tfrecords(in_file):
     df = pd.read_csv(in_file)
 
     for i in range(df.shape[0]):
-        feats = ["ucity_id", "clevel1_id", "ccity_name", "device_type", "manufacturer",
-                "channel", "top", "l1", "time", "stat_date","l2"]
+        feats = ["ucity_id", "ccity_name", "device_type", "manufacturer",
+                "channel", "top", "time", "stat_date"]
         id = np.array([])
         for j in feats:
             id = np.append(id,df[j][i])
         app_list = np.array(df["app_list"][i].split(","))
+        level2_list = np.array(df["clevel2_id"][i].split(","))
         features = tf.train.Features(feature={
             "y": tf.train.Feature(float_list=tf.train.FloatList(value=[df["y"][i]])),
             "z": tf.train.Feature(float_list=tf.train.FloatList(value=[df["z"][i]])),
             "ids": tf.train.Feature(int64_list=tf.train.Int64List(value=id.astype(np.int))),
-            "app_list":tf.train.Feature(int64_list=tf.train.Int64List(value=app_list.astype(np.int)))
+            "app_list":tf.train.Feature(int64_list=tf.train.Int64List(value=app_list.astype(np.int))),
+            "level2_list": tf.train.Feature(int64_list=tf.train.Int64List(value=level2_list.astype(np.int)))
         })
 
         example = tf.train.Example(features = features)

tensnsorflow/es/train.py

@@ -54,7 +54,8 @@ def input_fn(filenames, batch_size=32, num_epochs=1, perform_shuffle=False):
             "y": tf.FixedLenFeature([], tf.float32),
             "z": tf.FixedLenFeature([], tf.float32),
             "ids": tf.FixedLenFeature([11], tf.int64),
-            "app_list": tf.VarLenFeature(tf.int64)
+            "app_list": tf.VarLenFeature(tf.int64),
+            "level2_list": tf.VarLenFeature(tf.int64)
         }
 
         parsed = tf.parse_single_example(record, features)
@@ -101,6 +102,7 @@ def model_fn(features, labels, mode, params):
 
     feat_ids = features['ids']
     app_list = features['app_list']
+    level2_list = features['level2_list']
 
     if FLAGS.task_type != "infer":
         y = labels['y']
@@ -110,9 +112,11 @@ def model_fn(features, labels, mode, params):
     with tf.variable_scope("Shared-Embedding-layer"):
         embedding_id = tf.nn.embedding_lookup(Feat_Emb,feat_ids)
         app_id = tf.nn.embedding_lookup_sparse(Feat_Emb, sp_ids=app_list, sp_weights=None, combiner="sum")
+        level2 = tf.nn.embedding_lookup_sparse(Feat_Emb, sp_ids=level2_list, sp_weights=None, combiner="sum")
+
 
         # x_concat = tf.reshape(embedding_id,shape=[-1, common_dims])  # None * (F * K)
-        x_concat = tf.concat([tf.reshape(embedding_id,shape=[-1,common_dims]),app_id], axis=1)
+        x_concat = tf.concat([tf.reshape(embedding_id,shape=[-1,common_dims]),app_id,level2], axis=1)
 
     with tf.name_scope("CVR_Task"):
         if mode == tf.estimator.ModeKeys.TRAIN: