修改user feature表

eda/esmm/Feature_pipline/data2ffm.py

@@ -147,7 +147,7 @@ def get_data():
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \
           "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id " \
-          "from esmm_train_data e left join user_feature u on e.device_id = u.device_id " \
+          "from esmm_train_data e left join user_feature_clean u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id left join cid_time on e.cid_id = cid_time.cid_id " \
           "where e.stat_date >= '{}'".format(start)
     df = con_sql(db, sql)
@@ -208,7 +208,7 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \
           "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id,e.cid_id " \
-          "from esmm_pre_data e left join user_feature u on e.device_id = u.device_id " \
+          "from esmm_pre_data e left join user_feature_clean u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id left join cid_time on e.cid_id = cid_time.cid_id"
     df = con_sql(db, sql)
     df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name",

tensnsorflow/ffm.py

@@ -174,8 +174,7 @@ def get_data():
     ucity_id = list(set(df["ucity_id"].values.tolist()))
     manufacturer = list(set(df["manufacturer"].values.tolist()))
     channel = list(set(df["channel"].values.tolist()))
-    print("before transform")
-    print(df.shape)
+
 
     return df,validate_date,ucity_id,ccity_name,manufacturer,channel
 
@@ -184,28 +183,26 @@ def transform(a,validate_date):
     model = multiFFMFormatPandas()
     df = model.fit_transform(a, y="y", n=160000, processes=22)
     df = pd.DataFrame(df)
-    print("after transform")
-    print(df.shape)
-    # df["stat_date"] = df[0].apply(lambda x: x.split(",")[0])
-    # df["device_id"] = df[0].apply(lambda x: x.split(",")[1])
-    # df["y"] = df[0].apply(lambda x: x.split(",")[2])
-    # df["z"] = df[0].apply(lambda x: x.split(",")[3])
-    # df["number"] = np.random.randint(1, 2147483647, df.shape[0])
-    # df["seq"] = list(range(df.shape[0]))
-    # df["seq"] = df["seq"].astype("str")
-    # df["data"] = df[0].apply(lambda x: ",".join(x.split(",")[2:]))
-    # df["data"] = df["seq"].str.cat(df["data"], sep=",")
-    # df = df.drop([0,"seq"], axis=1)
-    # print(df.head(2))
-
-    # train = df[df["stat_date"] != validate_date]
-    # train = train.drop("stat_date",axis=1)
-    # test = df[df["stat_date"] == validate_date]
-    # test = test.drop("stat_date",axis=1)
-    # print("train shape")
-    # print(train.shape)
-    # train.to_csv(path + "tr.csv", sep="\t", index=False)
-    # test.to_csv(path + "va.csv", sep="\t", index=False)
+    df["stat_date"] = df[0].apply(lambda x: x.split(",")[0])
+    df["device_id"] = df[0].apply(lambda x: x.split(",")[1])
+    df["y"] = df[0].apply(lambda x: x.split(",")[2])
+    df["z"] = df[0].apply(lambda x: x.split(",")[3])
+    df["number"] = np.random.randint(1, 2147483647, df.shape[0])
+    df["seq"] = list(range(df.shape[0]))
+    df["seq"] = df["seq"].astype("str")
+    df["data"] = df[0].apply(lambda x: ",".join(x.split(",")[2:]))
+    df["data"] = df["seq"].str.cat(df["data"], sep=",")
+    df = df.drop([0,"seq"], axis=1)
+    print(df.head(2))
+
+    train = df[df["stat_date"] != validate_date]
+    train = train.drop("stat_date",axis=1)
+    test = df[df["stat_date"] == validate_date]
+    test = test.drop("stat_date",axis=1)
+    print("train shape")
+    print(train.shape)
+    train.to_csv(path + "tr.csv", sep="\t", index=False)
+    test.to_csv(path + "va.csv", sep="\t", index=False)
 
     return model
 
@@ -223,23 +220,23 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
 
     print("before filter:")
     print(df.shape)
-    print(df.loc[df["device_id"]=="358035085192742"].shape)
+
     df = df[df["ucity_id"].isin(ucity_id)]
     print("after ucity filter:")
     print(df.shape)
-    print(df.loc[df["device_id"] == "358035085192742"].shape)
+
     df = df[df["ccity_name"].isin(ccity_name)]
     print("after ccity_name filter:")
     print(df.shape)
-    print(df.loc[df["device_id"] == "358035085192742"].shape)
+
     df = df[df["manufacturer"].isin(manufacturer)]
     print("after manufacturer filter:")
     print(df.shape)
-    print(df.loc[df["device_id"] == "358035085192742"].shape)
+
     df = df[df["channel"].isin(channel)]
     print("after channel filter:")
     print(df.shape)
-    print(df.loc[df["device_id"] == "358035085192742"].shape)
+
     df["cid_id"] = df["cid_id"].astype("str")
     df["clevel1_id"] = df["clevel1_id"].astype("str")
     df["top"] = df["top"].astype("str")