修改data2ffm

eda/esmm/Feature_pipline/data2ffm.py

@@ -38,6 +38,10 @@ class multiFFMFormatPandas:
         self.y = None
 
     def fit(self, df, y=None):
+        b = df.dtypes
+        c = list(b.values)
+        d =tuple(df.dtypes.to_dict())
+        f = dict(zip(d,c))
         self.y = y
         df_ffm = df[df.columns.difference([self.y])]
         if self.field_index_ is None:
@@ -49,17 +53,24 @@ class multiFFMFormatPandas:
         if self.feature_index_ is None:
             self.feature_index_ = dict()
 
-        for col in df.columns:
-            self.feature_index_[col] = 1
-            last_idx = 1
-            vals = df[col].unique()
-            for val in vals:
-                if pd.isnull(val):
-                    continue
-                name = '{}_{}'.format(col, val)
-                if name not in self.feature_index_:
-                    self.feature_index_[name] = last_idx
-                    last_idx += 1
+        last_idx = 1
+        l = list(df.columns)
+        l.remove(y)
+        for col in l:
+            if f[col]=="O":
+                vals = df[col].unique()
+                for val in vals:
+                    if pd.isnull(val):
+                        continue
+                    name = '{}_{}'.format(col, val)
+                    if name not in self.feature_index_:
+                        self.feature_index_[name] = last_idx
+                        last_idx += 1
+            else:
+                self.feature_index_[col] = last_idx
+                last_idx += 1
+        print("last_idx")
+        print(last_idx-1)
         return self
 
     def fit_transform(self, df, y=None,n=50000,processes=4):
@@ -99,6 +110,7 @@ class multiFFMFormatPandas:
         result_map = {}
         for i in data_list:
             result_map.update(i.get())
+
         pool.close()
         pool.join()
 
@@ -175,7 +187,8 @@ def get_data():
     ucity_id = list(set(df["ucity_id"].values.tolist()))
     manufacturer = list(set(df["manufacturer"].values.tolist()))
     channel = list(set(df["channel"].values.tolist()))
-    return df,validate_date,ucity_id,ccity_name,manufacturer,channel
+    level2_ids = list(set(df["level2_ids"].values.tolist()))
+    return df,validate_date,ucity_id,ccity_name,manufacturer,channel,level2_ids
 
 
 def transform(a,validate_date):
@@ -207,14 +220,13 @@ def transform(a,validate_date):
     return model
 
 
-def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
+def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \
           "u.device_type,u.manufacturer,u.channel,c.top,df.level2_ids,e.device_id,e.cid_id " \
           "from esmm_pre_data e left join user_feature u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id " \
-          "left join diary_feat df on e.cid_id = df.diary_id " \
-          "where e.device_id = '358035085192742'"
+          "left join diary_feat df on e.cid_id = df.diary_id"
     df = con_sql(db, sql)
     df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name",
                             6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "level2_ids",
@@ -239,6 +251,12 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     print("after channel filter:")
     print(df.shape)
 
+    df = df[df["level2_ids"].isin(level2_ids)]
+    print("after level2_ids filter:")
+    print(df.shape)
+
+    df[(df["ucity_id"] == "beijing") & (df["top"] == 66)].to_csv(path + "top66.csv", sep="\t", index=False)
+
     df["cid_id"] = df["cid_id"].astype("str")
     df["clevel1_id"] = df["clevel1_id"].astype("str")
     df["top"] = df["top"].astype("str")
@@ -248,7 +266,7 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     df["y"] = df["label"].str.cat(
         [df["device_id"].values.tolist(), df["ucity_id"].values.tolist(), df["cid_id"].values.tolist(),
          df["y"].values.tolist(), df["z"].values.tolist()], sep=",")
-    df = df.drop(["z","label","device_id","cid_id"], axis=1).fillna(0.0)
+    df = df.drop(["z","label","device_id","cid_id"], axis=1).fillna("na")
     print("before transform")
     print(df.shape)
     temp_series = model.transform(df,n=160000, processes=22)
@@ -267,12 +285,12 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     df = df.drop([0, "seq"], axis=1)
     print(df.head())
 
-    print(df.loc[df["device_id"] == "358035085192742"].shape)
+    # print(df.loc[df["device_id"] == "358035085192742"].shape)
     native_pre = df[df["label"] == "0"]
     native_pre = native_pre.drop("label", axis=1)
     print("native")
     print(native_pre.shape)
-    print(native_pre.loc[native_pre["device_id"] == "358035085192742"].shape)
+    # print(native_pre.loc[native_pre["device_id"] == "358035085192742"].shape)
     native_pre.to_csv(path+"native.csv",sep="\t",index=False)
     print("native_pre shape")
     print(native_pre.shape)
@@ -281,7 +299,7 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     nearby_pre = nearby_pre.drop("label", axis=1)
     print("nearby")
     print(nearby_pre.shape)
-    print(nearby_pre.loc[nearby_pre["device_id"] == "358035085192742"].shape)
+    # print(nearby_pre.loc[nearby_pre["device_id"] == "358035085192742"].shape)
     nearby_pre.to_csv(path + "nearby.csv", sep="\t", index=False)
     print("nearby_pre shape")
     print(nearby_pre.shape)
@@ -291,9 +309,9 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
 if __name__ == "__main__":
     path = "/home/gmuser/esmm_data/"
     a = time.time()
-    temp, validate_date, ucity_id, ccity_name, manufacturer, channel = get_data()
+    temp, validate_date, ucity_id,ccity_name,manufacturer,channel,level2_ids = get_data()
     model = transform(temp, validate_date)
-    get_predict_set(ucity_id, model, ccity_name, manufacturer, channel)
+    get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids)
     b = time.time()
     print("cost(分钟)")
-    print((b - a) / 60)
+    print((b-a)/60)

tensnsorflow/ffm.py

@@ -255,6 +255,8 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids):
     print("after level2_ids filter:")
     print(df.shape)
 
+    df[(df["ucity_id"] == "beijing") & (df["top"] == 66)].to_csv(path + "top66.csv", sep="\t", index=False)
+
     df["cid_id"] = df["cid_id"].astype("str")
     df["clevel1_id"] = df["clevel1_id"].astype("str")
     df["top"] = df["top"].astype("str")
@@ -283,12 +285,12 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids):
     df = df.drop([0, "seq"], axis=1)
     print(df.head())
 
-    print(df.loc[df["device_id"] == "358035085192742"].shape)
+    # print(df.loc[df["device_id"] == "358035085192742"].shape)
     native_pre = df[df["label"] == "0"]
     native_pre = native_pre.drop("label", axis=1)
     print("native")
     print(native_pre.shape)
-    print(native_pre.loc[native_pre["device_id"] == "358035085192742"].shape)
+    # print(native_pre.loc[native_pre["device_id"] == "358035085192742"].shape)
     native_pre.to_csv(path+"native.csv",sep="\t",index=False)
     print("native_pre shape")
     print(native_pre.shape)
@@ -297,7 +299,7 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel,level2_ids):
     nearby_pre = nearby_pre.drop("label", axis=1)
     print("nearby")
     print(nearby_pre.shape)
-    print(nearby_pre.loc[nearby_pre["device_id"] == "358035085192742"].shape)
+    # print(nearby_pre.loc[nearby_pre["device_id"] == "358035085192742"].shape)
     nearby_pre.to_csv(path + "nearby.csv", sep="\t", index=False)
     print("nearby_pre shape")
     print(nearby_pre.shape)