add ,manufacturer,channel filter

eda/esmm/Feature_pipline/data2ffm.py

@@ -171,7 +171,9 @@ def get_data():
     print("features:{}".format(features))
     ccity_name = list(set(df["ccity_name"].values.tolist()))
     ucity_id = list(set(df["ucity_id"].values.tolist()))
-    return df,validate_date,ucity_id,ccity_name
+    manufacturer = list(set(df["manufacturer"].values.tolist()))
+    channel = list(set(df["channel"].values.tolist()))
+    return df,validate_date,ucity_id,ccity_name,manufacturer,channel
 
 
 def transform(a,validate_date):
@@ -202,7 +204,7 @@ def transform(a,validate_date):
     return model
 
 
-def get_predict_set(ucity_id,model,ccity_name):
+def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \
           "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id,e.cid_id " \
@@ -218,6 +220,8 @@ def get_predict_set(ucity_id,model,ccity_name):
     print("after ucity filter:")
     print(df.shape)
     df = df[df["ccity_name"].isin(ccity_name)]
+    df = df[df["manufacturer"].isin(manufacturer)]
+    df = df[df["channel"].isin(channel)]
     print("after ccity_name filter:")
     print(df.shape)
     df["cid_id"] = df["cid_id"].astype("str")
@@ -262,9 +266,9 @@ def get_predict_set(ucity_id,model,ccity_name):
 if __name__ == "__main__":
     path = "/home/gaoyazhe/data/"
     a = time.time()
-    df, validate_date, ucity_id,ccity_name = get_data()
+    df, validate_date, ucity_id,ccity_name,manufacturer,channel = get_data()
     model = transform(df, validate_date)
-    get_predict_set(ucity_id,model,ccity_name)
+    get_predict_set(ucity_id,model,ccity_name,manufacturer,channel)
     b = time.time()
     print("cost(分钟)")
     print((b-a)/60)

tensnsorflow/ffm.py

@@ -171,7 +171,10 @@ def get_data():
     print("features:{}".format(features))
     ccity_name = list(set(df["ccity_name"].values.tolist()))
     ucity_id = list(set(df["ucity_id"].values.tolist()))
-    return df,validate_date,ucity_id,ccity_name
+    manufacturer = list(set(df["manufacturer"].values.tolist()))
+    channel = list(set(df["channel"].values.tolist()))
+
+    return df,validate_date,ucity_id,ccity_name,manufacturer,channel
 
 
 def transform(a,validate_date):
@@ -202,7 +205,7 @@ def transform(a,validate_date):
     return model
 
 
-def get_predict_set(ucity_id,model,ccity_name):
+def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \
           "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id,e.cid_id " \
@@ -212,6 +215,7 @@ def get_predict_set(ucity_id,model,ccity_name):
     df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name",
                             6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "time",
                             11:"device_id",12:"cid_id"})
+
     print("before filter:")
     print(df.shape)
     df = df[df["ucity_id"].isin(ucity_id)]
@@ -219,6 +223,8 @@ def get_predict_set(ucity_id,model,ccity_name):
     print(df.shape)
     df = df[df["ccity_name"].isin(ccity_name)]
     print("after ccity_name filter:")
+    df = df[df["manufacturer"].isin(manufacturer)]
+    df = df[df["channel"].isin(channel)]
     print(df.shape)
     df["cid_id"] = df["cid_id"].astype("str")
     df["clevel1_id"] = df["clevel1_id"].astype("str")
@@ -262,9 +268,9 @@ def get_predict_set(ucity_id,model,ccity_name):
 if __name__ == "__main__":
     path = "/home/gmuser/ffm/"
     a = time.time()
-    df, validate_date, ucity_id,ccity_name = get_data()
+    df, validate_date, ucity_id,ccity_name,manufacturer,channel = get_data()
     model = transform(df, validate_date)
-    get_predict_set(ucity_id,model,ccity_name)
+    get_predict_set(ucity_id,model,ccity_name,manufacturer,channel)
     b = time.time()
     print("cost(分钟)")
     print((b-a)/60)