Skip to content

F
ffm-baseline
Commit 1d7d0e0c authored by 张彦钊's avatar 张彦钊
Browse files
Options

add level2

parent 00b54a0d
Show whitespace changes
Inline Side-by-side
Showing with 49 additions and 24 deletions
eda/esmm/Feature_pipline/data2ffm.py
View file @ 1d7d0e0c
...@@ -146,26 +146,28 @@ def get_data(): ...@@ -146,26 +146,28 @@ def get_data():
print(start) print(start)
db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test') db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \ sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \
"u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id " \ "u.device_type,u.manufacturer,u.channel,c.top,df.level2_ids,e.device_id " \
"from esmm_train_data e left join user_feature u on e.device_id = u.device_id " \ "from esmm_train_data e left join user_feature u on e.device_id = u.device_id " \
"left join cid_type_top c on e.device_id = c.device_id left join cid_time on e.cid_id = cid_time.cid_id " \ "left join cid_type_top c on e.device_id = c.device_id " \
"left join diary_feat df on e.cid_id = df.diary_id " \
"where e.stat_date >= '{}'".format(start) "where e.stat_date >= '{}'".format(start)
df = con_sql(db, sql) df = con_sql(db, sql)
print(df.shape) print(df.shape)
df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id",4: "clevel1_id", 5: "ccity_name", df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id",4: "clevel1_id", 5: "ccity_name",
6:"device_type",7:"manufacturer",8:"channel",9:"top",10:"time",11:"device_id"}) 6:"device_type",7:"manufacturer",8:"channel",9:"top",10:"level2_ids",11:"device_id"})
print("esmm data ok") print("esmm data ok")
print(df.head(2)) # print(df.head(2)
df["clevel1_id"] = df["clevel1_id"].astype("str") df["clevel1_id"] = df["clevel1_id"].astype("str")
df["y"] = df["y"].astype("str") df["y"] = df["y"].astype("str")
df["z"] = df["z"].astype("str") df["z"] = df["z"].astype("str")
df["top"] = df["top"].astype("str") df["top"] = df["top"].astype("str")
df["y"] = df["stat_date"].str.cat([df["device_id"].values.tolist(),df["y"].values.tolist(),df["z"].values.tolist()], sep=",") df["y"] = df["stat_date"].str.cat([df["device_id"].values.tolist(),df["y"].values.tolist(),df["z"].values.tolist()], sep=",")
df = df.drop(["z","stat_date","device_id"], axis=1).fillna(0.0)
df = df.drop(["z","stat_date","device_id"], axis=1).fillna("na")
print(df.head(2)) print(df.head(2))
features = 0 features = 0
for i in ["ucity_id","clevel1_id","ccity_name","device_type","manufacturer","channel"]: for i in ["ucity_id","clevel1_id","ccity_name","device_type","manufacturer","channel","level2_ids","top"]:
features = features + len(df[i].unique()) features = features + len(df[i].unique())
print("fields:{}".format(df.shape[1]-1)) print("fields:{}".format(df.shape[1]-1))
print("features:{}".format(features)) print("features:{}".format(features))
...@@ -196,8 +198,9 @@ def transform(a,validate_date): ...@@ -196,8 +198,9 @@ def transform(a,validate_date):
train = train.drop("stat_date",axis=1) train = train.drop("stat_date",axis=1)
test = df[df["stat_date"] == validate_date] test = df[df["stat_date"] == validate_date]
test = test.drop("stat_date",axis=1) test = test.drop("stat_date",axis=1)
# print("train shape") print("train shape")
# print(train.shape) print(train.shape)
train.to_csv(path + "tr.csv", sep="\t", index=False) train.to_csv(path + "tr.csv", sep="\t", index=False)
test.to_csv(path + "va.csv", sep="\t", index=False) test.to_csv(path + "va.csv", sep="\t", index=False)
...@@ -207,23 +210,35 @@ def transform(a,validate_date): ...@@ -207,23 +210,35 @@ def transform(a,validate_date):
def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel): def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test') db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \ sql = "select e.y,e.z,e.label,e.ucity_id,e.clevel1_id,e.ccity_name," \
"u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id,e.cid_id " \ "u.device_type,u.manufacturer,u.channel,c.top,df.level2_ids,e.device_id,e.cid_id " \
"from esmm_pre_data e left join user_feature u on e.device_id = u.device_id " \ "from esmm_pre_data e left join user_feature u on e.device_id = u.device_id " \
"left join cid_type_top c on e.device_id = c.device_id left join cid_time on e.cid_id = cid_time.cid_id" "left join cid_type_top c on e.device_id = c.device_id " \
"left join diary_feat df on e.cid_id = df.diary_id " \
"where e.device_id = '358035085192742'"
df = con_sql(db, sql) df = con_sql(db, sql)
df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name", df = df.rename(columns={0: "y", 1: "z", 2: "label", 3: "ucity_id", 4: "clevel1_id", 5: "ccity_name",
6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "time", 6: "device_type", 7: "manufacturer", 8: "channel", 9: "top", 10: "level2_ids",
11:"device_id",12:"cid_id"}) 11:"device_id",12:"cid_id"})
print("before filter:") print("before filter:")
print(df.shape) print(df.shape)
df = df[df["ucity_id"].isin(ucity_id)] df = df[df["ucity_id"].isin(ucity_id)]
print("after ucity filter:") print("after ucity filter:")
print(df.shape) print(df.shape)
df = df[df["ccity_name"].isin(ccity_name)] df = df[df["ccity_name"].isin(ccity_name)]
print("after ccity_name filter:")
print(df.shape)
df = df[df["manufacturer"].isin(manufacturer)] df = df[df["manufacturer"].isin(manufacturer)]
print("after manufacturer filter:")
print(df.shape)
df = df[df["channel"].isin(channel)] df = df[df["channel"].isin(channel)]
print("after ccity_name filter:") print("after channel filter:")
print(df.shape) print(df.shape)
df["cid_id"] = df["cid_id"].astype("str") df["cid_id"] = df["cid_id"].astype("str")
df["clevel1_id"] = df["clevel1_id"].astype("str") df["clevel1_id"] = df["clevel1_id"].astype("str")
df["top"] = df["top"].astype("str") df["top"] = df["top"].astype("str")
...@@ -234,9 +249,12 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel): ...@@ -234,9 +249,12 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
[df["device_id"].values.tolist(), df["ucity_id"].values.tolist(), df["cid_id"].values.tolist(), [df["device_id"].values.tolist(), df["ucity_id"].values.tolist(), df["cid_id"].values.tolist(),
df["y"].values.tolist(), df["z"].values.tolist()], sep=",") df["y"].values.tolist(), df["z"].values.tolist()], sep=",")
df = df.drop(["z","label","device_id","cid_id"], axis=1).fillna(0.0) df = df.drop(["z","label","device_id","cid_id"], axis=1).fillna(0.0)
print(df.head(2)) print("before transform")
df = model.transform(df,n=160000, processes=22) print(df.shape)
df = pd.DataFrame(df) temp_series = model.transform(df,n=160000, processes=22)
df = pd.DataFrame(temp_series)
print("after transform")
print(df.shape)
df["label"] = df[0].apply(lambda x: x.split(",")[0]) df["label"] = df[0].apply(lambda x: x.split(",")[0])
df["device_id"] = df[0].apply(lambda x: x.split(",")[1]) df["device_id"] = df[0].apply(lambda x: x.split(",")[1])
df["city_id"] = df[0].apply(lambda x: x.split(",")[2]) df["city_id"] = df[0].apply(lambda x: x.split(",")[2])
...@@ -249,26 +267,33 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel): ...@@ -249,26 +267,33 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
df = df.drop([0, "seq"], axis=1) df = df.drop([0, "seq"], axis=1)
print(df.head()) print(df.head())
print(df.loc[df["device_id"] == "358035085192742"].shape)
native_pre = df[df["label"] == "0"] native_pre = df[df["label"] == "0"]
native_pre = native_pre.drop("label", axis=1) native_pre = native_pre.drop("label", axis=1)
print("native")
print(native_pre.shape)
print(native_pre.loc[native_pre["device_id"] == "358035085192742"].shape)
native_pre.to_csv(path+"native.csv",sep="\t",index=False) native_pre.to_csv(path+"native.csv",sep="\t",index=False)
# print("native_pre shape") print("native_pre shape")
# print(native_pre.shape) print(native_pre.shape)
nearby_pre = df[df["label"] == "1"] nearby_pre = df[df["label"] == "1"]
nearby_pre = nearby_pre.drop("label", axis=1) nearby_pre = nearby_pre.drop("label", axis=1)
print("nearby")
print(nearby_pre.shape)
print(nearby_pre.loc[nearby_pre["device_id"] == "358035085192742"].shape)
nearby_pre.to_csv(path + "nearby.csv", sep="\t", index=False) nearby_pre.to_csv(path + "nearby.csv", sep="\t", index=False)
# print("nearby_pre shape") print("nearby_pre shape")
# print(nearby_pre.shape) print(nearby_pre.shape)
if __name__ == "__main__": if __name__ == "__main__":
path = "/home/gmuser/esmm_data/" path = "/home/gmuser/esmm_data/"
a = time.time() a = time.time()
df, validate_date, ucity_id,ccity_name,manufacturer,channel = get_data() temp, validate_date, ucity_id, ccity_name, manufacturer, channel = get_data()
model = transform(df, validate_date) model = transform(temp, validate_date)
get_predict_set(ucity_id,model,ccity_name,manufacturer,channel) get_predict_set(ucity_id, model, ccity_name, manufacturer, channel)
b = time.time() b = time.time()
print("cost(分钟)") print("cost(分钟)")
print((b-a)/60) print((b - a) / 60)
tensnsorflow/ffm.py
View file @ 1d7d0e0c
...@@ -290,7 +290,7 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel): ...@@ -290,7 +290,7 @@ def get_predict_set(ucity_id,model,ccity_name,manufacturer,channel):
if __name__ == "__main__": if __name__ == "__main__":
path = "/home/gmuser/ffm/" path = "/home/gmuser/esmm_data/"
a = time.time() a = time.time()
temp, validate_date, ucity_id,ccity_name,manufacturer,channel = get_data() temp, validate_date, ucity_id,ccity_name,manufacturer,channel = get_data()
model = transform(temp, validate_date) model = transform(temp, validate_date)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment