add data

tensnsorflow/ffm.py

@@ -31,20 +31,20 @@ def test():
 
 def get_data():
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
-    sql = "select device_id,y,z,stat_date,ucity_id,cid_id,clevel1_id,ccity_name from esmm_train_data limit 2000"
+    sql = "select device_id,y,z,stat_date,ucity_id,cid_id,clevel1_id,ccity_name from esmm_train_data"
     df = con_sql(db,sql)
     df = df.rename(columns={0:"device_id",1: "y",2:"z",3:"stat_date",4:"ucity_id",5:"cid_id",
                                 6:"clevel1_id",7:"ccity_name"})
     print("esmm data ok")
     print(df.head())
-    df["clevel1_id"] = df["clevel1_id"].astype("str")
-    df["cid_id"] = df["cid_id"].astype("str")
-    df["y"] = df["y"].astype("str")
-    df["z"] = df["z"].astype("str")
-    df["y"] = df["stat_date"].str.cat([df["device_id"].values.tolist(),df["ucity_id"].values.tolist(), df["cid_id"].values.tolist(),
-                                       df["y"].values.tolist(),df["z"].values.tolist()], sep=",")
-    df = df.drop("z", axis=1)
-    print(df.head())
+    # df["clevel1_id"] = df["clevel1_id"].astype("str")
+    # df["cid_id"] = df["cid_id"].astype("str")
+    # df["y"] = df["y"].astype("str")
+    # df["z"] = df["z"].astype("str")
+    # df["y"] = df["stat_date"].str.cat([df["device_id"].values.tolist(),df["ucity_id"].values.tolist(), df["cid_id"].values.tolist(),
+    #                                    df["y"].values.tolist(),df["z"].values.tolist()], sep=",")
+    # df = df.drop("z", axis=1)
+    # print(df.head())
     print("shape")
     print(df.shape)
     device_tuple = tuple(set(df["device_id"].values.tolist()))
@@ -98,11 +98,12 @@ def transform(df):
 def get_statistics(device_tuple):
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='eagle')
     sql = "select device_id,device_type,total,精选,直播,鼻部,眼部,微整,牙齿,轮廓,美肤抗衰," \
-          "吸脂,脂肪填充,隆胸,私密,毛发管理,公立,韩国 from home_tab_click where device_id in {}".format(device_tuple)
-    df = con_sql(db, sql)
+          "吸脂,脂肪填充,隆胸,私密,毛发管理,公立,韩国 from home_tab_click"
+    df = con_sql(db, sql).drop_duplicates()
     df = df.rename(columns={0:"device_id",1:"device_type",2:"channel",3:"total"})
-    for i in df.columns.difference(["device_id", "device_type","channel","total"]):
-        df[i] = df[i]/df["total"]
+    # for i in df.columns.difference(["device_id", "device_type","channel","total"]):
+    #     df[i] = df[i]/df["total"]
+    # df = df.drop("total", axis=1)
     return df