add level2_ids

tensnsorflow/ffm.py

@@ -147,34 +147,23 @@ def get_data():
     start = "2018-11-19"
     print(start)
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
-    sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \
-          "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,s.hospital_id,s.doctor_id,e.cid_id " \
+    sql = "select e.y,e.z,e.stat_date,e.ucity_id,f.level2_ids,e.ccity_name," \
+          "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,s.hospital_id,s.doctor_id " \
           "from esmm_train_data e left join user_feature u on e.device_id = u.device_id " \
           "left join cid_type_top c on e.device_id = c.device_id left join cid_time on e.cid_id = cid_time.cid_id " \
-          "left join service_hospital s on e.diary_service_id = s.id " \
+          "left join service_hospital s on e.diary_service_id = s.id left join jerry_prod.diary_feat f" \
           "where e.stat_date >= '{}'".format(start)
     df = con_sql(db, sql)
     print(df.shape)
     df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id",4: "clevel1_id", 5: "ccity_name",
                             6:"device_type",7:"manufacturer",8:"channel",9:"top",10:"time",
-                            11:"hospital_id",12:"doctor_id",13:"cid_id"})
+                            11:"hospital_id",12:"doctor_id"})
     print("esmm data ok")
-    db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
-    sql = "select cid from diary_video"
-    diary_video = con_sql(db, sql)[0].values.tolist()
-    video_df = df.loc[df["cid_id"].isin(diary_video)]
-    print("video_df.shape")
-    print(video_df.shape)
-    video_df["video"] = "yes"
-    other_df = df.loc[~df["cid_id"].isin(diary_video)]
-    other_df["video"] = "no"
-    df = video_df.append(other_df).sort_index()
-    print(df.shape)
-    df = df.drop("cid_id", axis=1)
-    print(df.head(6))
+
+    print(df.head(2))
     features = 0
     category = ["ucity_id","clevel1_id","ccity_name","device_type","manufacturer","channel","top","doctor_id",
-                "hospital_id","video"]
+                "hospital_id"]
     for i in category:
         df[i] = df[i].fillna("na")
         features = features + len(df[i].unique())
@@ -212,8 +201,8 @@ def transform(a,validate_date):
     test = test.drop("stat_date",axis=1)
     # print("train shape")
     # print(train.shape)
-    train.to_csv(path + "tr_hdc.csv", sep="\t", index=False)
-    test.to_csv(path + "va_hdc.csv", sep="\t", index=False)
+    train.to_csv(path + "tr.csv", sep="\t", index=False)
+    test.to_csv(path + "va.csv", sep="\t", index=False)
 
     return model