修改训练集sql

eda/esmm/Model_pipline/feature.py

@@ -37,7 +37,7 @@ def get_data():
     validate_date = con_sql(db, sql)[0].values.tolist()[0]
     print("validate_date:" + validate_date)
     temp = datetime.datetime.strptime(validate_date, "%Y-%m-%d")
-    start = (temp - datetime.timedelta(days=300)).strftime("%Y-%m-%d")
+    start = (temp - datetime.timedelta(days=60)).strftime("%Y-%m-%d")
     print(start)
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.stat_date,e.ucity_id,feat.level2_ids,e.ccity_name," \

eda/esmm/Model_pipline/to_database.py

@@ -36,10 +36,10 @@ def native_set_join(lst):
 def main():
 
     # native queue
-    df2 = pd.read_csv('/home/gmuser/esmm_data/native.csv')
+    df2 = pd.read_csv('/data/esmm/native.csv')
     df2['cid_id'] = df2['cid_id'].astype(str)
 
-    df1 = pd.read_csv("/home/gmuser/esmm_data/native/pred.txt",sep='\t',header=None,names=["ctr","cvr","ctcvr"])
+    df1 = pd.read_csv("/data/esmm/native/pred.txt",sep='\t',header=None,names=["ctr","cvr","ctcvr"])
     df2["ctr"],df2["cvr"],df2["ctcvr"] = df1["ctr"],df1["cvr"],df1["ctcvr"]
     df3 = df2.groupby(by=["uid","city"]).apply(lambda x: x.sort_values(by="ctcvr",ascending=False)).reset_index(drop=True).groupby(by=["uid","city"]).agg({'cid_id':native_set_join}).reset_index(drop=False)
     df3.columns = ["device_id","city_id","native_queue"]
@@ -47,10 +47,10 @@ def main():
 
 
     # nearby queue
-    df2 = pd.read_csv('/home/gmuser/esmm_data/nearby.csv')
+    df2 = pd.read_csv('/data/esmm/nearby.csv')
     df2['cid_id'] = df2['cid_id'].astype(str)
 
-    df1 = pd.read_csv("/home/gmuser/esmm_data/nearby/pred.txt",sep='\t',header=None,names=["ctr","cvr","ctcvr"])
+    df1 = pd.read_csv("/data/esmm/nearby/pred.txt",sep='\t',header=None,names=["ctr","cvr","ctcvr"])
     df2["ctr"], df2["cvr"], df2["ctcvr"] = df1["ctr"], df1["cvr"], df1["ctcvr"]
     df4 = df2.groupby(by=["uid","city"]).apply(lambda x: x.sort_values(by="ctcvr",ascending=False)).reset_index(drop=True).groupby(by=["uid","city"]).agg({'cid_id':nearby_set_join}).reset_index(drop=False)
     df4.columns = ["device_id","city_id","nearby_queue"]
@@ -64,8 +64,6 @@ def main():
     df_all["time"] = ctime
     print("union_device_count",df_all.shape)
 
-
-
     host='10.66.157.22'
     port=4000
     user='root'
@@ -74,17 +72,17 @@ def main():
     charset='utf8'
 
     engine = create_engine(str(r"mysql+mysqldb://%s:" + '%s' + "@%s:%s/%s") % (user, password, host, port, db))
-
+    df_merge = df_all['device_id'] + df_all['city_id']
+    df_merge_str = (str(list(df_merge.values))).strip('[]')
     try:
         # df_merge = df_all[['device_id','city_id']].apply(lambda x: ''.join(x),axis=1)
-        df_merge = df_all['device_id'] + df_all['city_id']
-        df_merge_str = (str(list(df_merge.values))).strip('[]')
+
         delete_str = 'delete from esmm_device_diary_queue where concat(device_id,city_id) in ({0})'.format(df_merge_str)
         con = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
         cur = con.cursor()
         cur.execute(delete_str)
         con.commit()
-        df_all.to_sql('esmm_device_diary_queue',con=engine,if_exists='append',index=False,chunksize=8000)
+        df_all.to_sql('esmm_device_diary_queue',con=engine,if_exists='append',index=False)
     except Exception as e:
         print(e)