sort py

eda/esmm/Model_pipline/sort_and_2sql.py

@@ -19,13 +19,14 @@ def con_sql(sql):
     return result
 
 def set_join(lst):
-    return ','.join([str(i) for i in list(lst)])
+    # return ','.join([str(i) for i in list(lst)])
+    return ','.join([str(i) for i in lst.unique().tolist()])
 
 def main():
 
     # native queue
     df2 = pd.read_csv('/home/gaoyazhe/data/native.csv',usecols=[0,1,2],header=0,names=['uid','city','cid_id'],sep='\t')
-    df2['cid_id'] = df2['cid_id'].astype('object')
+    df2['cid_id'] = df2['cid_id'].astype(str)
 
     df1 = pd.read_csv("/home/gaoyazhe/data/native/pred.txt",sep='\t',header=None,names=["ctr","cvr","ctcvr"])
     df2["ctr"],df2["cvr"],df2["ctcvr"] = df1["ctr"],df1["cvr"],df1["ctcvr"]
@@ -36,7 +37,7 @@ def main():
 
     # nearby queue
     df2 = pd.read_csv('/home/gaoyazhe/data/nearby.csv',usecols=[0,1,2],header=0,names=['uid','city','cid_id'],sep='\t')
-    df2['cid_id'] = df2['cid_id'].astype('object')
+    df2['cid_id'] = df2['cid_id'].astype(str)
 
     df1 = pd.read_csv("/home/gaoyazhe/data/nearby/pred.txt",sep='\t',header=None,names=["ctr","cvr","ctcvr"])
     df2["ctr"], df2["cvr"], df2["ctcvr"] = df1["ctr"], df1["cvr"], df1["ctcvr"]
@@ -46,6 +47,8 @@ def main():
 
     #union
     df_all = pd.merge(df3,df4,on=['device_id','city_id'],how='outer').fillna("")
+    df_all['device_id'] = df_all['device_id'].astype(str)
+    df_all['city_id'] = df_all['city_id'].astype(str)
     ctime = int(time.time())
     df_all["time"] = ctime
     print("union_device_count",df_all.shape)
@@ -62,7 +65,8 @@ def main():
     engine = create_engine(str(r"mysql+mysqldb://%s:" + '%s' + "@%s:%s/%s") % (user, password, host, port, db))
 
     try:
-        df_merge = df_all[['device_id','city_id']].apply(lambda x: ''.join(x),axis=1)
+        # df_merge = df_all[['device_id','city_id']].apply(lambda x: ''.join(x),axis=1)
+        df_merge = df_all['device_id'] + df_all['city_id']
         df_merge_str = (str(list(df_merge.values))).strip('[]')
         delete_str = 'delete from esmm_device_diary_queue where concat(device_id,city_id) in ({0})'.format(df_merge_str)
         con = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')