改成分批删数据

eda/esmm/Model_pipline/to_database.py

@@ -72,17 +72,24 @@ def main():
     charset='utf8'
 
     df_merge = df_all['device_id'] + df_all['city_id']
-    df_merge_str = (str(list(df_merge.values))).strip('[]')
-
+    to_delete = list(df_merge.values)
+    total = len(to_delete)
+    df_merge_str = [str(to_delete[:int(total/5)]).strip('[]')]
+    for i in range(2,6):
+        start = int(total*(i -1)/5)
+        end = int(total*i/5)
+        tmp = str(to_delete[start:end]).strip('[]')
+        df_merge_str.append(tmp)
 
     try:
-        delete_str = 'delete from esmm_device_diary_queue where concat(device_id,city_id) in ({0})'.format(df_merge_str)
-        print(delete_str)
-        con = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
-        cur = con.cursor()
-        cur.execute(delete_str)
-        con.commit()
-        print("delete done")
+        for i in df_merge_str:
+            delete_str = 'delete from esmm_device_diary_queue where concat(device_id,city_id) in ({0})'.format(i)
+            con = pymysql.connect(host='172.16.40.158', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
+            cur = con.cursor()
+            cur.execute(delete_str)
+            con.commit()
+            print("delete done")
+            con.close()
         engine = create_engine(str(r"mysql+pymysql://%s:" + '%s' + "@%s:%s/%s") % (user, password, host, port, db))
         df_all.to_sql('esmm_device_diary_queue',con=engine,if_exists='append',index=False,chunksize=8000)
     except Exception as e: