Commit cd4f7988 authored by 张彦钊's avatar 张彦钊

multi hot insert database

parent 2a413f5c
import pandas as pd import pandas as pd
import pymysql import pymysql
import datetime import datetime
from sqlalchemy import create_engine
def con_sql(db,sql): def con_sql(db,sql):
cursor = db.cursor() cursor = db.cursor()
...@@ -27,6 +28,7 @@ def multi(): ...@@ -27,6 +28,7 @@ def multi():
df["l3"] = "lost" df["l3"] = "lost"
for i in list(df["level"].unique()): for i in list(df["level"].unique()):
l = i.split(";") l = i.split(";")
l = sorted(l)
if len(l) == 3: if len(l) == 3:
df.loc[df["level"] == i, ["l1"]] = l[0] df.loc[df["level"] == i, ["l1"]] = l[0]
df.loc[df["level"] == i, ["l2"]] = l[1] df.loc[df["level"] == i, ["l2"]] = l[1]
...@@ -38,15 +40,28 @@ def multi(): ...@@ -38,15 +40,28 @@ def multi():
df.loc[df["level"] == i, ["l1"]] = l[0] df.loc[df["level"] == i, ["l1"]] = l[0]
df = df.drop("level",axis=1) df = df.drop("level",axis=1)
print(df.head()) print(df.head())
a = list(df["l1"].unique()) # a = list(df["l1"].unique())
b = list(df["l2"].unique()) # b = list(df["l2"].unique())
c = list(df["l3"].unique()) # c = list(df["l3"].unique())
print(len(a)) # print(len(a))
print(a) # print(a)
print(len(b)) # print(len(b))
print(b) # print(b)
print(len(c)) # print(len(c))
print(c) # print(c)
yconnect = create_engine('mysql+pymysql://root:3SYz54LS9#^9sBvC@10.66.157.22:4000/jerry_test?charset=utf8')
n = 200000
for i in range(0, df.shape[0], n):
if i == 0:
temp = df.iloc[0:n]
elif i + n > df.shape[0]:
temp = df.iloc[i:]
else:
temp = df.loc[i:i + n]
pd.io.sql.to_sql(temp, "cid_level2", yconnect, schema='jerry_test', if_exists='replace', index=False)
print("insert done")
if __name__ == "__main__": if __name__ == "__main__":
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment