change test file

rerank_esmm.py

@@ -3,7 +3,7 @@ import datetime
 import json
 import redis
 import pandas as pd
-import time
+from sqlalchemy import create_engine
 
 
 def get_esmm_users():
@@ -186,21 +186,38 @@ def data_base(df2):
     #     print(e)
 
 
-def to_data_base():
+def to_data_base(df):
     db = pymysql.connect(host='172.16.40.158', port=4000, user='root',
                          passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     cursor = db.cursor()
-    sql = "select device_id from esmm_resort_diary_queue"
+    sql = "select distinct device_id from esmm_resort_diary_queue"
     cursor.execute(sql)
     result = cursor.fetchall()
-    db.close()
-    print(result)
-    if result is not None:
-        print([i[0] for i in result])
-        return [i[0] for i in result]
-    else:
-        print(122)
-        return []
+    cursor.close()
+    old_uid = [i[0] for i in result]
+    if len(old_uid) > 0:
+        old_uid = set(df["device_id"].values)&set(old_uid)
+        old_number = len(old_uid)
+        if old_number > 0:
+            sql = "delete from esmm_resort_diary_queue where device_id in {} limit 2000".format(tuple(old_uid))
+            if old_number > 2000:
+                cursor = db.cursor()
+                for i in range(int(old_number/2000)+1):
+                    cursor.execute(sql)
+                    db.commit()
+                cursor.close()
+            else:
+                cursor = db.cursor()
+                cursor.execute(sql)
+                db.commit()
+                cursor.close()
+            db.close()
+
+    yconnect = create_engine('mysql+pymysql://root:3SYz54LS9#^9sBvC@172.16.40.158:4000/jerry_test?charset=utf8')
+    pd.io.sql.to_sql(df, "esmm_resort_diary_queue", yconnect, schema='jerry_test', if_exists='append', index=False,
+                     chunksize=1000)
+    print("insert done")
+
 
 
 
@@ -209,31 +226,31 @@ def to_data_base():
 
 
 if __name__ == "__main__":
-    # users_list = get_esmm_users()
-    # total_samples = list()
-    # name_tag = get_searchworlds_to_tagid()
-    # # TODO 把下面的截断改掉
-    # for i in users_list[:60]:
-    #     tag_list = get_user_profile(i[0])
-    #     queues = get_queues(i[0],i[1])
-    #     if len(queues) > 0 and len(tag_list) > 0:
-    #         new_native = tag_boost(queues[0], tag_list)
-    #         new_nearby = tag_boost(queues[1], tag_list)
-    #
-    #         insert_time = str(datetime.datetime.now().strftime('%Y%m%d%H%M'))
-    #         sample = [i[0],i[1],new_native,new_nearby,queues[2],queues[3],insert_time]
-    #         total_samples.append(sample)
-    #
-    # if len(total_samples) > 0:
-    #     df = pd.DataFrame(total_samples)
-    #     df = df.rename(columns={0: "device_id", 1: "city_id",2:"native_queue",
-    #                             3:"nearby_queue",4:"nation_queue",5:"megacity_queue",6:"time"})
-    #     print(df.head(2))
-    #     to_data_base(df)
-    #     print("good boy")
-    # to_data_base(df)
-    # to kv
-    to_data_base()
+    users_list = get_esmm_users()
+    total_samples = list()
+    name_tag = get_searchworlds_to_tagid()
+    # TODO 把下面的截断改掉
+    for i in users_list[:60]:
+        tag_list = get_user_profile(i[0])
+        queues = get_queues(i[0],i[1])
+        if len(queues) > 0 and len(tag_list) > 0:
+            new_native = tag_boost(queues[0], tag_list)
+            new_nearby = tag_boost(queues[1], tag_list)
+
+            insert_time = str(datetime.datetime.now().strftime('%Y%m%d%H%M'))
+            sample = [i[0],i[1],new_native,new_nearby,queues[2],queues[3],insert_time]
+            total_samples.append(sample)
+
+    if len(total_samples) > 0:
+        df = pd.DataFrame(total_samples)
+        df = df.rename(columns={0: "device_id", 1: "city_id",2:"native_queue",
+                                3:"nearby_queue",4:"nation_queue",5:"megacity_queue",6:"time"})
+        print(df.head(2))
+        to_data_base(df)
+        print("good boy")
+
+    # TODO to kv
+
 
 
     # cid_str = "16473983,16296886,16199213,16193883,16419499,16372783,16430184,16617593,16498902,16238415,16214258,15715721,16213338,15349114,14091428,16268804,15485655,16448547,16179842,16685025,16612412,16683132,15646229,16482213,16485831,16436136,16353856,16400696,16193006,16294202,16393228,16716816,16713343,16780702,16107140,16647027,16112786,16503037,16372681,16207971,16179934,16480641,16295094,16204980,16317847,16434907,16117929,15633591,16116818"