get redis keys test

test/get_redis_keys_hash.py

@@ -4,6 +4,7 @@ import time
 import pymysql
 def getRedisConn():
     conn = redis.Redis(host="172.18.51.10", port=6379,db=0)
+    conn.execute_command()
     return conn
 
 
@@ -51,57 +52,215 @@ def getDeviceIds():
     return ids_set
 
 
-if __name__ == '__main__':
+def redis2():
+    sum = 0
+    conn = getRedisConn2()
+    # key = "feed:recommend:by:user:portrait*"
+    # key = "doris:user_portrait:tag3:candidate_latest_dict:*"
+    # key = "doris:user_portrait:tag3:candidate_list:*" #1
+    # key = "device:latest:action:tag:names:update:*" #1831254
+    # key = "device:latest:action:tags:update:*" #1698541
+    # key = "feed:recommend:device_id:*" #1895536 #@bind('doris/recommend/feed')
+    # key = "feed:recommend:topic:device_id:*" #995546 #@bind('doris/recommend/feed')
+    # key = "user:service_portrait_tags2:cl_id:*" #21830
+    # key = "doris:aichannel_smart_rank:tag3:read:device_id:*" #727
+    # key = "doris:user_portrait:tag3:read_v2:device_id:*"  # 1658011 过期时间设置
+    key = "aaa:diary*"  # 5
+    cursor = 0
+    keys_map = {}
+    while True:
+        del_datas = set()
+        cursor, data = conn.scan(cursor=cursor, match=key, count=10000)
+        for d in data:
+            if len(d) > len(key) - 2:
+                del_datas.add(str(d, encoding='utf-8'))
+        # pipline = conn.pipeline()
+        # for d in del_datas:
+        #     pipline.delete(d)
+        #     # pipline.expire(d,60*60*24*2)
+        # pipline.execute()
+        sum += len(data)
+        print(cursor, len(data), sum)
+        # if cursor == 0 or len(data) == 0:
+        if cursor == 0:
+            break
 
-    # conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4(),getRedisConn5()]
-    # conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4()]
-    conns = [getRedisConn2()]
-    for index,conn in enumerate(conns):
-        start = int(time.time())
-        num = 0
-        res = {}
+def redis1():
+    keys = ['doris:query_compre_question_id:have_read_list'
+        , 'query_wiki_have_read:'
+        , 'doris:query_compre_answer:have_read_list:'
+        , 'doris_feed:doctor_have_read:device_id:'
+        , 'have_read_live_playback_device_id'
+        , 'doris_feed:ganhuo_have_read:device_id:'
+        , 'doris_feed:home_video_diary:device_id:'
+        , 'doris_feed:home_video_answer:device_id:'
+        , 'doris_feed:home_video_tractate:device_id:'
+        , 'doris_feed:good_look_have_read:device_id:'
+        , 'doris_feed:ganhuo_have_read:device_id:'
+        , 'doris_feed:bendi_have_read:device_id:'
+        , 'service:homepage_slide_tab:have_read:sku_ids:'
+        , 'doris_feed:home_recommend_diary_v1:device_id:'
+        , 'search:query_type:word:'
+        , 'doris_search:'
+        , 'servicehot:'
+        , 'interest:'
+        , 'device:'
+        , 'gaia:hot_tractate_keyword:city_id:'
+        , 'gaia:hot_wiki_keyword:city_id:']
+    for key_k in keys:
+        key = key_k + '*'
+        sum = 0
+        conn = getRedisConn1()
+        pipline = conn.pipeline()
+        # key = "have_reply_answer_comment*" #139285
+        # key = "device_register_qa_read_set:869574031646601*" #139285
+        # key = "doris_feed:*" #139285
         cursor = 0
+        key_sum = 0
         while True:
-            cursor, data = conn.scan(cursor=cursor, count=1000)
+            del_datas = set()
+            cursor, data = conn.scan(cursor=cursor, match=key, count=10000)
+            for d in data:
+                dd = str(d, encoding='utf-8')
+                if len(dd) > 0 and dd.startswith(key_k):
+                    del_datas.add(str(d, encoding='utf-8'))
             pipline = conn.pipeline()
+            for d in del_datas:
+                # pipline.delete(d)
+                pipline.expire(d, 60 * 60 * 24 * 3)
+            pipline.execute()
+            sum += len(data)
+            key_sum += len(del_datas)
+            print(cursor, len(data), sum)
+            # if cursor == 0 or len(data) == 0:
+            if cursor == 0:
+                break
+        print(key,key_sum)
+    # break
+
 
+def redis4():
+    nodes=['7877da182171e313bc9326729f82999d1b629c79'
+           ,'a4d4034faa81b935c2fd583053105b37f1c92ff1'
+           ,'a8835d4c987847302bac66c5fc17ee1faae91fa3'
+           ,'5a48236679f22637508651530633c9cc2f56f489'
+           ,'fc3715919081c2cf3b30a2f8defb055c03564fdc'
+           ,'eec89d7480980749c998add484e80f23fe5022a4'
+           ,'f3f1ec6df458a5093c31663517a3cadaed5ab29c'
+           ,'d35c630aad0a8b7f579bf4100f2860401b5d4f52']
+    keys = ['doris:user_portrait:tag3:candidate:tractate:device_id:']
+    for key_k in keys:
+        key = key_k + '*'
+        sum = 0
+        key_sum = 0
+        conn = getRedisConn4()
+        pipline = conn.pipeline()
+        # key = "have_reply_answer_comment*" #139285
+        # key = "device_register_qa_read_set:869574031646601*" #139285
+        # key = "doris_feed:*" #139285
+        for node in nodes:
+            cursor = "0"
+            while True:
+                del_datas = set()
+                cursor, data = conn.execute_command("scan {} match {} count {} {}".format(cursor,key,10000,node))
+                cursor = str(cursor,encoding='utf-8')
+                for d in data:
+                    dd = str(d, encoding='utf-8')
+                    if len(dd) > 0 and dd.startswith(key_k):
+                        del_datas.add(str(d, encoding='utf-8'))
+                # pipline = conn.pipeline()
+                # for d in del_datas:
+                #     # pipline.delete(d)
+                #     pipline.expire(d, 60 * 60 * 24 * 3)
+                # pipline.execute()
+                sum += len(data)
+                key_sum += len(del_datas)
+                print(node,cursor, len(data), sum)
+                # if cursor == 0 or len(data) == 0:
+                if cursor == "0":
+                    break
+        print(key,key_sum)
+
+    nodes = ['7877da182171e313bc9326729f82999d1b629c79'
+        , 'a4d4034faa81b935c2fd583053105b37f1c92ff1'
+        , 'a8835d4c987847302bac66c5fc17ee1faae91fa3'
+        , '5a48236679f22637508651530633c9cc2f56f489'
+        , 'fc3715919081c2cf3b30a2f8defb055c03564fdc'
+        , 'eec89d7480980749c998add484e80f23fe5022a4'
+        , 'f3f1ec6df458a5093c31663517a3cadaed5ab29c'
+        , 'd35c630aad0a8b7f579bf4100f2860401b5d4f52']
+    conn = getRedisConn4()
+    num = 0
+    key_sum = 0
+    res = {}
+    keys_set = set()
+    for node in nodes:
+        cursor = "0"
+        while True:
+            cursor, data = conn.execute_command("scan {} match {} count {} {}".format(cursor, "*", 10000, node))
+            cursor = str(cursor, encoding='utf-8')
+            # pipline = conn.pipeline()
             keys = []
             for d in data:
                 key = str(d, encoding='utf-8')
-                keys.append(key)
-
-                t = pipline.type(key)
-                idle = pipline.object('idletime', key)
-                ttl = pipline.ttl(key)
+                is_flag = True
+                for kk in ['streaming:candidate:','tag3:user_portrait:topn:']:
+                    if key.startswith(kk):
+                        is_flag = False
+                        break
+                if is_flag:
+                    end = key.split(":")[-1]
+                    keys_set.add(key[0:len(key)-len(end)])
+                    keys.append(key)
 
-            datass = pipline.execute()
-            for i in range(0, len(datass), 3):
-                t = str(datass[i], encoding='utf-8')
-                if idle and ttl and t=="hash":
-                    idle = datass[i + 1]
-                    ttl = datass[i + 2]
-                    key = keys[i // 3]
-
-                    add_d = res.setdefault(t, {})
-
-                    add_dd = add_d.setdefault(key, {})
-                    add_dd["ttl"] = ttl
-                    add_dd["idle"] = idle
-
-                    add_d[key] = add_dd
+            key_sum += len(data)
+            print("node:{},curor:{},key_sum:{},keys size:{}".format(node,str(cursor), str(key_sum), str(len(keys_set))))
+            if cursor == "0":
+                break
+    print(list(keys_set)[0:100])
 
-                    res[t] = add_d
+if __name__ == '__main__':
 
-                    num += 1
 
-            print("curor:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(num),str(len(data)),str(len(datass))))
 
-            # num += len(data)
-            if cursor == 0 or len(data) == 0:
-                break
 
-        end = int(time.time())
-        print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
+    conn = getRedisConn4()
+    num = 0
+    key_sum = 0
+    res = {}
+    cursor = 0
+    while True:
+        cursor, data = conn.scan(cursor=cursor, count=10000)
+        pipline = conn.pipeline()
+        keys = []
+        for d in data:
+            key = str(d, encoding='utf-8')
+            keys.append(key)
+            t = pipline.type(key)
+            idle = pipline.object('idletime', key)
+            ttl = pipline.ttl(key)
+        datass = pipline.execute()
+        for i in range(0, len(datass), 3):
+            t = str(datass[i], encoding='utf-8')
+            if idle and ttl and t=="set" and datass[i + 2] == -1:
+                idle = datass[i + 1]
+                ttl = datass[i + 2]
+                key = keys[i // 3]
+                add_d = res.setdefault(t, {})
+                add_dd = add_d.setdefault(key, {})
+                add_dd["ttl"] = ttl
+                add_dd["idle"] = idle
+                add_d[key] = add_dd
+                res[t] = add_d
+                num += 1
+        print("curor:{},key_sum:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(key_sum),str(num),str(len(data)),str(len(datass))))
+        key_sum += len(data)
+        if cursor == 0 or len(data) == 0:
+            break
 
-        open('/tmp/redis_keys_hash_2_{}.json'.format(str(index+1)), mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))
+    #
+    #     end = int(time.time())
+    #     print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
+    #
+    #     open('/tmp/redis_keys_hash_2.json', mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))