get redis keys test

test/get_redis_keys_hash.py

+import redis
+import json
+import time
+import pymysql
+def getRedisConn():
+    conn = redis.Redis(host="172.18.51.10", port=6379,db=0)
+    return conn
+
+
+def getRedisConn1():
+    pool = redis.ConnectionPool(host="172.16.40.133",password="ReDis!GmTx*0aN6",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
+    return conn
+
+def getRedisConn2():
+    pool = redis.ConnectionPool(host="172.16.40.173",password="ReDis!GmTx*0aN9",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
+    return conn
+
+def getRedisConn3():
+    pool = redis.ConnectionPool(host="172.16.40.164",password="ReDis!GmTx*0aN12",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
+    return conn
+
+def getRedisConn4():
+    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
+    conn = redis.Redis(connection_pool=pool)
+    return conn
+
+# def getRedisConn5():
+#     pool = redis.ConnectionPool(host="172.16.50.159",password="XfkMCCdWDIU%ls$h3",port=6379,db=0)
+#     conn = redis.Redis(connection_pool=pool)
+#     return conn
+
+def getDeviceIds():
+    ids_set = set()
+
+    db = pymysql.connect(host='172.16.30.136', port=3306, user='doris_olap', passwd='bA27hXasdfswuolap',
+                         db='doris_olap')
+    sql = "select distinct cl_id from user_tag3_portrait;"
+
+    cursor = db.cursor()
+    cursor.execute(sql)
+    datas = cursor.fetchall()
+    for d in datas:
+        device_id = str(d[0])
+        if device_id and len(device_id) > 0:
+            ids_set.add(str(d[0]))
+
+    print("deviceIds size:{}".format(str(len(ids_set))))
+    return ids_set
+
+
+if __name__ == '__main__':
+
+    # conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4(),getRedisConn5()]
+    # conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4()]
+    conns = [getRedisConn2()]
+    for index,conn in enumerate(conns):
+        start = int(time.time())
+        num = 0
+        res = {}
+        cursor = 0
+        while True:
+            cursor, data = conn.scan(cursor=cursor, count=1000)
+            pipline = conn.pipeline()
+
+            keys = []
+            for d in data:
+                key = str(d, encoding='utf-8')
+                keys.append(key)
+
+                t = pipline.type(key)
+                idle = pipline.object('idletime', key)
+                ttl = pipline.ttl(key)
+
+            datass = pipline.execute()
+            for i in range(0, len(datass), 3):
+                t = str(datass[i], encoding='utf-8')
+                if idle and ttl and t=="hash":
+                    idle = datass[i + 1]
+                    ttl = datass[i + 2]
+                    key = keys[i // 3]
+
+                    add_d = res.setdefault(t, {})
+
+                    add_dd = add_d.setdefault(key, {})
+                    add_dd["ttl"] = ttl
+                    add_dd["idle"] = idle
+
+                    add_d[key] = add_dd
+
+                    res[t] = add_d
+
+            print("curor:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(num),str(len(data)),str(len(datass))))
+
+            num += len(data)
+            if cursor == 0 or len(data) == 0:
+                break
+
+        end = int(time.time())
+        print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
+
+        open('/tmp/redis_keys_hash_2_{}.json'.format(str(index+1)), mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))
+