1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import redis
import json
import time
import pymysql
def getRedisConn():
conn = redis.Redis(host="172.18.51.10", port=6379,db=0)
return conn
def getRedisConn1():
pool = redis.ConnectionPool(host="172.16.40.133",password="ReDis!GmTx*0aN6",port=6379,db=0)
conn = redis.Redis(connection_pool=pool)
return conn
def getRedisConn2():
pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
conn = redis.Redis(connection_pool=pool)
return conn
def getRedisConn3():
pool = redis.ConnectionPool(host="172.16.40.164",password="ReDis!GmTx*0aN12",port=6379,db=0)
conn = redis.Redis(connection_pool=pool)
return conn
def getRedisConn4():
pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
conn = redis.Redis(connection_pool=pool)
return conn
# def getRedisConn5():
# pool = redis.ConnectionPool(host="172.16.50.159",password="XfkMCCdWDIU%ls$h3",port=6379,db=0)
# conn = redis.Redis(connection_pool=pool)
# return conn
def getDeviceIds():
ids_set = set()
db = pymysql.connect(host='172.16.30.136', port=3306, user='doris_olap', passwd='bA27hXasdfswuolap',
db='doris_olap')
sql = "select distinct cl_id from user_tag3_portrait;"
cursor = db.cursor()
cursor.execute(sql)
datas = cursor.fetchall()
for d in datas:
device_id = str(d[0])
if device_id and len(device_id) > 0:
ids_set.add(str(d[0]))
print("deviceIds size:{}".format(str(len(ids_set))))
return ids_set
if __name__ == '__main__':
# conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4(),getRedisConn5()]
conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4()]
# conns = [getRedisConn2(),getRedisConn3(),getRedisConn4()]
for index,conn in enumerate(conns):
start = int(time.time())
num = 0
res = {}
cursor = 0
while True:
cursor, data = conn.scan(cursor=cursor, count=10000)
pipline = conn.pipeline()
keys = []
for d in data:
key = str(d, encoding='utf-8')
keys.append(key)
t = pipline.type(key)
idle = pipline.object('idletime', key)
ttl = pipline.ttl(key)
datass = pipline.execute()
for i in range(0, len(datass), 3):
t = str(datass[i], encoding='utf-8')
idle = datass[i + 1]
ttl = datass[i + 2]
key = keys[i // 3]
add_d = res.setdefault(t, {})
add_dd = add_d.setdefault(key, {})
add_dd["ttl"] = ttl
add_dd["idle"] = idle
add_d[key] = add_dd
res[t] = add_d
print("curor:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(num),str(len(data)),str(len(datass))))
num += len(data)
if cursor == 0 or len(data) == 0:
break
end = int(time.time())
print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
open('/tmp/redis_keys_{}.json'.format(str(index+1)), mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))