Skip to content

S
serviceRec
Commit 858c00c8 authored by 郭羽's avatar 郭羽
Browse files
Options

get redis keys test

parent 2492c10a
Show whitespace changes
Inline Side-by-side
Showing with 182 additions and 23 deletions
test/get_redis_keys_hash.py
View file @ 858c00c8
......@@ -4,6 +4,7 @@ import time
import pymysql
def getRedisConn():
conn = redis.Redis(host="172.18.51.10", port=6379,db=0)
conn.execute_command()
return conn
......@@ -51,57 +52,215 @@ def getDeviceIds():
return ids_set
def redis2():
sum = 0
conn = getRedisConn2()
# key = "feed:recommend:by:user:portrait*"
# key = "doris:user_portrait:tag3:candidate_latest_dict:*"
# key = "doris:user_portrait:tag3:candidate_list:*" #1
# key = "device:latest:action:tag:names:update:*" #1831254
# key = "device:latest:action:tags:update:*" #1698541
# key = "feed:recommend:device_id:*" #1895536 #@bind('doris/recommend/feed')
# key = "feed:recommend:topic:device_id:*" #995546 #@bind('doris/recommend/feed')
# key = "user:service_portrait_tags2:cl_id:*" #21830
# key = "doris:aichannel_smart_rank:tag3:read:device_id:*" #727
# key = "doris:user_portrait:tag3:read_v2:device_id:*" # 1658011 过期时间设置
key = "aaa:diary*" # 5
cursor = 0
keys_map = {}
while True:
del_datas = set()
cursor, data = conn.scan(cursor=cursor, match=key, count=10000)
for d in data:
if len(d) > len(key) - 2:
del_datas.add(str(d, encoding='utf-8'))
# pipline = conn.pipeline()
# for d in del_datas:
# pipline.delete(d)
# # pipline.expire(d,60*60*24*2)
# pipline.execute()
sum += len(data)
print(cursor, len(data), sum)
# if cursor == 0 or len(data) == 0:
if cursor == 0:
break
def redis1():
keys = ['doris:query_compre_question_id:have_read_list'
, 'query_wiki_have_read:'
, 'doris:query_compre_answer:have_read_list:'
, 'doris_feed:doctor_have_read:device_id:'
, 'have_read_live_playback_device_id'
, 'doris_feed:ganhuo_have_read:device_id:'
, 'doris_feed:home_video_diary:device_id:'
, 'doris_feed:home_video_answer:device_id:'
, 'doris_feed:home_video_tractate:device_id:'
, 'doris_feed:good_look_have_read:device_id:'
, 'doris_feed:ganhuo_have_read:device_id:'
, 'doris_feed:bendi_have_read:device_id:'
, 'service:homepage_slide_tab:have_read:sku_ids:'
, 'doris_feed:home_recommend_diary_v1:device_id:'
, 'search:query_type:word:'
, 'doris_search:'
, 'servicehot:'
, 'interest:'
, 'device:'
, 'gaia:hot_tractate_keyword:city_id:'
, 'gaia:hot_wiki_keyword:city_id:']
for key_k in keys:
key = key_k + '*'
sum = 0
conn = getRedisConn1()
pipline = conn.pipeline()
# key = "have_reply_answer_comment*" #139285
# key = "device_register_qa_read_set:869574031646601*" #139285
# key = "doris_feed:*" #139285
cursor = 0
key_sum = 0
while True:
del_datas = set()
cursor, data = conn.scan(cursor=cursor, match=key, count=10000)
for d in data:
dd = str(d, encoding='utf-8')
if len(dd) > 0 and dd.startswith(key_k):
del_datas.add(str(d, encoding='utf-8'))
pipline = conn.pipeline()
for d in del_datas:
# pipline.delete(d)
pipline.expire(d, 60 * 60 * 24 * 3)
pipline.execute()
sum += len(data)
key_sum += len(del_datas)
print(cursor, len(data), sum)
# if cursor == 0 or len(data) == 0:
if cursor == 0:
break
print(key,key_sum)
# break
def redis4():
nodes=['7877da182171e313bc9326729f82999d1b629c79'
,'a4d4034faa81b935c2fd583053105b37f1c92ff1'
,'a8835d4c987847302bac66c5fc17ee1faae91fa3'
,'5a48236679f22637508651530633c9cc2f56f489'
,'fc3715919081c2cf3b30a2f8defb055c03564fdc'
,'eec89d7480980749c998add484e80f23fe5022a4'
,'f3f1ec6df458a5093c31663517a3cadaed5ab29c'
,'d35c630aad0a8b7f579bf4100f2860401b5d4f52']
keys = ['doris:user_portrait:tag3:candidate:tractate:device_id:']
for key_k in keys:
key = key_k + '*'
sum = 0
key_sum = 0
conn = getRedisConn4()
pipline = conn.pipeline()
# key = "have_reply_answer_comment*" #139285
# key = "device_register_qa_read_set:869574031646601*" #139285
# key = "doris_feed:*" #139285
for node in nodes:
cursor = "0"
while True:
del_datas = set()
cursor, data = conn.execute_command("scan {} match {} count {} {}".format(cursor,key,10000,node))
cursor = str(cursor,encoding='utf-8')
for d in data:
dd = str(d, encoding='utf-8')
if len(dd) > 0 and dd.startswith(key_k):
del_datas.add(str(d, encoding='utf-8'))
# pipline = conn.pipeline()
# for d in del_datas:
# # pipline.delete(d)
# pipline.expire(d, 60 * 60 * 24 * 3)
# pipline.execute()
sum += len(data)
key_sum += len(del_datas)
print(node,cursor, len(data), sum)
# if cursor == 0 or len(data) == 0:
if cursor == "0":
break
print(key,key_sum)
nodes = ['7877da182171e313bc9326729f82999d1b629c79'
, 'a4d4034faa81b935c2fd583053105b37f1c92ff1'
, 'a8835d4c987847302bac66c5fc17ee1faae91fa3'
, '5a48236679f22637508651530633c9cc2f56f489'
, 'fc3715919081c2cf3b30a2f8defb055c03564fdc'
, 'eec89d7480980749c998add484e80f23fe5022a4'
, 'f3f1ec6df458a5093c31663517a3cadaed5ab29c'
, 'd35c630aad0a8b7f579bf4100f2860401b5d4f52']
conn = getRedisConn4()
num = 0
key_sum = 0
res = {}
keys_set = set()
for node in nodes:
cursor = "0"
while True:
cursor, data = conn.execute_command("scan {} match {} count {} {}".format(cursor, "*", 10000, node))
cursor = str(cursor, encoding='utf-8')
# pipline = conn.pipeline()
keys = []
for d in data:
key = str(d, encoding='utf-8')
is_flag = True
for kk in ['streaming:candidate:','tag3:user_portrait:topn:']:
if key.startswith(kk):
is_flag = False
break
if is_flag:
end = key.split(":")[-1]
keys_set.add(key[0:len(key)-len(end)])
keys.append(key)
key_sum += len(data)
print("node:{},curor:{},key_sum:{},keys size:{}".format(node,str(cursor), str(key_sum), str(len(keys_set))))
if cursor == "0":
break
print(list(keys_set)[0:100])
if __name__ == '__main__':
# conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4(),getRedisConn5()]
# conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4()]
conns = [getRedisConn2()]
for index,conn in enumerate(conns):
start = int(time.time())
conn = getRedisConn4()
num = 0
key_sum = 0
res = {}
cursor = 0
while True:
cursor, data = conn.scan(cursor=cursor, count=1000)
cursor, data = conn.scan(cursor=cursor, count=10000)
pipline = conn.pipeline()
keys = []
for d in data:
key = str(d, encoding='utf-8')
keys.append(key)
t = pipline.type(key)
idle = pipline.object('idletime', key)
ttl = pipline.ttl(key)
datass = pipline.execute()
for i in range(0, len(datass), 3):
t = str(datass[i], encoding='utf-8')
if idle and ttl and t=="hash":
if idle and ttl and t=="set" and datass[i + 2] == -1:
idle = datass[i + 1]
ttl = datass[i + 2]
key = keys[i // 3]
add_d = res.setdefault(t, {})
add_dd = add_d.setdefault(key, {})
add_dd["ttl"] = ttl
add_dd["idle"] = idle
add_d[key] = add_dd
res[t] = add_d
num += 1
print("curor:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(num),str(len(data)),str(len(datass))))
# num += len(data)
print("curor:{},key_sum:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(key_sum),str(num),str(len(data)),str(len(datass))))
key_sum += len(data)
if cursor == 0 or len(data) == 0:
break
end = int(time.time())
print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
open('/tmp/redis_keys_hash_2_{}.json'.format(str(index+1)), mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))
#
# end = int(time.time())
# print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
#
# open('/tmp/redis_keys_hash_2.json', mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment