import redis
import json
import time
import pymysql
from rediscluster import StrictRedisCluster
import numpy as np
# def getRedisConn():
#     conn = redis.Redis(host="172.18.51.10", port=6379,db=0)
#     conn.execute_command()
#     return conn
#
#
# def getRedisConn1():
#     pool = redis.ConnectionPool(host="172.16.40.133",password="ReDis!GmTx*0aN6",port=6379,db=0)
#     conn = redis.Redis(connection_pool=pool)
#     return conn
#
# def getRedisConn2():
#     pool = redis.ConnectionPool(host="172.16.40.173",password="ReDis!GmTx*0aN9",port=6379,db=0)
#     conn = redis.Redis(connection_pool=pool)
#     return conn
#
# def getRedisConn3():
#     pool = redis.ConnectionPool(host="172.16.40.164",password="ReDis!GmTx*0aN12",port=6379,db=0)
#     conn = redis.Redis(connection_pool=pool)
#     return conn
#
def getRedisConn4():
    # startup_nodes = [
    #     {'host': '172.16.179.131', 'port': '7000'},
    # ]
    # conn = StrictRedisCluster(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,decode_responses=True)
    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
    conn = redis.Redis(connection_pool=pool)
    return conn
#
# def getRedisConn5():
#     pool = redis.ConnectionPool(host="172.16.50.159",password="XfkMCCdWDIU%ls$h3",port=6379,db=0)
#     conn = redis.Redis(connection_pool=pool)
#     return conn
#
# def getDeviceIds():
#     ids_set = set()
#
#     db = pymysql.connect(host='172.16.30.136', port=3306, user='doris_olap', passwd='bA27hXasdfswuolap',
#                          db='doris_olap')
#     sql = "select distinct cl_id from user_tag3_portrait;"
#
#     cursor = db.cursor()
#     cursor.execute(sql)
#     datas = cursor.fetchall()
#     for d in datas:
#         device_id = str(d[0])
#         if device_id and len(device_id) > 0:
#             ids_set.add(str(d[0]))
#
#     print("deviceIds size:{}".format(str(len(ids_set))))
#     return ids_set
#
#
#
# keys = set()
# sum = 0
# key_sum = 0
# conn2 = getRedisConn2()
# pipline = conn2.pipeline()
# # pipline = conn.pipeline()
# # key = "have_reply_answer_comment*" #139285
# # key = "device_register_qa_read_set:869574031646601*" #139285
# # key = "doris_feed:*" #139285
# cursor = 0
# while True:
#     del_datas = set()
#     cursor, data = conn2.scan(cursor,"*",10000)
#     # cursor = str(cursor, encoding='utf-8')
#     for d in data:
#         dd = str(d, encoding='utf-8')
#         pipline.type(dd)
#     pip_datas = pipline.execute()
#     for i,pp in enumerate(pip_datas):
#         if str(pp,encoding='utf-8') == "list":
#             keys.add(data[i])
#             key_sum += 1
#     # if cursor == 0 or len(data) == 0:
#     sum += len(data)
#     print(cursor,sum,key_sum)
#     if cursor == 0:
#         break
#     print(key, key_sum)
#
#
#
# def redis2():
#     sum = 0
#     conn = getRedisConn2()
#     # key = "feed:recommend:by:user:portrait*"
#     # key = "doris:user_portrait:tag3:candidate_latest_dict:*"
#     # key = "doris:user_portrait:tag3:candidate_list:*" #1
#     # key = "device:latest:action:tag:names:update:*" #1831254
#     # key = "device:latest:action:tags:update:*" #1698541
#     # key = "feed:recommend:device_id:*" #1895536 #@bind('doris/recommend/feed')
#     # key = "feed:recommend:topic:device_id:*" #995546 #@bind('doris/recommend/feed')
#     # key = "user:service_portrait_tags2:cl_id:*" #21830
#     # key = "doris:aichannel_smart_rank:tag3:read:device_id:*" #727
#     # key = "doris:user_portrait:tag3:read_v2:device_id:*"  # 1658011 过期时间设置
#     key = "aaa:diary*"  # 5
#     cursor = 0
#     keys_map = {}
#     while True:
#         del_datas = set()
#         cursor, data = conn.scan(cursor=cursor, match=key, count=10000)
#         for d in data:
#             if len(d) > len(key) - 2:
#                 del_datas.add(str(d, encoding='utf-8'))
#         # pipline = conn.pipeline()
#         # for d in del_datas:
#         #     pipline.delete(d)
#         #     # pipline.expire(d,60*60*24*2)
#         # pipline.execute()
#         sum += len(data)
#         print(cursor, len(data), sum)
#         # if cursor == 0 or len(data) == 0:
#         if cursor == 0:
#             break
#
# def redis1():
#     keys = ['doris:query_compre_question_id:have_read_list'
#         , 'query_wiki_have_read:'
#         , 'doris:query_compre_answer:have_read_list:'
#         , 'doris_feed:doctor_have_read:device_id:'
#         , 'have_read_live_playback_device_id'
#         , 'doris_feed:ganhuo_have_read:device_id:'
#         , 'doris_feed:home_video_diary:device_id:'
#         , 'doris_feed:home_video_answer:device_id:'
#         , 'doris_feed:home_video_tractate:device_id:'
#         , 'doris_feed:good_look_have_read:device_id:'
#         , 'doris_feed:ganhuo_have_read:device_id:'
#         , 'doris_feed:bendi_have_read:device_id:'
#         , 'service:homepage_slide_tab:have_read:sku_ids:'
#         , 'doris_feed:home_recommend_diary_v1:device_id:'
#         , 'search:query_type:word:'
#         , 'doris_search:'
#         , 'servicehot:'
#         , 'interest:'
#         , 'device:'
#         , 'gaia:hot_tractate_keyword:city_id:'
#         , 'gaia:hot_wiki_keyword:city_id:']
#     for key_k in keys:
#         key = key_k + '*'
#         sum = 0
#         conn = getRedisConn1()
#         pipline = conn.pipeline()
#         # key = "have_reply_answer_comment*" #139285
#         # key = "device_register_qa_read_set:869574031646601*" #139285
#         # key = "doris_feed:*" #139285
#         cursor = 0
#         key_sum = 0
#         while True:
#             del_datas = set()
#             cursor, data = conn.scan(cursor=cursor, match=key, count=10000)
#             for d in data:
#                 dd = str(d, encoding='utf-8')
#                 if len(dd) > 0 and dd.startswith(key_k):
#                     del_datas.add(str(d, encoding='utf-8'))
#             pipline = conn.pipeline()
#             for d in del_datas:
#                 # pipline.delete(d)
#                 pipline.expire(d, 60 * 60 * 24 * 3)
#             pipline.execute()
#             sum += len(data)
#             key_sum += len(del_datas)
#             print(cursor, len(data), sum)
#             # if cursor == 0 or len(data) == 0:
#             if cursor == 0:
#                 break
#         print(key,key_sum)
#     # break
#
# conn2 = getRedisConn2()
# conn = getRedisConn4()
# # doris:tag_v3:coldstart:diary:
# # for key in ["rims:tag_v3:coldstart:answer","rims:tag_v3:coldstart:tractate"]:
# # key='user_portrait:doris:tag3:es:queue'
# # key='rims:tag_v3:coldstart:answer'
# key='rims:tag_v3:coldstart:tractate'
# ls = [str(s, encoding='utf-8') for s in conn2.lrange(key, 0, -1)]
# conn.delete(key)
# for i in range(0,len(ls),1000):
#     data = ls[i:i+1000]
#     conn.lpush(key,*data)
# # for i in range(0,len(ls),10000):
# #     conn.lpush(key, )
#
#
# #677
#
# #659
#
# def tett():
# # keys = ['rims:tag_v3:coldstart:'] #177
# # keys = ['doris:tag_v3:coldstart:'] #471
# # keys = ['rims:tag_v3:coldstart:','doris:tag_v3:coldstart:']
# keys = ['coldstart:light:clinic:beauty:']#11
# conn2 = getRedisConn2()
#
# for key_k in keys:
#     key = key_k + '*'
#     sum = 0
#     key_sum = 0
#     conn = getRedisConn4()
#     # pipline = conn.pipeline()
#     # key = "have_reply_answer_comment*" #139285
#     # key = "device_register_qa_read_set:869574031646601*" #139285
#     # key = "doris_feed:*" #139285
#     cursor = 0
#     while True:
#         del_datas = set()
#         cursor, data = conn2.scan(cursor,key,10000)
#         # cursor = str(cursor, encoding='utf-8')
#         for d in data:
#             dd = str(d, encoding='utf-8')
#             if len(dd) > 0 and dd.startswith(key_k):
#                 del_datas.add(str(d, encoding='utf-8'))
#         # pipline = conn.pipeline()
#         for d in del_datas:
#             if d in ["rims:tag_v3:coldstart:answer","rims:tag_v3:coldstart:tractate"]:
#                 continue
#             if str(conn2.type(d),encoding='utf-8') == "list":
#                 print(d)
#                 # ls = conn2.lrange(d, 0, -1)
#                 ls = [str(s, encoding='utf-8') for s in conn2.lrange(d, 0, -1)]
#                 conn.delete(d)
#                 conn.lpush(d, *ls)
#             # conn.delete(d)
#         #     # pipline.expire(d, 60 * 60 * 24 * 3)
#         # pipline.execute()
#         sum += len(data)
#         key_sum += len(del_datas)
#         print(cursor, len(data), key_sum)
#         # if cursor == 0 or len(data) == 0:
#         if cursor == 0:
#             break
#     print(key, key_sum)
#
#
#
# def redis4():
nodes=['7877da182171e313bc9326729f82999d1b629c79'
       ,'a4d4034faa81b935c2fd583053105b37f1c92ff1'
       ,'a8835d4c987847302bac66c5fc17ee1faae91fa3'
       ,'5a48236679f22637508651530633c9cc2f56f489'
       ,'fc3715919081c2cf3b30a2f8defb055c03564fdc'
       ,'eec89d7480980749c998add484e80f23fe5022a4'
       ,'f3f1ec6df458a5093c31663517a3cadaed5ab29c'
       ,'d35c630aad0a8b7f579bf4100f2860401b5d4f52']
# keys = ['rims:tag_v3:coldstart:','doris:tag_v3:coldstart:']
search_keys = ['']
do_delete = False
idletime_max = 60 * 60 * 180 * 24
# conn2 = getRedisConn2()
for search_key in search_keys:
    search_key = search_key + '*'
    sum = 0
    key_sum = 0
    conn = getRedisConn4()
    # pipline = conn.pipeline()
    # key = "have_reply_answer_comment*" #139285
    # key = "device_register_qa_read_set:869574031646601*" #139285
    # key = "doris_feed:*" #139285
    for node in nodes:
        cursor = "0"
        node_key_count = 0
        node_del_key_list = []
        while True:
            cursor, keys = conn.execute_command("scan {} match {} count {} {}".format(cursor,search_key,10000,node))
            keys = list(map(lambda key:str(key, encoding='utf-8'),keys))

            node_key_count += len(keys)
            cursor = str(cursor,encoding='utf-8')

            pipline = conn.pipeline()
            for key in keys:
                pipline.object('idletime', key)

            idletime_list = pipline.execute()
            for idx,idletime in enumerate(idletime_list):
                if idletime > idletime_max:
                    node_del_key_list.append(keys[idx])

            if cursor == "0":
                break

        node_del_key_count = len(node_del_key_list)
        print("node: {}, cursor: {}, node_key_count: {}, node_del_key_count: {}, node_del_key_rate: {}%".format(node, cursor, node_key_count, node_del_key_count, 100 * len(node_del_key_list) / node_key_count))
        print("node: {}, node_del_key_sample_100: {}".format(node, np.random.choice(node_del_key_list, 50, replace= False)))

        if do_delete:
            node_del_batch_size = 10000
            node_has_del_key_count = 0
            for batch_start_idx in range(0,node_del_key_count, node_del_batch_size):
                node_del_key_batch_list = node_del_key_list[batch_start_idx : batch_start_idx + node_del_batch_size]
                pipline = conn.pipeline()
                for node_del_key in node_del_key_batch_list:
                    pipline.delete(node_del_key)
                    # pass
                pipline.execute()
                node_has_del_key_count += len(node_del_key_batch_list)
                print("node: {}, node_has_del_key_count: {}, node_del_key_count: {}, node_has_del_key_rate: {}%".format(node, node_has_del_key_count, node_del_key_count, 100 * node_has_del_key_count / node_del_key_count))
                print("node: {}, node_has_del_key_batch_sample_50: {}".format(node, np.random.choice(node_del_key_batch_list, 50, replace= False)))

        print("")
#
#     nodes = ['7877da182171e313bc9326729f82999d1b629c79'
#         , 'a4d4034faa81b935c2fd583053105b37f1c92ff1'
#         , 'a8835d4c987847302bac66c5fc17ee1faae91fa3'
#         , '5a48236679f22637508651530633c9cc2f56f489'
#         , 'fc3715919081c2cf3b30a2f8defb055c03564fdc'
#         , 'eec89d7480980749c998add484e80f23fe5022a4'
#         , 'f3f1ec6df458a5093c31663517a3cadaed5ab29c'
#         , 'd35c630aad0a8b7f579bf4100f2860401b5d4f52']
#     conn = getRedisConn4()
#     num = 0
#     key_sum = 0
#     res = {}
#     keys_set = set()
#     for node in nodes:
#         cursor = "0"
#         while True:
#             cursor, data = conn.execute_command("scan {} match {} count {} {}".format(cursor, "*", 10000, node))
#             cursor = str(cursor, encoding='utf-8')
#             # pipline = conn.pipeline()
#             keys = []
#             for d in data:
#                 key = str(d, encoding='utf-8')
#                 is_flag = True
#                 for kk in ['streaming:candidate:','tag3:user_portrait:topn:']:
#                     if key.startswith(kk):
#                         is_flag = False
#                         break
#                 if is_flag:
#                     end = key.split(":")[-1]
#                     keys_set.add(key[0:len(key)-len(end)])
#                     keys.append(key)
#             key_sum += len(data)
#             print("node:{},curor:{},key_sum:{},keys size:{}".format(node,str(cursor), str(key_sum), str(len(keys_set))))
#             if cursor == "0":
#                 break
#         print(list(keys_set)[0:100])
#
#
#
# def redis5():
#     key_sum = 0
#     conn = getRedisConn5()
#     conn4 = getRedisConn4()
#     nodes=['d6b136db85c4c97c7bf9817f45bd4a2039e45fed'
#            ,'1ea04348ffc7a9858b2f86f5ef3b6565125f43df'
#            ,'9327036c8f0b4406f198f857b98198bd3472c85c'
#            ,'b6f89870957ef22d2aff0c6fa18a6bb9401a8294'
#            ,'4c5e23ed43c5b2bb4706a6335308920ac5990f94'
#            ,'0927a69bd5df3007175e55bb3ff0ac11bb9cd816'
#            ,'492a36ac916ed461a214d9c1e76ab21760a4b40d'
#            ,'5d8b9b2d7814df5664958e10178947a2fbd07f90']
#     for node in nodes:
#         cursor = "0"
#         while True:
#             cursor, data = conn.execute_command("scan {} match {} count {} {}".format(cursor, "*", 10000, node))
#             cursor = str(cursor, encoding='utf-8')
#             for d in data:
#                 dd = str(d, encoding='utf-8')
#                 # print(dd)
#                 t = str(conn.type(dd),encoding='utf-8')
#                 if t == "string":
#                     conn4.set(dd,conn.get(dd))
#                 if t == "hash":
#                     conn4.hmset(dd,conn.hgetall(dd))
#             key_sum += len(data)
#             print(node, cursor, len(data), key_sum)
#             if cursor == "0":
#                 break
#
#
# def write2ToRedis4_copy():
# res={"hash":0,"string":0,"list":0}
# key_sum = 0
# conn = getRedisConn2()
# conn4 = getRedisConn4()
# cursor = 0
# while True:
#     cursor, data = conn.scan(cursor=cursor,match="*",count=10000)
#     for d in data:
#         dd = str(d, encoding='utf-8')
#         # t = str(conn.type(dd), encoding='utf-8')
#         # ttl = conn.ttl(dd)
#         # print(dd)
#         t = str(conn.type(dd), encoding='utf-8')
#         # if t == "string":
#         #     conn4.set(dd, conn.get(dd))
#         #     if ttl:
#         #         conn4.expire(dd, ttl)
#         #     res["string"] += 1
#         # if t == "hash":
#         #     conn4.hmset(dd, conn.hgetall(dd))
#         #     if ttl:
#         #         conn4.expire(dd, ttl)
#         #     res["hash"] += 1
#         if t == "list":
#             datas = [conn.lrange(dd, 0, -1)]
#             conn4.lpush(dd, conn.lrange(dd,0,-1))
#             print(dd)
#             # if ttl:
#             #     conn4.expire(dd, ttl)
#             res["list"] += 1
#     key_sum += len(data)
#     print(cursor, len(data), key_sum)
#     if cursor == 0:
#         break
#
# def write2ToRedis4():
# res={"hash":0,"string":0,"list":0}
# key_sum = 0
# conn = getRedisConn2()
# conn4 = getRedisConn4()
# pipeline4 = conn4.pipeline()
# cursor = 0
# while True:
#     cursor, data = conn.scan(cursor=cursor,match="*",count=20000)
#     keys = []
#     for d in data:
#         dd = str(d, encoding='utf-8')
#         keys.append(dd)
#     pipeline = conn.pipeline()
#     for k in keys:
#         pipeline.type(k)
#     pip_datas1 = pipeline.execute()
#     pipeline = conn.pipeline()
#     for i,p in enumerate(pip_datas1):
#         t = str(p, encoding='utf-8')
#         dd = keys[i]
#         pipeline.ttl(dd)
#         if t == "string":
#             pipeline.get(dd)
#         elif t == "hash":
#             pipeline.hgetall(dd)
#         elif t == "list":
#             pipeline.lrange(dd, 0, -1)
#         else:
#             pipeline.get("123456787654321")
#     pip_datas2 = pipeline.execute()
#     for i in range(0,len(pip_datas2),2):
#         dd = keys[i//2]
#         t = str(pip_datas1[i//2], encoding='utf-8')
#         ttl = pip_datas2[i]
#         v = pip_datas2[i+1]
#         if t == "string":
#             if v:
#                 pipeline4.set(dd, v)
#             if ttl:
#                 pipeline4.expire(dd, ttl)
#             res["string"] += 1
#         if t == "hash":
#             if v:
#                 pipeline4.hmset(dd, v)
#             if ttl:
#                 pipeline4.expire(dd, ttl)
#             res["hash"] += 1
#         if t == "list":
#             if v:
#                 pipeline4.lpush(dd, v)
#             if ttl:
#                 pipeline4.expire(dd, ttl)
#             res["list"] += 1
#     pipeline4.execute()
#     key_sum += len(data)
#     print(cursor, len(data), key_sum)
#     if cursor == 0:
#         break
#
#
# if __name__ == '__main__':
#
#     conn = getRedisConn4()
#     num = 0
#     key_sum = 0
#     res = {}
#     cursor = 0
#     while True:
#         cursor, data = conn.scan(cursor=cursor, count=10000)
#         pipline = conn.pipeline()
#         keys = []
#         for d in data:
#             key = str(d, encoding='utf-8')
#             keys.append(key)
#             t = pipline.type(key)
#             idle = pipline.object('idletime', key)
#             ttl = pipline.ttl(key)
#         datass = pipline.execute()
#         for i in range(0, len(datass), 3):
#             t = str(datass[i], encoding='utf-8')
#             if idle and ttl and t=="set" and datass[i + 2] == -1:
#                 idle = datass[i + 1]
#                 ttl = datass[i + 2]
#                 key = keys[i // 3]
#                 add_d = res.setdefault(t, {})
#                 add_dd = add_d.setdefault(key, {})
#                 add_dd["ttl"] = ttl
#                 add_dd["idle"] = idle
#                 add_d[key] = add_dd
#                 res[t] = add_d
#                 num += 1
#         print("curor:{},key_sum:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(key_sum),str(num),str(len(data)),str(len(datass))))
#         key_sum += len(data)
#         if cursor == 0 or len(data) == 0:
#             break
#
#     #
#     #     end = int(time.time())
#     #     print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))
#     #
#     #     open('/tmp/redis_keys_hash_2.json', mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))
#