import redis
import json
import time
import pymysql
def getRedisConn():
    conn = redis.Redis(host="172.18.51.10", port=6379,db=0)
    return conn


def getRedisConn1():
    pool = redis.ConnectionPool(host="172.16.40.133",password="ReDis!GmTx*0aN6",port=6379,db=0)
    conn = redis.Redis(connection_pool=pool)
    return conn

def getRedisConn2():
    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
    conn = redis.Redis(connection_pool=pool)
    return conn

def getRedisConn3():
    pool = redis.ConnectionPool(host="172.16.40.164",password="ReDis!GmTx*0aN12",port=6379,db=0)
    conn = redis.Redis(connection_pool=pool)
    return conn

def getRedisConn4():
    pool = redis.ConnectionPool(host="172.16.50.145",password="XfkMCCdWDIU%ls$h",port=6379,db=0)
    conn = redis.Redis(connection_pool=pool)
    return conn

# def getRedisConn5():
#     pool = redis.ConnectionPool(host="172.16.50.159",password="XfkMCCdWDIU%ls$h3",port=6379,db=0)
#     conn = redis.Redis(connection_pool=pool)
#     return conn

def getDeviceIds():
    ids_set = set()

    db = pymysql.connect(host='172.16.30.136', port=3306, user='doris_olap', passwd='bA27hXasdfswuolap',
                         db='doris_olap')
    sql = "select distinct cl_id from user_tag3_portrait;"

    cursor = db.cursor()
    cursor.execute(sql)
    datas = cursor.fetchall()
    for d in datas:
        device_id = str(d[0])
        if device_id and len(device_id) > 0:
            ids_set.add(str(d[0]))

    print("deviceIds size:{}".format(str(len(ids_set))))
    return ids_set


if __name__ == '__main__':

    # conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4(),getRedisConn5()]
    conns = [getRedisConn1(),getRedisConn2(),getRedisConn3(),getRedisConn4()]
    # conns = [getRedisConn2(),getRedisConn3(),getRedisConn4()]
    for index,conn in enumerate(conns):
        start = int(time.time())
        num = 0
        res = {}
        cursor = 0
        while True:
            cursor, data = conn.scan(cursor=cursor, count=10000)
            pipline = conn.pipeline()

            keys = []
            for d in data:
                key = str(d, encoding='utf-8')
                keys.append(key)

                t = pipline.type(key)
                idle = pipline.object('idletime', key)
                ttl = pipline.ttl(key)

            datass = pipline.execute()
            for i in range(0, len(datass), 3):
                t = str(datass[i], encoding='utf-8')
                idle = datass[i + 1]
                ttl = datass[i + 2]
                key = keys[i // 3]

                add_d = res.setdefault(t, {})

                add_dd = add_d.setdefault(key, {})
                add_dd["ttl"] = ttl
                add_dd["idle"] = idle

                add_d[key] = add_dd

                res[t] = add_d

            print("curor:{},num:{},keys size:{},pipline size:{}".format(str(cursor),str(num),str(len(data)),str(len(datass))))

            num += len(data)
            if cursor == 0 or len(data) == 0:
                break

        end = int(time.time())
        print("conn:{},size:{},cost:{}s".format(str(index+1),str(num), str(end - start)))

        open('/tmp/redis_keys_{}.json'.format(str(index+1)), mode='w', encoding='utf-8').write(json.dumps(res, ensure_ascii=False))