Merge branch 'zhao' into 'master'

用户画像重排增加if判断

See merge request !34

eda/esmm/Model_pipline/rerank_esmm.py

@@ -170,11 +170,11 @@ def to_data_base(df):
 
 
 if __name__ == "__main__":
-    users_list = get_esmm_users()[:2000]
-    # TODO 删除上面的2000
+    users_list = get_esmm_users()
     print("user number")
     print(len(users_list))
 
+    if len(users_list) > 0:
         name_tag = get_searchworlds_to_tagid()
         n = 500
         split_users_list = [users_list[i:i + n] for i in range(0, len(users_list), n)]
@@ -197,6 +197,8 @@ if __name__ == "__main__":
                                         3:"nearby_queue",4:"nation_queue",5:"megacity_queue",6:"time"})
 
                 to_data_base(df)
+    else:
+        print("没有获取到用户")
 
 
 

tensnsorflow/content_09_05.py

+
+# 新增一个用户画像对比接口
+from itertools import chain, islice, cycle
+import datetime
+from collections import Counter
+
+from gm_types.gaia import DIARY_ORDER_TYPE
+from gm_types.doris import ANSWER_SORT_TYPE
+from gm_types.doris import ARTICLE_SORT_TYPE
+from gm_types.mimas import CONTENT_CLASS
+from gm_types.doris import CARD_TYPE
+from gm_types.gaia import CITY_LEVEL
+from gm_rpcd.all import bind
+import traceback
+from search.utils.diary import recall_diary
+from search.utils.answer import recall_answers
+from search.utils.article import recall_articles
+from gm_rpcd.all import context
+from libs.algorithms import drop_dup
+from libs.cache import redis_client
+from libs.error import logging_exception
+
+from extend.models.gaia import City, CityScale
+from extend.models.gold import (
+    QAQueue,
+    WikiQueue,
+    IconQueue,
+    UserTopicQueue,
+    DoctorTopicQueue,
+    DiaryQueue,
+    ArticleQueue,
+    AnswerQueue,
+    DeviceQAQueue,
+    DeviceIconQueue,
+    DeviceUserTopicQueue,
+    DeviceDoctorTopicQueue,
+    DeviceAnswerQueue,
+    DeviceArticleQueue,
+    DeviceDiaryQueue,
+    QuestionQueue,
+    DeviceQuestionQueue
+)
+import logging
+import redis
+import json
+from django.conf import settings
+import traceback
+
+MAX_LOAD = 200
+logger = logging.getLogger(__name__)
+
+
+@bind("doris/recommend/get_diaries")
+def get_diaries(tags, city, offset=0, size=10, city_tag_id=None):
+    # NOTE: city as city id
+    sort_params = {}
+
+    if city_tag_id:
+        sort_params["user_city_tag_id"] = city_tag_id
+
+    elif city:
+        try:
+            x = City.objects.get(id=city)
+            sort_params["user_city_tag_id"] = x.tag_id
+        except City.DoesNotExist:
+            pass
+
+    filters = {
+        "is_sink": False,
+        "has_before_cover": True,
+        "has_after_cover": True,
+        "content_level_is_good": True
+    }
+    if tags:
+        filters["closure_tag_ids"] = tags
+
+    tail = offset + size
+    diaries_ids = []
+    if tail < MAX_LOAD:
+        diaries = recall_diary(None, 0, 200, filters, DIARY_ORDER_TYPE.RECOMMEND, sort_params, fields=["id", "user.id"])
+        diaries_items = [(diary['id'], diary['user']['id']) for diary in diaries]
+        drop_dup_diaries = drop_dup(diaries_items)
+        drop_dup_size = len(drop_dup_diaries)
+        if tail <= drop_dup_size:
+            diaries_ids = [item[0] for item in drop_dup_diaries[offset:tail]]
+    if len(diaries_ids) == 0:  # 如果头200条去重结束 后面的排序不去重
+        diaries = recall_diary(None, offset, size, filters, DIARY_ORDER_TYPE.RECOMMEND, sort_params, fields=["id"])
+        diaries_ids = [diary['id'] for diary in diaries]
+
+    return {"diaries_ids": diaries_ids}
+
+
+@bind("doris/recommend/get_articles")
+def get_articles(tags, offset=0, size=10):
+    filters = {
+        "content_level": [CONTENT_CLASS.EXCELLENT, CONTENT_CLASS.FINE]
+    }
+    if tags:
+        filters["tag_ids"] = tags
+
+    articles = recall_articles(None, offset, size, filters, ARTICLE_SORT_TYPE.RECOMMEND, {})
+    article_ids = [article['id'] for article in articles]
+    return {"article_ids": article_ids}
+
+
+@bind("doris/recommend/get_answers")
+def get_answers(tags, offset=0, size=10):
+    filters = {
+        "content_level": [CONTENT_CLASS.EXCELLENT, CONTENT_CLASS.FINE]
+    }
+
+    if tags:
+        filters["tag_ids"] = tags
+    tail = offset + size
+    answer_ids = []
+    if tail < MAX_LOAD:
+        answers = recall_answers(None, 0, MAX_LOAD, filters, ANSWER_SORT_TYPE.RECOMMEND, {}, fields=["id", "user_id"])
+        answers = filter(lambda answer: "id" in answer and "user_id" in answer, answers)
+        answer_items = [(answer["id"], answer["user_id"]) for answer in answers]
+        drop_dup_answers = drop_dup(answer_items)
+        if tail <= len(drop_dup_answers):
+            answer_ids = [item[0] for item in drop_dup_answers[offset:tail]]
+    if len(answer_ids) == 0:
+        answers = recall_answers(None, offset, size, filters, ANSWER_SORT_TYPE.RECOMMEND, {})
+        answer_ids = [answer['id'] for answer in answers]
+    return {"answer_ids": answer_ids}
+
+@bind('doris/recommend/icon')
+def fetch_icon(device_id, size):
+    try:
+        card_type = "icon"
+        try:
+             que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+             que = IconQueue.objects.last()
+        if not que:
+            return {"icon": []}
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        data = list(islice(cycle(que), cursor, cursor + size))
+        return {card_type: list(map(int, data))}
+    except:
+        logging_exception()
+        return {"icon": []}
+
+@bind('doris/recommend/homepage_polymer')
+def fetch_polymer_ids(device_id, size):
+    try:
+        card_type = "polymer_ids"
+        try:
+            que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+            que = IconQueue.objects.last()
+        if not que:
+            return {"polymer_ids": []}
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        data = list(islice(cycle(que), cursor, cursor + size))
+        return {card_type: list(map(int, data))}
+    except:
+        logging_exception()
+        return {"polymer_ids": []}
+
+@bind('doris/recommend/feed')
+def recommend_feed(device_id, card_type, city_id, size):
+    try:
+        return RecommendFeed.dispatch(device_id, card_type,
+                                      city_id, size)
+    except:
+        logging_exception()
+        return {card_type: []}
+
+
+class RecommendFeed:
+    @classmethod
+    def dispatch(cls, device_id, card_type, city_id, size):
+        data = []
+        if card_type == CARD_TYPE.QA:
+            data = cls.fetch_qa(device_id, card_type, size)
+        elif card_type == CARD_TYPE.ANSWER:
+            data = cls.fetch_answer(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.ARTICLE:
+            data = cls.fetch_article(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.QUESTION:
+            data = cls.fetch_question(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.DIARY:
+            data = cls.fetch_diary(device_id, card_type, city_id, size)
+
+        elif card_type == CARD_TYPE.USERTOPIC:
+            data = cls.fetch_user_topic(device_id,card_type,size)
+        elif card_type == CARD_TYPE.DOCTORTOPIC:
+            data = cls.fetch_doctor_topic(device_id,card_type,size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.ENCYCLOPEDIA:
+            data = cls.fetch_wiki(device_id,card_type,size)
+        return {card_type: data}
+
+    @staticmethod
+    def current_date():
+        return datetime.datetime.now().strftime('%Y-%m-%d')
+
+    @staticmethod
+    def fetch_question(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceQuestionQueue.objects.get(device_id=device_id)
+        except DeviceQuestionQueue.DoesNotExist:
+            que = QuestionQueue.objects.last()
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_icon(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+            que = IconQueue.objects.last()
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_wiki(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+
+            que = WikiQueue.objects.last()
+            if not que:
+                return []
+            # que = list(filter(None, que.queue.split(',')))
+            que = json.loads(que.queue)
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_answer(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                          card_type=card_type, date=RecommendFeed.current_date())
+            try:
+                que = DeviceAnswerQueue.objects.get(device_id=device_id)
+            except DeviceAnswerQueue.DoesNotExist:
+                que = AnswerQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_qa(device_id, card_type, size):
+        try:
+            def get_after_filter_qa():
+                try:
+                    return json.loads(gmkv.get(after_filter_key))
+                except:
+                    return []
+
+            def write_after_filter_qa(cid_list):
+                try:
+                    if gmkv.exists(after_filter_key):
+                        gmkv.set(after_filter_key, json.dumps(cid_list))
+                    else:
+                        gmkv.set(after_filter_key, json.dumps(cid_list),ex = 6*60*60)
+
+                except:
+                    logging_exception()
+                    logger.error("catch exception,err_log:%s" % traceback.format_exc())
+
+            def filter_qa(device_id,cid_list):
+                try:
+                    key = str(device_id) + "_dislike_qa"
+                    if gmkv.exists(key):
+                        dislike = gmkv.smembers(key)
+                        if len(cid_list) > 0:
+                            if type(cid_list[0]) == int or type(cid_list[0]) == str:
+                                cid_list = [i for i in cid_list if str(i).encode('utf-8') not in dislike]
+                            else:
+                                cid_list = [i for i in cid_list if i not in dislike]
+                        return cid_list
+                    else:
+                        return cid_list
+                except:
+                    return cid_list
+
+            def read_history(cid_list):
+                if redis_client.exists(today_qa_key):
+                    redis_client.sadd(today_qa_key, *cid_list)
+                else:
+                    redis_client.sadd(today_qa_key, *cid_list)
+                    redis_client.expire(today_qa_key, 15 * 24 * 60 * 60)
+                if redis_client.exists(read_qa_key) and redis_client.exists(old_qa_key):
+                    redis_client.sdiffstore(read_qa_key, read_qa_key, old_qa_key)
+                    redis_client.delete(old_qa_key)
+                    redis_client.expire(read_qa_key, time=13 * 24 * 60 * 60)
+
+                redis_client.sadd(read_qa_key, *cid_list)
+
+            def get_gmkv(redis_ip, redis_port, redis_db, redis_password=""):
+                try:
+                    if len(redis_password) == 0:
+                        cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, socket_timeout=2)
+                    else:
+                        cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, password=redis_password,
+                                              socket_timeout=2)
+                    cli_ins.ping()
+
+                    return cli_ins
+                except:
+                    return None
+
+            search_qa_recommend_list = list()
+            read_qa_key = "TS:recommend_answer_set:device_id:" + str(device_id)
+            old_qa_key = "TS:recommend_answer_set:device_id:{}:{}"\
+                .format(device_id,(datetime.date.today() - datetime.timedelta(days=14)).strftime("%Y-%m-%d"))
+            today_qa_key = "TS:recommend_answer_set:device_id:{}:{}"\
+                .format(device_id, datetime.date.today().strftime("%Y-%m-%d"))
+            answer_queue_key = "qa_is_tail:" + str(device_id)
+            after_filter_key = "device_qa_after_filter:device_id:" + str(device_id)
+            gmkv = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gmkv = get_gmkv(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"],
+                                         redis_db=gm_kv_host_item["db"],
+                                         redis_password=gm_kv_host_item["password"])
+                if gmkv:
+                    break
+
+            if device_id != '0':
+                search_qa_recommend_key = "TS:search_recommend_answer_queue:device_id:" + str(device_id)
+                if redis_client.exists(search_qa_recommend_key):
+                    search_qa_recommend_dict = redis_client.hgetall(search_qa_recommend_key)
+                    queue_list = json.loads(search_qa_recommend_dict[b'answer_queue'])
+                    queue_list = filter_qa(device_id, queue_list)
+                    if len(queue_list) == 0:
+                        redis_client.delete(search_qa_recommend_key)
+                    elif len(queue_list) == 1:
+                        size = size - 1
+                        search_qa_recommend_list = queue_list
+                        redis_client.delete(search_qa_recommend_key)
+                    else:
+                        size = size - 1
+                        search_qa_recommend_list.append(queue_list[0])
+                        redis_client.hset(search_qa_recommend_key,"answer_queue",json.dumps(queue_list[1:]))
+
+                if gmkv.exists(answer_queue_key):
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                elif gmkv.exists(after_filter_key):
+                    que = get_after_filter_qa()
+                    que = filter_qa(device_id,que)
+                    if len(que) == 0:
+                        gmkv.set(answer_queue_key,"tail",ex = 6*60*60)
+                        if len(search_qa_recommend_list) > 0:
+                            search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                            read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+                    elif len(que) <= size:
+                        search_qa_recommend_list.extend(que)
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+                    else:
+                        search_qa_recommend_list.extend(que[:size])
+                        write_after_filter_qa(que[size:])
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+
+                try:
+                    que = DeviceQAQueue.objects.get(device_id=device_id)
+                except DeviceQAQueue.DoesNotExist:
+                    que = AnswerQueue.objects.last()
+                if not que:
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                qa = list(filter(None, que.queue.split(',')))
+                if device_id != "0":
+                    qa = filter_qa(device_id,qa)
+                if len(qa) == 0:
+                    if device_id != "0":
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                elif len(qa) <= size:
+                    search_qa_recommend_list.extend(qa)
+                    search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                    if device_id != "0":
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                else:
+                    search_qa_recommend_list.extend(qa[:size])
+                    search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                    if device_id != "0":
+                        write_after_filter_qa(qa[size:])
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+
+            else:
+                key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                              card_type=card_type, date=RecommendFeed.current_date())
+                try:
+                    que = DeviceQAQueue.objects.get(device_id=device_id)
+                except DeviceQAQueue.DoesNotExist:
+                    que = AnswerQueue.objects.last()
+                if not que:
+                    return []
+                que = list(filter(None, que.queue.split(',')))
+
+                # adjust args.
+                cursor = redis_client.get(key) or 0
+                cursor = int(cursor) % len(que)
+                size = min(size, len(que))
+                # redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                data = list(islice(cycle(que), cursor, cursor + size))
+                data = list(map(int, data))
+                if cursor + 2 * size < len(que):
+                    redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                else:
+                    try:
+                        context.request_logger.app(reset_answer_queue=True)
+                        cursor = 0
+                        redis_client.set(key, cursor, ex=24 * 60 * 60)
+                    except:
+                        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                return data
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_article(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceArticleQueue.objects.get(device_id=device_id)
+        except DeviceArticleQueue.DoesNotExist:
+            que = ArticleQueue.objects.last()
+
+        if not que:
+            return []
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_user_topic(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id, card_type=card_type,
+                                                          date=RecommendFeed.current_date())
+
+            if (device_id != '0') and size >= 2:
+
+                search_topic_recommend_key = "TS:search_recommend_tractate_queue:device_id:" + str(device_id)
+                search_topic_recommend_list = list()
+                search_cursor_ts = 0
+                if redis_client.exists(search_topic_recommend_key):
+                    search_topic_recommend_dict = redis_client.hgetall(search_topic_recommend_key)
+                    if b'cursor' in search_topic_recommend_dict:
+                        search_cursor_ts = json.loads(search_topic_recommend_dict[b'cursor'])
+                        if search_cursor_ts < 30:
+                            search_topic_recommend_list = json.loads(search_topic_recommend_dict[b'tractate_queue'])
+                            if search_cursor_ts < len(search_topic_recommend_list):
+                                size = size - 2
+
+            try:
+                que = DeviceUserTopicQueue.objects.get(device_id=device_id)
+            except DeviceUserTopicQueue.DoesNotExist:
+                que = UserTopicQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            data = list(islice(cycle(que), cursor, cursor + size))
+            data = list(map(int, data))
+
+            if cursor + 2 * size < len(que):
+                redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            else:
+                try:
+                    context.request_logger.app(reset_queue=True)
+                    cursor = 0
+                    redis_client.set(key, cursor, ex=24 * 60 * 60)
+                except:
+                    redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+
+            if device_id != '0' and size >= 2:
+                if len(search_topic_recommend_list) > 0 and search_cursor_ts < len(search_topic_recommend_list):
+                    queue = search_topic_recommend_list[search_cursor_ts:search_cursor_ts + 2]
+                    queue.extend(data)
+                    data = queue
+                    new_search_cursor = search_cursor_ts + 2
+                    redis_client.hset(search_topic_recommend_key, 'cursor', new_search_cursor)
+                    redis_client.expire(search_topic_recommend_key, 30 * 24 * 60 * 60)
+                read_topic_key = "TS:recommend_tractate_set:device_id:" + str(device_id)
+                if len(data) > 0:
+                    redis_client.sadd(read_topic_key, *data)
+            return data
+
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_doctor_topic(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                          card_type=card_type, date=RecommendFeed.current_date())
+            try:
+                que = DeviceDoctorTopicQueue.objects.get(device_id=device_id)
+            except DeviceDoctorTopicQueue.DoesNotExist:
+                que = DoctorTopicQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+        except:
+            logging_exception()
+            return []
+
+
+    @classmethod
+    def get_gm_kv_ins(cls,redis_ip, redis_port, redis_db, redis_password=""):
+        try:
+            if len(redis_password) == 0:
+                cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, socket_timeout=2)
+            else:
+                cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, password=redis_password,
+                                      socket_timeout=2)
+            cli_ins.ping()
+
+            return cli_ins
+        except:
+            return None
+
+    @classmethod
+    def fetch_diary_queue_data(cls, city_id, device_id=None):
+        local = list()
+        nearby = list()
+        nation = list()
+        megacity = list()
+        use_city_id = city_id
+        try:
+            gm_kv_ins = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gm_kv_ins = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"], redis_db=gm_kv_host_item["db"],redis_password=gm_kv_host_item["password"])
+                if gm_kv_ins:
+                    break
+
+            specify_city_id_key = "diary_queue:city_id:" + use_city_id
+            world_city_id_key = "diary_queue:city_id:world"
+            if device_id is not None:
+                specify_city_id_key = "device_diary_queue:device_id:" + device_id + ":city_id:" + use_city_id
+
+            city_val_dict = gm_kv_ins.hgetall(specify_city_id_key)
+            if len(city_val_dict) == 0:
+                city_val_dict = gm_kv_ins.hgetall(world_city_id_key)
+                use_city_id = "world"
+
+            if b"native_queue" in city_val_dict and city_val_dict[b"native_queue"]:
+                local = list(filter(None, city_val_dict[b"native_queue"].split(b",")))
+            if b"nearby_queue" in city_val_dict and city_val_dict[b"nearby_queue"]:
+                nearby = list(filter(None, city_val_dict[b"nearby_queue"].split(b",")))
+            if b"nation_queue" in city_val_dict and city_val_dict[b"nation_queue"]:
+                nation = list(filter(None, city_val_dict[b"nation_queue"].split(b",")))
+            if b"megacity_queue" in city_val_dict and city_val_dict[b"megacity_queue"]:
+                megacity = list(filter(None, city_val_dict[b"megacity_queue"].split(b",")))
+
+            return (local, nearby, nation, megacity, use_city_id)
+
+        except:
+            logging_exception()
+            logger.error("catch exception,err_log:%s" % traceback.format_exc())
+            qs = DiaryQueue.objects.filter(city_id__in=[city_id, 'world'])
+            # Assume that world queue must exist.
+            if len(qs) == 1:
+                obj = qs[0]
+            else:
+                obj = qs[0] if qs[0].city_id == city_id else qs[1]
+
+            if obj.native_queue:
+                local = list(filter(None, obj.native_queue.split(',')))
+            if obj.nearby_queue:
+                nearby = list(filter(None, obj.nearby_queue.split(',')))
+            if obj.nation_queue:
+                nation = list(filter(None, obj.nation_queue.split(',')))
+            if obj.megacity_queue:
+                megacity = list(filter(None, obj.megacity_queue.split(',')))
+
+            use_city_id = obj.city_id if obj else use_city_id
+            return (local, nearby, nation, megacity, use_city_id)
+
+    @classmethod
+    def fetch_device_diary_queue_data(cls, city_id, device_id):
+        local = list()
+        nearby = list()
+        nation = list()
+        megacity = list()
+        use_city_id = city_id
+        try:
+            gm_kv_ins = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gm_kv_ins = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"], redis_db=gm_kv_host_item["db"],redis_password=gm_kv_host_item["password"])
+                if gm_kv_ins:
+                    break
+
+            specify_city_id_key = "device_diary_queue:device_id:" + device_id + ":city_id:" + use_city_id
+            city_val_dict = gm_kv_ins.hgetall(specify_city_id_key)
+
+            if b"native_queue" in city_val_dict and city_val_dict[b"native_queue"]:
+                local = list(filter(None, city_val_dict[b"native_queue"].split(b",")))
+
+            if b"nearby_queue" in city_val_dict and city_val_dict[b"nearby_queue"]:
+                nearby = list(filter(None, city_val_dict[b"nearby_queue"].split(b",")))
+            if b"nation_queue" in city_val_dict and city_val_dict[b"nation_queue"]:
+                nation = list(filter(None, city_val_dict[b"nation_queue"].split(b",")))
+            if b"megacity_queue" in city_val_dict and city_val_dict[b"megacity_queue"]:
+                megacity = list(filter(None, city_val_dict[b"megacity_queue"].split(b",")))
+
+            return (local, nearby, nation, megacity, use_city_id)
+        except:
+            logging_exception()
+            logger.error("catch exception,err_log:%s" % traceback.format_exc())
+            obj = DeviceDiaryQueue.objects.filter(device_id=device_id, city_id=city_id).first()
+            if obj and obj.native_queue:
+                local = list(filter(None, obj.native_queue.split(',')))
+            if obj and obj.nearby_queue:
+                nearby = list(filter(None, obj.nearby_queue.split(',')))
+            if obj and obj.nation_queue:
+                nation = list(filter(None, obj.nation_queue.split(',')))
+            if obj and obj.megacity_queue:
+                megacity = list(filter(None, obj.megacity_queue.split(',')))
+
+            use_city_id = obj.city_id if obj else use_city_id
+            return (local, nearby, nation, megacity, use_city_id)
+
+    @classmethod
+    def fetch_diary(cls, device_id, card_type, city_id, size):
+        try:
+            def read_history(cid_list):
+                if redis_client.exists(today_key):
+                    redis_client.sadd(today_key, *cid_list)
+                else:
+                    redis_client.sadd(today_key, *cid_list)
+                    redis_client.expire(today_key, 15 * 24 * 60 * 60)
+                if redis_client.exists(read_key) and redis_client.exists(old_key):
+                    redis_client.sdiffstore(read_key, read_key, old_key)
+                    redis_client.delete(old_key)
+                    redis_client.expire(read_key, time=13 * 24 * 60 * 60)
+
+                redis_client.sadd(read_key, *cid_list)
+
+            def dislike_cid_filter(device_id, cid_list):
+                try:
+                    key = str(device_id) + "_dislike_diary"
+                    if gmkv.exists(key):
+                        dislike = gmkv.smembers(key)
+                        if len(cid_list) > 0:
+                            if type(cid_list[0]) == int or type(cid_list[0]) == str:
+                                cid_list = [i for i in cid_list if str(i).encode('utf-8') not in dislike]
+                            else:
+                                cid_list = [i for i in cid_list if i not in dislike]
+                    return cid_list
+                except:
+                    return cid_list
+
+            def fetch_after_filter_queue(device_id, city_id, name):
+                local = list()
+                try:
+                    key = "device_diary_queue_after_filter:device_id:" + device_id + ":city_id:" + city_id + name
+                    if gmkv.exists(key):
+                        return json.loads(gmkv.get(key))
+                    else:
+                        return local
+                except:
+
+                    return local
+
+            def write_after_filter_queue(device_id, city_id, cid_list, name):
+                try:
+                    key = "device_diary_queue_after_filter:device_id:" + device_id + ":city_id:" + city_id + name
+                    if gmkv.exists(key):
+                        gmkv.set(key, json.dumps(cid_list))
+                    else:
+                        gmkv.set(key, json.dumps(cid_list), ex=6 * 60 * 60)
+
+                except:
+                    logging_exception()
+                    logger.error("catch exception,err_log:%s" % traceback.format_exc())
+
+            def get_data(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+                nx = int(round(x * 1.0 / (x + y + z + m) * size))
+                ny = int(round(y * 1.0 / (x + y + z + m) * size))
+                nz = int(round(z * 1.0 / (x + y + z + m) * size))
+                nm = int(round(m * 1.0 / (x + y + z + m) * size))
+                nxyz = [nx, ny, nm, nz]
+                xyz = [x, y, m, z]
+                counter = Counter([nx, ny, nm, nz])
+                if counter[0] == 2:
+                    nxyz[nxyz.index(0)] += size - sum(nxyz)
+                else:
+                    nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+                nx, ny, nm, nz = nxyz
+
+                local_filter = dislike_cid_filter(device_id, cx)
+                if len(local_filter) == 0:
+                    local_filter = dislike_cid_filter(device_id, local)
+                slocal = local_filter[:nx]
+                have_x = local_filter[nx:]
+                x_list = list(map(int, have_x))
+                write_after_filter_queue(device_id, city_id, x_list, "native")
+
+                ny += (nx - len(slocal))
+
+                nearby_filter = dislike_cid_filter(device_id, cy)
+                if len(nearby_filter) == 0:
+                    nearby_filter = dislike_cid_filter(device_id, nearby)
+                snearby = nearby_filter[:ny]
+                have_y = nearby_filter[ny:]
+                y_list = list(map(int, have_y))
+                write_after_filter_queue(device_id, city_id, y_list, "nearby")
+
+                nm += (ny - len(snearby))
+
+                megacity_filter = dislike_cid_filter(device_id, cm)
+                if len(megacity_filter) == 0:
+                    megacity_filter = dislike_cid_filter(device_id, megacity)
+                smegacity = megacity_filter[:nm]
+                have_m = megacity_filter[nm:]
+                m_list = list(map(int, have_m))
+                write_after_filter_queue(device_id, city_id, m_list, "megacity")
+
+                nz += (nm - len(smegacity))
+
+                nation_filter = dislike_cid_filter(device_id, cz)
+                if len(nation_filter) == 0:
+                    nation_filter = dislike_cid_filter(device_id, nation)
+                snation = nation_filter[:nz]
+                have_z = snation[nz:]
+                z_list = list(map(int, have_z))
+                write_after_filter_queue(device_id, city_id, z_list, "nation")
+
+                return chain(slocal, snearby, smegacity, snation)
+
+            if device_id != '0':
+                portrait_list = list()
+                click_diary_size = 1
+                search_diary_size = 4
+                read_key = "TS:recommend_diary_set:device_id:" + str(device_id)
+                old_key = "TS:recommend_diary_set:device_id:{}:{}" \
+                    .format(device_id, (datetime.date.today() - datetime.timedelta(days=14)).strftime("%Y-%m-%d"))
+                today_key = "TS:recommend_diary_set:device_id:{}:{}" \
+                    .format(device_id, datetime.date.today().strftime("%Y-%m-%d"))
+                user_portrait_diary_key = 'user_portrait_recommend_diary_queue:device_id:%s:%s' % \
+                                          (device_id, datetime.datetime.now().strftime('%Y-%m-%d'))
+                gmkv = None
+                for gm_kv_host_item in settings.GM_KV_HOSTS:
+                    gmkv = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"],
+                                             redis_db=gm_kv_host_item["db"],
+                                             redis_password=gm_kv_host_item["password"])
+                    if gmkv:
+                        break
+
+                if redis_client.exists(user_portrait_diary_key):
+                    user_portrait_diary_dict = redis_client.hgetall(user_portrait_diary_key)
+                    user_portrait_cursor = str(user_portrait_diary_dict[b'cursor'], encoding='utf-8')
+                    if user_portrait_cursor == '0':
+                        if b'len_cursor' in user_portrait_diary_dict.keys():
+                            user_portrait_diary_list = json.loads(user_portrait_diary_dict[b'diary_queue'])
+                            filter_user_portrait_diary_list = dislike_cid_filter(device_id, user_portrait_diary_list)
+                            if len(filter_user_portrait_diary_list) > size:
+                                portrait_list = filter_user_portrait_diary_list[:size]
+                                redis_client.hset(user_portrait_diary_key, 'diary_queue',
+                                                  json.dumps(filter_user_portrait_diary_list[size:]))
+                                portrait_list = list(map(int, portrait_list))
+                                read_history(portrait_list)
+                                return portrait_list
+                            else:
+                                size = size - len(filter_user_portrait_diary_list)
+                                portrait_list = filter_user_portrait_diary_list
+                                redis_client.delete(user_portrait_diary_key)
+
+                search_diary_recommend_key = "TS:search_recommend_diary_queue:device_id:" + str(device_id)
+                search_list = list()
+
+                if redis_client.exists(search_diary_recommend_key) and size > 3:
+                    search_diary_recommend_dict = redis_client.hgetall(search_diary_recommend_key)
+                    search_diary_recommend_list = json.loads(search_diary_recommend_dict[b'diary_queue'])
+                    search_diary_recommend_list = dislike_cid_filter(device_id, search_diary_recommend_list)
+                    if len(search_diary_recommend_list) == 0:
+                        redis_client.delete(search_diary_recommend_key)
+                    elif len(search_diary_recommend_list) <= search_diary_size:
+                        search_list = search_diary_recommend_list
+                        size = size - len(search_diary_recommend_list)
+                        redis_client.delete(search_diary_recommend_key)
+
+                    else:
+                        search_list = search_diary_recommend_list[:search_diary_size]
+                        size = size - search_diary_size
+                        redis_client.hset(search_diary_recommend_key, 'diary_queue',
+                                          json.dumps(search_diary_recommend_list[search_diary_size:]))
+                    if size <= 0:
+                        portrait_list.extend(search_list)
+                        portrait_list = list(map(int, portrait_list))
+                        read_history(portrait_list)
+                        return portrait_list
+
+                diary_recommend_key = "TS:recommend_diary_queue:device_id:" + str(device_id)
+                ts_recommend_list = list()
+                if redis_client.exists(diary_recommend_key) and size > 0:
+                    diary_recommend_dict = redis_client.hgetall(diary_recommend_key)
+                    diary_recommend_list = json.loads(diary_recommend_dict[b'diary_queue'])
+                    diary_recommend_list = dislike_cid_filter(device_id, diary_recommend_list)
+                    if len(diary_recommend_list) == 0:
+                        redis_client.delete(diary_recommend_key)
+
+                    elif len(diary_recommend_list) <= click_diary_size:
+                        ts_recommend_list = diary_recommend_list
+                        redis_client.delete(diary_recommend_key)
+                        size = size - len(ts_recommend_list)
+
+                    else:
+                        size = size - click_diary_size
+                        ts_recommend_list = diary_recommend_list[:click_diary_size]
+                        diary_recommend_list_json = json.dumps(diary_recommend_list[click_diary_size:])
+                        redis_client.hset(diary_recommend_key, 'diary_queue', diary_recommend_list_json)
+                    if size <= 0:
+                        portrait_list.extend(search_list)
+                        portrait_list.extend(ts_recommend_list)
+                        portrait_list = list(map(int, portrait_list))
+                        read_history(portrait_list)
+                        return portrait_list
+
+                if size > 0:
+                    try:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_device_diary_queue_data(city_id,
+                                                                                                       device_id)
+                        if len(local) == 0 and len(nearby) == 0 and len(nation) == 0 and len(megacity) == 0:
+                            (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+
+                    except:
+                        logging_exception()
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                    x, y, m, z = cls.get_city_scale(city_id)
+                    cx = fetch_after_filter_queue(device_id, city_id, "native")
+                    cy = fetch_after_filter_queue(device_id, city_id, "nearby")
+                    cz = fetch_after_filter_queue(device_id, city_id, "nation")
+                    cm = fetch_after_filter_queue(device_id, city_id, "megacity")
+                    data = get_data(
+                        local, nearby, nation, megacity,
+                        cx, cy, cm, cz,
+                        x, y, z, m, size)
+
+                    portrait_list.extend(search_list)
+                    portrait_list.extend(ts_recommend_list)
+                    portrait_list.extend(data)
+                    if len(portrait_list) == 0:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                        portrait_list = cls.get_queue(local, nearby, nation, megacity,
+                                                      device_id, city_id, size, x, y, z, m)
+                    portrait_list = list(map(int, portrait_list))
+                    if len(portrait_list) != 0:
+                        read_history(portrait_list)
+                    return portrait_list
+            else:
+
+                try:
+                    (local, nearby, nation, megacity, city_id) = cls.fetch_device_diary_queue_data(city_id, device_id)
+                    if len(local) == 0 and len(nearby) == 0 and len(nation) == 0 and len(megacity) == 0:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                except:
+                    logging_exception()
+                    (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+
+                key = '{device_id}-{city_id}-{date}'.format(device_id=device_id,
+                                                            city_id=city_id, date=RecommendFeed.current_date())
+                # strategy rule: when user refresh over 30 loadings, reset native nearby nation queue cursor.
+                counter_key = key + '-counter_v1'
+                counter = redis_client.incr(counter_key)
+                if counter == 1:
+                    redis_client.expire(counter_key, 24 * 60 * 60)
+                cursor_key = key + '-cursor_v1'
+                cursor = redis_client.get(cursor_key) or b'0-0-0-0'
+                # if counter > 30:
+                #     cursor = b'0-0-0-0'
+                #     redis_client.delete(counter_key)
+                cx, cy, cm, cz = map(int, cursor.split(b'-'))
+
+                x, y, m, z = cls.get_city_scale(city_id)
+                data, ncx, ncy, ncm, ncz = cls.get_scale_data(
+                    local, nearby, nation, megacity,
+                    cx, cy, cm, cz,
+                    x, y, z, m, size
+                )
+
+                if ncx == cx and ncy == cy:  # native queue and nearby queue
+                    logger.info("diary queue reach end,cx:%d,cy:%d,cm:%d,cz:%d", cx, cy, cm, cz)
+                    # redis_client.delete(counter_key)
+                    # data, ncx, ncy, ncm, ncz = cls.get_scale_data(
+                    #     local, nearby, nation, megacity,
+                    #     0, 0, 0, 0,
+                    #     x, y, z, m, size
+                    # )
+                    ncx = ncy = ncm = ncz = 0
+
+                val = '-'.join(map(str, [ncx, ncy, ncm, ncz]))
+                redis_client.set(cursor_key, val, ex=24 * 60 * 60)
+                data = list(map(int, data))
+
+                return data
+        except:
+            logging_exception()
+            return []
+
+    @classmethod
+    def get_queue(cls,local, nearby, nation, megacity, device_id,city_id,size,x,y,z,m):
+        key = '{device_id}-{city_id}-{date}'.format(device_id=device_id,
+                                                    city_id=city_id, date=RecommendFeed.current_date())
+        counter_key = key + '-counter_v1'
+        counter = redis_client.incr(counter_key)
+        if counter == 1:
+            redis_client.expire(counter_key, 24 * 60 * 60)
+        cursor_key = key + '-cursor_v1'
+        cursor = redis_client.get(cursor_key) or b'0-0-0-0'
+        cx, cy, cm, cz = map(int, cursor.split(b'-'))
+
+        def get_scale(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+            nx = int(round(x * 1.0 / (x + y + z + m) * size))
+            ny = int(round(y * 1.0 / (x + y + z + m) * size))
+            nz = int(round(z * 1.0 / (x + y + z + m) * size))
+            nm = int(round(m * 1.0 / (x + y + z + m) * size))
+            nxyz = [nx, ny, nm, nz]
+            xyz = [x, y, m, z]
+            counter = Counter([nx, ny, nm, nz])
+            if counter[0] == 2:
+                nxyz[nxyz.index(0)] += size - sum(nxyz)
+            else:
+                nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+            nx, ny, nm, nz = nxyz
+
+            slocal = local[cx:cx + nx]
+            cx = min(cx + nx, len(local))
+            ny += (nx - len(slocal))
+
+            snearby = nearby[cy:cy + ny]
+            cy = min(cy + ny, len(nearby))
+            nm += (ny - len(snearby))
+
+            smegacity = megacity[cm: cm + nm]
+            cm = min(cm + nm, len(megacity))
+            nz += (nm - len(smegacity))
+
+            snation = nation[cz:cz + nz]
+            cz = min(cz + nz, len(nation))
+            return chain(slocal, snearby, smegacity, snation), cx, cy, cm, cz
+
+        data, ncx, ncy, ncm, ncz = get_scale(
+            local, nearby, nation, megacity,
+            cx, cy, cm, cz,
+            x, y, z, m, size)
+
+        if ncx == cx and ncy == cy:  # native queue and nearby queue
+            logger.info("diary queue reach end,cx:%d,cy:%d,cm:%d,cz:%d", cx, cy, cm, cz)
+
+            ncx = ncy = ncm = ncz = 0
+
+        val = '-'.join(map(str, [ncx, ncy, ncm, ncz]))
+        redis_client.set(cursor_key, val, ex=24 * 60 * 60)
+        return list(map(int, data))
+
+    @staticmethod
+    def get_city_scale(city_id):
+        try:
+            c = CityScale.objects.get(city_id=city_id)
+            x, y, z, m = c.native, c.nearby, c.nation, c.megacity
+        except CityScale.DoesNotExist:
+            try:
+                c = City.objects.get(id=city_id)
+                if c.level in (CITY_LEVEL.SUPER, CITY_LEVEL.ONE):
+                    x, y, m, z = 4, 3, 0, 3
+                elif c.level == CITY_LEVEL.TWO:
+                    x, y, m, z = 3, 3, 0, 3
+                elif c.level == CITY_LEVEL.THREE:
+                    x, y, m, z = 1, 4, 0, 5
+                else:
+                    x, y, m, z = 0, 0, 0, 10
+            except City.DoesNotExist:
+                x, y, m, z = 0, 0, 0, 10
+
+        return x, y, m, z
+
+    @staticmethod
+    def get_scale_data(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+        """
+
+        :param local:           local diary queue
+        :param nearby:          nearby diary queue
+        :param nation:          nation diary queue
+        :param megacity:        megacity diary queue
+        :param cx:              seen local diary offset
+        :param cy:              seen nearby diary offset
+        :param cz:              seen nation diary offset
+        :param cm:              seen megacity diary offset
+        :param x:               local diary scale factor
+        :param y:               nearby diary scale factor
+        :param z:               nation diary scale factor
+        :param m:               megacity diary scale factor
+        :param size:            nubmer of diary
+        :return:
+        """
+        # 本地 临近 特大城市 全国 四个层级 都按照的是四舍五入取得方式
+        # 针对出现的问题，本次相应的优化是：
+        # 1、如果出现两个层级为零，且有剩余坑位时，则按照本地 临近 全国的优先级，先给优先级高且为零的层级一个坑位。
+        # 2、如果所有层级都非零，且有剩余坑位时，则优先给权重占比大的层级一个坑位。
+        # 3、如果只有一个层级为零，且有剩余坑位时，则优先填充权重占比大的层级一个坑位。
+        nx = int(round(x * 1.0 / (x + y + z + m) * size))
+        ny = int(round(y * 1.0 / (x + y + z + m) * size))
+        nz = int(round(z * 1.0 / (x + y + z + m) * size))
+        nm = int(round(m * 1.0 / (x + y + z + m) * size))
+        nxyz = [nx, ny, nm, nz]
+        xyz = [x, y, m, z]
+        counter = Counter([nx, ny, nm, nz])
+        if counter[0] == 2:
+            nxyz[nxyz.index(0)] += size - sum(nxyz)
+        else:
+            nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+        nx, ny, nm, nz = nxyz
+
+        slocal = local[cx:cx + nx]
+        cx = min(cx + nx, len(local))
+        ny += (nx - len(slocal))
+
+        snearby = nearby[cy:cy + ny]
+        cy = min(cy + ny, len(nearby))
+        nm += (ny - len(snearby))
+
+        smegacity = megacity[cm: cm + nm]
+        cm = min(cm + nm, len(megacity))
+        nz += (nm - len(smegacity))
+
+        snation = nation[cz:cz + nz]
+        cz = min(cz + nz, len(nation))
+
+        return chain(slocal, snearby, smegacity, snation), cx, cy, cm, cz
+
+
+@bind('doris/recommend/feed_rerank')
+def recommend_feed_rerank(device_id, card_type, city_id, size):
+    try:
+        return RecommendFeedRerank.dispatch(device_id, card_type,
+                                      city_id, size)
+    except:
+        logging_exception()
+        return {card_type: []}
+
+
+class RecommendFeedRerank:
+    @classmethod
+    def dispatch(cls, device_id, card_type, city_id, size):
+        data = []
+        if card_type == CARD_TYPE.QA:
+            data = cls.fetch_qa(device_id, card_type, size)
+        elif card_type == CARD_TYPE.ANSWER:
+            data = cls.fetch_answer(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.ARTICLE:
+            data = cls.fetch_article(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.QUESTION:
+            data = cls.fetch_question(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.DIARY:
+            data = cls.fetch_diary(device_id, card_type, city_id, size)
+
+        elif card_type == CARD_TYPE.USERTOPIC:
+            data = cls.fetch_user_topic(device_id,card_type,size)
+        elif card_type == CARD_TYPE.DOCTORTOPIC:
+            data = cls.fetch_doctor_topic(device_id,card_type,size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.ENCYCLOPEDIA:
+            data = cls.fetch_wiki(device_id,card_type,size)
+        return {card_type: data}
+
+    @staticmethod
+    def current_date():
+        return datetime.datetime.now().strftime('%Y-%m-%d')
+
+    @staticmethod
+    def fetch_question(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceQuestionQueue.objects.get(device_id=device_id)
+        except DeviceQuestionQueue.DoesNotExist:
+            que = QuestionQueue.objects.last()
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_icon(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+            que = IconQueue.objects.last()
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_wiki(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+
+            que = WikiQueue.objects.last()
+            if not que:
+                return []
+            # que = list(filter(None, que.queue.split(',')))
+            que = json.loads(que.queue)
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_answer(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                          card_type=card_type, date=RecommendFeed.current_date())
+            try:
+                que = DeviceAnswerQueue.objects.get(device_id=device_id)
+            except DeviceAnswerQueue.DoesNotExist:
+                que = AnswerQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_qa(device_id, card_type, size):
+        try:
+            def get_after_filter_qa():
+                try:
+                    return json.loads(gmkv.get(after_filter_key))
+                except:
+                    return []
+
+            def write_after_filter_qa(cid_list):
+                try:
+                    if gmkv.exists(after_filter_key):
+                        gmkv.set(after_filter_key, json.dumps(cid_list))
+                    else:
+                        gmkv.set(after_filter_key, json.dumps(cid_list),ex = 6*60*60)
+
+                except:
+                    logging_exception()
+                    logger.error("catch exception,err_log:%s" % traceback.format_exc())
+
+            def filter_qa(device_id,cid_list):
+                try:
+                    key = str(device_id) + "_dislike_qa"
+                    if gmkv.exists(key):
+                        dislike = gmkv.smembers(key)
+                        if len(cid_list) > 0:
+                            if type(cid_list[0]) == int or type(cid_list[0]) == str:
+                                cid_list = [i for i in cid_list if str(i).encode('utf-8') not in dislike]
+                            else:
+                                cid_list = [i for i in cid_list if i not in dislike]
+                        return cid_list
+                    else:
+                        return cid_list
+                except:
+                    return cid_list
+
+            def read_history(cid_list):
+                if redis_client.exists(today_qa_key):
+                    redis_client.sadd(today_qa_key, *cid_list)
+                else:
+                    redis_client.sadd(today_qa_key, *cid_list)
+                    redis_client.expire(today_qa_key, 15 * 24 * 60 * 60)
+                if redis_client.exists(read_qa_key) and redis_client.exists(old_qa_key):
+                    redis_client.sdiffstore(read_qa_key, read_qa_key, old_qa_key)
+                    redis_client.delete(old_qa_key)
+                    redis_client.expire(read_qa_key, time=13 * 24 * 60 * 60)
+
+                redis_client.sadd(read_qa_key, *cid_list)
+
+            def get_gmkv(redis_ip, redis_port, redis_db, redis_password=""):
+                try:
+                    if len(redis_password) == 0:
+                        cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, socket_timeout=2)
+                    else:
+                        cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, password=redis_password,
+                                              socket_timeout=2)
+                    cli_ins.ping()
+
+                    return cli_ins
+                except:
+                    return None
+
+            search_qa_recommend_list = list()
+            read_qa_key = "TS:recommend_answer_set:device_id:" + str(device_id)
+            old_qa_key = "TS:recommend_answer_set:device_id:{}:{}"\
+                .format(device_id,(datetime.date.today() - datetime.timedelta(days=14)).strftime("%Y-%m-%d"))
+            today_qa_key = "TS:recommend_answer_set:device_id:{}:{}"\
+                .format(device_id, datetime.date.today().strftime("%Y-%m-%d"))
+            answer_queue_key = "qa_is_tail:" + str(device_id)
+            after_filter_key = "device_qa_after_filter:device_id:" + str(device_id)
+            gmkv = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gmkv = get_gmkv(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"],
+                                         redis_db=gm_kv_host_item["db"],
+                                         redis_password=gm_kv_host_item["password"])
+                if gmkv:
+                    break
+
+            if device_id != '0':
+                search_qa_recommend_key = "TS:search_recommend_answer_queue:device_id:" + str(device_id)
+                if redis_client.exists(search_qa_recommend_key):
+                    search_qa_recommend_dict = redis_client.hgetall(search_qa_recommend_key)
+                    queue_list = json.loads(search_qa_recommend_dict[b'answer_queue'])
+                    queue_list = filter_qa(device_id, queue_list)
+                    if len(queue_list) == 0:
+                        redis_client.delete(search_qa_recommend_key)
+                    elif len(queue_list) == 1:
+                        size = size - 1
+                        search_qa_recommend_list = queue_list
+                        redis_client.delete(search_qa_recommend_key)
+                    else:
+                        size = size - 1
+                        search_qa_recommend_list.append(queue_list[0])
+                        redis_client.hset(search_qa_recommend_key,"answer_queue",json.dumps(queue_list[1:]))
+
+                if gmkv.exists(answer_queue_key):
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                elif gmkv.exists(after_filter_key):
+                    que = get_after_filter_qa()
+                    que = filter_qa(device_id,que)
+                    if len(que) == 0:
+                        gmkv.set(answer_queue_key,"tail",ex = 6*60*60)
+                        if len(search_qa_recommend_list) > 0:
+                            search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                            read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+                    elif len(que) <= size:
+                        search_qa_recommend_list.extend(que)
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+                    else:
+                        search_qa_recommend_list.extend(que[:size])
+                        write_after_filter_qa(que[size:])
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+
+                try:
+                    que = DeviceQAQueue.objects.get(device_id=device_id)
+                except DeviceQAQueue.DoesNotExist:
+                    que = AnswerQueue.objects.last()
+                if not que:
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                qa = list(filter(None, que.queue.split(',')))
+                if device_id != "0":
+                    qa = filter_qa(device_id,qa)
+                if len(qa) == 0:
+                    if device_id != "0":
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                elif len(qa) <= size:
+                    search_qa_recommend_list.extend(qa)
+                    search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                    if device_id != "0":
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                else:
+                    search_qa_recommend_list.extend(qa[:size])
+                    search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                    if device_id != "0":
+                        write_after_filter_qa(qa[size:])
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+
+            else:
+                key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                              card_type=card_type, date=RecommendFeed.current_date())
+                try:
+                    que = DeviceQAQueue.objects.get(device_id=device_id)
+                except DeviceQAQueue.DoesNotExist:
+                    que = AnswerQueue.objects.last()
+                if not que:
+                    return []
+                que = list(filter(None, que.queue.split(',')))
+
+                # adjust args.
+                cursor = redis_client.get(key) or 0
+                cursor = int(cursor) % len(que)
+                size = min(size, len(que))
+                # redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                data = list(islice(cycle(que), cursor, cursor + size))
+                data = list(map(int, data))
+                if cursor + 2 * size < len(que):
+                    redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                else:
+                    try:
+                        context.request_logger.app(reset_answer_queue=True)
+                        cursor = 0
+                        redis_client.set(key, cursor, ex=24 * 60 * 60)
+                    except:
+                        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                return data
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_article(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceArticleQueue.objects.get(device_id=device_id)
+        except DeviceArticleQueue.DoesNotExist:
+            que = ArticleQueue.objects.last()
+
+        if not que:
+            return []
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_user_topic(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id, card_type=card_type,
+                                                          date=RecommendFeed.current_date())
+
+            if (device_id != '0') and size >= 2:
+
+                search_topic_recommend_key = "TS:search_recommend_tractate_queue:device_id:" + str(device_id)
+                search_topic_recommend_list = list()
+                search_cursor_ts = 0
+                if redis_client.exists(search_topic_recommend_key):
+                    search_topic_recommend_dict = redis_client.hgetall(search_topic_recommend_key)
+                    if b'cursor' in search_topic_recommend_dict:
+                        search_cursor_ts = json.loads(search_topic_recommend_dict[b'cursor'])
+                        if search_cursor_ts < 30:
+                            search_topic_recommend_list = json.loads(search_topic_recommend_dict[b'tractate_queue'])
+                            if search_cursor_ts < len(search_topic_recommend_list):
+                                size = size - 2
+
+            try:
+                que = DeviceUserTopicQueue.objects.get(device_id=device_id)
+            except DeviceUserTopicQueue.DoesNotExist:
+                que = UserTopicQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            data = list(islice(cycle(que), cursor, cursor + size))
+            data = list(map(int, data))
+
+            if cursor + 2 * size < len(que):
+                redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            else:
+                try:
+                    context.request_logger.app(reset_queue=True)
+                    cursor = 0
+                    redis_client.set(key, cursor, ex=24 * 60 * 60)
+                except:
+                    redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+
+            if device_id != '0' and size >= 2:
+                if len(search_topic_recommend_list) > 0 and search_cursor_ts < len(search_topic_recommend_list):
+                    queue = search_topic_recommend_list[search_cursor_ts:search_cursor_ts + 2]
+                    queue.extend(data)
+                    data = queue
+                    new_search_cursor = search_cursor_ts + 2
+                    redis_client.hset(search_topic_recommend_key, 'cursor', new_search_cursor)
+                    redis_client.expire(search_topic_recommend_key, 30 * 24 * 60 * 60)
+                read_topic_key = "TS:recommend_tractate_set:device_id:" + str(device_id)
+                if len(data) > 0:
+                    redis_client.sadd(read_topic_key, *data)
+            return data
+
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_doctor_topic(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                          card_type=card_type, date=RecommendFeed.current_date())
+            try:
+                que = DeviceDoctorTopicQueue.objects.get(device_id=device_id)
+            except DeviceDoctorTopicQueue.DoesNotExist:
+                que = DoctorTopicQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+        except:
+            logging_exception()
+            return []
+
+
+    @classmethod
+    def get_gm_kv_ins(cls,redis_ip, redis_port, redis_db, redis_password=""):
+        try:
+            if len(redis_password) == 0:
+                cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, socket_timeout=2)
+            else:
+                cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, password=redis_password,
+                                      socket_timeout=2)
+            cli_ins.ping()
+
+            return cli_ins
+        except:
+            return None
+
+    @classmethod
+    def fetch_diary_queue_data(cls, city_id, device_id=None):
+        local = list()
+        nearby = list()
+        nation = list()
+        megacity = list()
+        use_city_id = city_id
+        try:
+            gm_kv_ins = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gm_kv_ins = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"], redis_db=gm_kv_host_item["db"],redis_password=gm_kv_host_item["password"])
+                if gm_kv_ins:
+                    break
+
+            specify_city_id_key = "diary_queue:city_id:" + use_city_id
+            world_city_id_key = "diary_queue:city_id:world"
+            if device_id is not None:
+                specify_city_id_key = "device_diary_queue_rerank:device_id:" + device_id + ":city_id:" + use_city_id
+
+            city_val_dict = gm_kv_ins.hgetall(specify_city_id_key)
+            if len(city_val_dict) == 0:
+                city_val_dict = gm_kv_ins.hgetall(world_city_id_key)
+                use_city_id = "world"
+
+            if b"native_queue" in city_val_dict and city_val_dict[b"native_queue"]:
+                local = list(filter(None, city_val_dict[b"native_queue"].split(b",")))
+            if b"nearby_queue" in city_val_dict and city_val_dict[b"nearby_queue"]:
+                nearby = list(filter(None, city_val_dict[b"nearby_queue"].split(b",")))
+            if b"nation_queue" in city_val_dict and city_val_dict[b"nation_queue"]:
+                nation = list(filter(None, city_val_dict[b"nation_queue"].split(b",")))
+            if b"megacity_queue" in city_val_dict and city_val_dict[b"megacity_queue"]:
+                megacity = list(filter(None, city_val_dict[b"megacity_queue"].split(b",")))
+
+            return (local, nearby, nation, megacity, use_city_id)
+
+        except:
+            logging_exception()
+            logger.error("catch exception,err_log:%s" % traceback.format_exc())
+            qs = DiaryQueue.objects.filter(city_id__in=[city_id, 'world'])
+            # Assume that world queue must exist.
+            if len(qs) == 1:
+                obj = qs[0]
+            else:
+                obj = qs[0] if qs[0].city_id == city_id else qs[1]
+
+            if obj.native_queue:
+                local = list(filter(None, obj.native_queue.split(',')))
+            if obj.nearby_queue:
+                nearby = list(filter(None, obj.nearby_queue.split(',')))
+            if obj.nation_queue:
+                nation = list(filter(None, obj.nation_queue.split(',')))
+            if obj.megacity_queue:
+                megacity = list(filter(None, obj.megacity_queue.split(',')))
+
+            use_city_id = obj.city_id if obj else use_city_id
+            return (local, nearby, nation, megacity, use_city_id)
+
+    @classmethod
+    def fetch_device_diary_queue_data(cls, city_id, device_id):
+        local = list()
+        nearby = list()
+        nation = list()
+        megacity = list()
+        use_city_id = city_id
+        try:
+            gm_kv_ins = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gm_kv_ins = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"], redis_db=gm_kv_host_item["db"],redis_password=gm_kv_host_item["password"])
+                if gm_kv_ins:
+                    break
+
+            specify_city_id_key = "device_diary_queue_rerank:device_id:" + device_id + ":city_id:" + use_city_id
+            city_val_dict = gm_kv_ins.hgetall(specify_city_id_key)
+
+            if b"native_queue" in city_val_dict and city_val_dict[b"native_queue"]:
+                local = list(filter(None, city_val_dict[b"native_queue"].split(b",")))
+            if b"nearby_queue" in city_val_dict and city_val_dict[b"nearby_queue"]:
+                nearby = list(filter(None, city_val_dict[b"nearby_queue"].split(b",")))
+            if b"nation_queue" in city_val_dict and city_val_dict[b"nation_queue"]:
+                nation = list(filter(None, city_val_dict[b"nation_queue"].split(b",")))
+            if b"megacity_queue" in city_val_dict and city_val_dict[b"megacity_queue"]:
+                megacity = list(filter(None, city_val_dict[b"megacity_queue"].split(b",")))
+
+            return (local, nearby, nation, megacity, use_city_id)
+        except:
+            logging_exception()
+            logger.error("catch exception,err_log:%s" % traceback.format_exc())
+            obj = DeviceDiaryQueue.objects.filter(device_id=device_id, city_id=city_id).first()
+            if obj and obj.native_queue:
+                local = list(filter(None, obj.native_queue.split(',')))
+            if obj and obj.nearby_queue:
+                nearby = list(filter(None, obj.nearby_queue.split(',')))
+            if obj and obj.nation_queue:
+                nation = list(filter(None, obj.nation_queue.split(',')))
+            if obj and obj.megacity_queue:
+                megacity = list(filter(None, obj.megacity_queue.split(',')))
+
+            use_city_id = obj.city_id if obj else use_city_id
+            return (local, nearby, nation, megacity, use_city_id)
+
+    @classmethod
+    def fetch_diary(cls, device_id, card_type, city_id, size):
+        try:
+            def read_history(cid_list):
+                if redis_client.exists(today_key):
+                    redis_client.sadd(today_key, *cid_list)
+                else:
+                    redis_client.sadd(today_key, *cid_list)
+                    redis_client.expire(today_key, 15 * 24 * 60 * 60)
+                if redis_client.exists(read_key) and redis_client.exists(old_key):
+                    redis_client.sdiffstore(read_key, read_key, old_key)
+                    redis_client.delete(old_key)
+                    redis_client.expire(read_key, time=13 * 24 * 60 * 60)
+
+                redis_client.sadd(read_key, *cid_list)
+
+            def dislike_cid_filter(device_id, cid_list):
+                try:
+                    key = str(device_id) + "_dislike_diary"
+                    if gmkv.exists(key):
+                        dislike = gmkv.smembers(key)
+                        if len(cid_list) > 0:
+                            if type(cid_list[0]) == int or type(cid_list[0]) == str:
+                                cid_list = [i for i in cid_list if str(i).encode('utf-8') not in dislike]
+                            else:
+                                cid_list = [i for i in cid_list if i not in dislike]
+                    return cid_list
+                except:
+                    return cid_list
+
+            def fetch_after_filter_queue(device_id, city_id, name):
+                local = list()
+                try:
+                    key = "device_diary_queue_after_filter:device_id:" + device_id + ":city_id:" + city_id + name
+                    if gmkv.exists(key):
+                        return json.loads(gmkv.get(key))
+                    else:
+                        return local
+                except:
+
+                    return local
+
+            def write_after_filter_queue(device_id, city_id, cid_list, name):
+                try:
+                    key = "device_diary_queue_after_filter:device_id:" + device_id + ":city_id:" + city_id + name
+                    if gmkv.exists(key):
+                        gmkv.set(key, json.dumps(cid_list))
+                    else:
+                        gmkv.set(key, json.dumps(cid_list), ex=6 * 60 * 60)
+
+                except:
+                    logging_exception()
+                    logger.error("catch exception,err_log:%s" % traceback.format_exc())
+
+            def get_data(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+                nx = int(round(x * 1.0 / (x + y + z + m) * size))
+                ny = int(round(y * 1.0 / (x + y + z + m) * size))
+                nz = int(round(z * 1.0 / (x + y + z + m) * size))
+                nm = int(round(m * 1.0 / (x + y + z + m) * size))
+                nxyz = [nx, ny, nm, nz]
+                xyz = [x, y, m, z]
+                counter = Counter([nx, ny, nm, nz])
+                if counter[0] == 2:
+                    nxyz[nxyz.index(0)] += size - sum(nxyz)
+                else:
+                    nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+                nx, ny, nm, nz = nxyz
+
+                local_filter = dislike_cid_filter(device_id, cx)
+                if len(local_filter) == 0:
+                    local_filter = dislike_cid_filter(device_id, local)
+                slocal = local_filter[:nx]
+                have_x = local_filter[nx:]
+                x_list = list(map(int, have_x))
+                write_after_filter_queue(device_id, city_id, x_list, "native")
+
+                ny += (nx - len(slocal))
+
+                nearby_filter = dislike_cid_filter(device_id, cy)
+                if len(nearby_filter) == 0:
+                    nearby_filter = dislike_cid_filter(device_id, nearby)
+                snearby = nearby_filter[:ny]
+                have_y = nearby_filter[ny:]
+                y_list = list(map(int, have_y))
+                write_after_filter_queue(device_id, city_id, y_list, "nearby")
+
+                nm += (ny - len(snearby))
+
+                megacity_filter = dislike_cid_filter(device_id, cm)
+                if len(megacity_filter) == 0:
+                    megacity_filter = dislike_cid_filter(device_id, megacity)
+                smegacity = megacity_filter[:nm]
+                have_m = megacity_filter[nm:]
+                m_list = list(map(int, have_m))
+                write_after_filter_queue(device_id, city_id, m_list, "megacity")
+
+                nz += (nm - len(smegacity))
+
+                nation_filter = dislike_cid_filter(device_id, cz)
+                if len(nation_filter) == 0:
+                    nation_filter = dislike_cid_filter(device_id, nation)
+                snation = nation_filter[:nz]
+                have_z = snation[nz:]
+                z_list = list(map(int, have_z))
+                write_after_filter_queue(device_id, city_id, z_list, "nation")
+
+                return chain(slocal, snearby, smegacity, snation)
+            if device_id != '0':
+                portrait_list = list()
+                click_diary_size = 1
+                search_diary_size = 4
+                read_key = "TS:recommend_diary_set:device_id:" + str(device_id)
+                old_key = "TS:recommend_diary_set:device_id:{}:{}" \
+                    .format(device_id, (datetime.date.today() - datetime.timedelta(days=14)).strftime("%Y-%m-%d"))
+                today_key = "TS:recommend_diary_set:device_id:{}:{}" \
+                    .format(device_id, datetime.date.today().strftime("%Y-%m-%d"))
+                user_portrait_diary_key = 'user_portrait_recommend_diary_queue:device_id:%s:%s' % \
+                                          (device_id, datetime.datetime.now().strftime('%Y-%m-%d'))
+                gmkv = None
+                for gm_kv_host_item in settings.GM_KV_HOSTS:
+                    gmkv = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"],
+                                                  redis_db=gm_kv_host_item["db"],
+                                                  redis_password=gm_kv_host_item["password"])
+                    if gmkv:
+                        break
+
+                if redis_client.exists(user_portrait_diary_key):
+                    user_portrait_diary_dict = redis_client.hgetall(user_portrait_diary_key)
+                    user_portrait_cursor = str(user_portrait_diary_dict[b'cursor'], encoding='utf-8')
+                    if user_portrait_cursor == '0':
+                        if b'len_cursor' in user_portrait_diary_dict.keys():
+                            user_portrait_diary_list = json.loads(user_portrait_diary_dict[b'diary_queue'])
+                            filter_user_portrait_diary_list = dislike_cid_filter(device_id, user_portrait_diary_list)
+                            if len(filter_user_portrait_diary_list) > size:
+                                portrait_list = filter_user_portrait_diary_list[:size]
+                                redis_client.hset(user_portrait_diary_key, 'diary_queue',
+                                                  json.dumps(filter_user_portrait_diary_list[size:]))
+                                portrait_list = list(map(int, portrait_list))
+                                read_history(portrait_list)
+                                return portrait_list
+                            else:
+                                size = size - len(filter_user_portrait_diary_list)
+                                portrait_list = filter_user_portrait_diary_list
+                                redis_client.delete(user_portrait_diary_key)
+
+                search_diary_recommend_key = "TS:search_recommend_diary_queue:device_id:" + str(device_id)
+                search_list = list()
+
+                if redis_client.exists(search_diary_recommend_key) and size > 3:
+                    search_diary_recommend_dict = redis_client.hgetall(search_diary_recommend_key)
+                    search_diary_recommend_list = json.loads(search_diary_recommend_dict[b'diary_queue'])
+                    search_diary_recommend_list = dislike_cid_filter(device_id, search_diary_recommend_list)
+                    if len(search_diary_recommend_list) == 0:
+                        redis_client.delete(search_diary_recommend_key)
+                    elif len(search_diary_recommend_list) <= search_diary_size:
+                        search_list = search_diary_recommend_list
+                        size = size - len(search_diary_recommend_list)
+                        redis_client.delete(search_diary_recommend_key)
+
+                    else:
+                        search_list = search_diary_recommend_list[:search_diary_size]
+                        size = size - search_diary_size
+                        redis_client.hset(search_diary_recommend_key, 'diary_queue',
+                                          json.dumps(search_diary_recommend_list[search_diary_size:]))
+                    if size <= 0:
+                        portrait_list.extend(search_list)
+                        portrait_list = list(map(int, portrait_list))
+                        read_history(portrait_list)
+                        return portrait_list
+
+                diary_recommend_key = "TS:recommend_diary_queue:device_id:" + str(device_id)
+                ts_recommend_list = list()
+                if redis_client.exists(diary_recommend_key) and size > 0:
+                    diary_recommend_dict = redis_client.hgetall(diary_recommend_key)
+                    diary_recommend_list = json.loads(diary_recommend_dict[b'diary_queue'])
+                    diary_recommend_list = dislike_cid_filter(device_id, diary_recommend_list)
+                    if len(diary_recommend_list) == 0:
+                        redis_client.delete(diary_recommend_key)
+
+                    elif len(diary_recommend_list) <= click_diary_size:
+                        ts_recommend_list = diary_recommend_list
+                        redis_client.delete(diary_recommend_key)
+                        size = size - len(ts_recommend_list)
+
+                    else:
+                        size = size - click_diary_size
+                        ts_recommend_list = diary_recommend_list[:click_diary_size]
+                        diary_recommend_list_json = json.dumps(diary_recommend_list[click_diary_size:])
+                        redis_client.hset(diary_recommend_key, 'diary_queue', diary_recommend_list_json)
+                    if size <= 0:
+                        portrait_list.extend(search_list)
+                        portrait_list.extend(ts_recommend_list)
+                        portrait_list = list(map(int, portrait_list))
+                        read_history(portrait_list)
+                        return portrait_list
+
+                if size > 0:
+                    try:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_device_diary_queue_data(city_id,
+                                                                                                       device_id)
+                        if len(local) == 0 and len(nearby) == 0 and len(nation) == 0 and len(megacity) == 0:
+                            (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+
+                    except:
+                        logging_exception()
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                    x, y, m, z = cls.get_city_scale(city_id)
+                    cx = fetch_after_filter_queue(device_id, city_id, "native")
+                    cy = fetch_after_filter_queue(device_id, city_id, "nearby")
+                    cz = fetch_after_filter_queue(device_id, city_id, "nation")
+                    cm = fetch_after_filter_queue(device_id, city_id, "megacity")
+                    data = get_data(
+                        local, nearby, nation, megacity,
+                        cx, cy, cm, cz,
+                        x, y, z, m, size)
+
+                    portrait_list.extend(search_list)
+                    portrait_list.extend(ts_recommend_list)
+                    portrait_list.extend(data)
+                    if len(portrait_list) == 0:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                        portrait_list = cls.get_queue(local, nearby, nation, megacity,
+                                                      device_id, city_id, size, x, y, z, m)
+                    portrait_list = list(map(int, portrait_list))
+                    if len(portrait_list) != 0:
+                        read_history(portrait_list)
+                    return portrait_list
+            else:
+
+                try:
+                    (local, nearby, nation, megacity, city_id) = cls.fetch_device_diary_queue_data(city_id, device_id)
+                    if len(local) == 0 and len(nearby) == 0 and len(nation) == 0 and len(megacity) == 0:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                except:
+                    logging_exception()
+                    (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+
+                key = '{device_id}-{city_id}-{date}'.format(device_id=device_id,
+                                                            city_id=city_id, date=RecommendFeed.current_date())
+                # strategy rule: when user refresh over 30 loadings, reset native nearby nation queue cursor.
+                counter_key = key + '-counter_v1'
+                counter = redis_client.incr(counter_key)
+                if counter == 1:
+                    redis_client.expire(counter_key, 24 * 60 * 60)
+                cursor_key = key + '-cursor_v1'
+                cursor = redis_client.get(cursor_key) or b'0-0-0-0'
+                # if counter > 30:
+                #     cursor = b'0-0-0-0'
+                #     redis_client.delete(counter_key)
+                cx, cy, cm, cz = map(int, cursor.split(b'-'))
+
+                x, y, m, z = cls.get_city_scale(city_id)
+                data, ncx, ncy, ncm, ncz = cls.get_scale_data(
+                    local, nearby, nation, megacity,
+                    cx, cy, cm, cz,
+                    x, y, z, m, size
+                )
+
+                if ncx == cx and ncy == cy:  # native queue and nearby queue
+                    logger.info("diary queue reach end,cx:%d,cy:%d,cm:%d,cz:%d", cx, cy, cm, cz)
+                    # redis_client.delete(counter_key)
+                    # data, ncx, ncy, ncm, ncz = cls.get_scale_data(
+                    #     local, nearby, nation, megacity,
+                    #     0, 0, 0, 0,
+                    #     x, y, z, m, size
+                    # )
+                    ncx = ncy = ncm = ncz = 0
+
+                val = '-'.join(map(str, [ncx, ncy, ncm, ncz]))
+                redis_client.set(cursor_key, val, ex=24 * 60 * 60)
+                data = list(map(int, data))
+
+                return data
+        except:
+            logging_exception()
+            return []
+
+
+    @classmethod
+    def get_queue(cls,local, nearby, nation, megacity, device_id,city_id,size,x,y,z,m):
+        key = '{device_id}-{city_id}-{date}'.format(device_id=device_id,
+                                                    city_id=city_id, date=RecommendFeed.current_date())
+        counter_key = key + '-counter_v1'
+        counter = redis_client.incr(counter_key)
+        if counter == 1:
+            redis_client.expire(counter_key, 24 * 60 * 60)
+        cursor_key = key + '-cursor_v1'
+        cursor = redis_client.get(cursor_key) or b'0-0-0-0'
+        cx, cy, cm, cz = map(int, cursor.split(b'-'))
+
+        def get_scale(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+            nx = int(round(x * 1.0 / (x + y + z + m) * size))
+            ny = int(round(y * 1.0 / (x + y + z + m) * size))
+            nz = int(round(z * 1.0 / (x + y + z + m) * size))
+            nm = int(round(m * 1.0 / (x + y + z + m) * size))
+            nxyz = [nx, ny, nm, nz]
+            xyz = [x, y, m, z]
+            counter = Counter([nx, ny, nm, nz])
+            if counter[0] == 2:
+                nxyz[nxyz.index(0)] += size - sum(nxyz)
+            else:
+                nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+            nx, ny, nm, nz = nxyz
+
+            slocal = local[cx:cx + nx]
+            cx = min(cx + nx, len(local))
+            ny += (nx - len(slocal))
+
+            snearby = nearby[cy:cy + ny]
+            cy = min(cy + ny, len(nearby))
+            nm += (ny - len(snearby))
+
+            smegacity = megacity[cm: cm + nm]
+            cm = min(cm + nm, len(megacity))
+            nz += (nm - len(smegacity))
+
+            snation = nation[cz:cz + nz]
+            cz = min(cz + nz, len(nation))
+            return chain(slocal, snearby, smegacity, snation), cx, cy, cm, cz
+
+        data, ncx, ncy, ncm, ncz = get_scale(
+            local, nearby, nation, megacity,
+            cx, cy, cm, cz,
+            x, y, z, m, size)
+
+        if ncx == cx and ncy == cy:  # native queue and nearby queue
+            logger.info("diary queue reach end,cx:%d,cy:%d,cm:%d,cz:%d", cx, cy, cm, cz)
+
+            ncx = ncy = ncm = ncz = 0
+
+        val = '-'.join(map(str, [ncx, ncy, ncm, ncz]))
+        redis_client.set(cursor_key, val, ex=24 * 60 * 60)
+        return list(map(int, data))
+
+    @staticmethod
+    def get_city_scale(city_id):
+        try:
+            c = CityScale.objects.get(city_id=city_id)
+            x, y, z, m = c.native, c.nearby, c.nation, c.megacity
+        except CityScale.DoesNotExist:
+            try:
+                c = City.objects.get(id=city_id)
+                if c.level in (CITY_LEVEL.SUPER, CITY_LEVEL.ONE):
+                    x, y, m, z = 4, 3, 0, 3
+                elif c.level == CITY_LEVEL.TWO:
+                    x, y, m, z = 3, 3, 0, 3
+                elif c.level == CITY_LEVEL.THREE:
+                    x, y, m, z = 1, 4, 0, 5
+                else:
+                    x, y, m, z = 0, 0, 0, 10
+            except City.DoesNotExist:
+                x, y, m, z = 0, 0, 0, 10
+
+        return x, y, m, z
+
+    @staticmethod
+    def get_scale_data(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+        """
+
+        :param local:           local diary queue
+        :param nearby:          nearby diary queue
+        :param nation:          nation diary queue
+        :param megacity:        megacity diary queue
+        :param cx:              seen local diary offset
+        :param cy:              seen nearby diary offset
+        :param cz:              seen nation diary offset
+        :param cm:              seen megacity diary offset
+        :param x:               local diary scale factor
+        :param y:               nearby diary scale factor
+        :param z:               nation diary scale factor
+        :param m:               megacity diary scale factor
+        :param size:            nubmer of diary
+        :return:
+        """
+        # 本地 临近 特大城市 全国 四个层级 都按照的是四舍五入取得方式
+        # 针对出现的问题，本次相应的优化是：
+        # 1、如果出现两个层级为零，且有剩余坑位时，则按照本地 临近 全国的优先级，先给优先级高且为零的层级一个坑位。
+        # 2、如果所有层级都非零，且有剩余坑位时，则优先给权重占比大的层级一个坑位。
+        # 3、如果只有一个层级为零，且有剩余坑位时，则优先填充权重占比大的层级一个坑位。
+        nx = int(round(x * 1.0 / (x + y + z + m) * size))
+        ny = int(round(y * 1.0 / (x + y + z + m) * size))
+        nz = int(round(z * 1.0 / (x + y + z + m) * size))
+        nm = int(round(m * 1.0 / (x + y + z + m) * size))
+        nxyz = [nx, ny, nm, nz]
+        xyz = [x, y, m, z]
+        counter = Counter([nx, ny, nm, nz])
+        if counter[0] == 2:
+            nxyz[nxyz.index(0)] += size - sum(nxyz)
+        else:
+            nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+        nx, ny, nm, nz = nxyz
+
+        slocal = local[cx:cx + nx]
+        cx = min(cx + nx, len(local))
+        ny += (nx - len(slocal))
+
+        snearby = nearby[cy:cy + ny]
+        cy = min(cy + ny, len(nearby))
+        nm += (ny - len(snearby))
+
+        smegacity = megacity[cm: cm + nm]
+        cm = min(cm + nm, len(megacity))
+        nz += (nm - len(smegacity))
+
+        snation = nation[cz:cz + nz]
+        cz = min(cz + nz, len(nation))
+
+        return chain(slocal, snearby, smegacity, snation), cx, cy, cm, cz

tensnsorflow/dd.py

+@bind("doris/recommend/get_articles")
+def get_articles(tags, offset=0, size=10):
+    filters = {
+        "content_level": [CONTENT_CLASS.EXCELLENT, CONTENT_CLASS.FINE]
+    }
+    if tags:
+        filters["tag_ids"] = tags
+
+    articles = recall_articles(None, offset, size, filters, ARTICLE_SORT_TYPE.RECOMMEND, {})
+    article_ids = [article['id'] for article in articles]
+    return {"article_ids": article_ids}
+
+
+@bind("doris/recommend/get_answers")
+def get_answers(tags, offset=0, size=10):
+    filters = {
+        "content_level": [CONTENT_CLASS.EXCELLENT, CONTENT_CLASS.FINE]
+    }
+
+    if tags:
+        filters["tag_ids"] = tags
+    tail = offset + size
+    answer_ids = []
+    if tail < MAX_LOAD:
+        answers = recall_answers(None, 0, MAX_LOAD, filters, ANSWER_SORT_TYPE.RECOMMEND, {}, fields=["id", "user_id"])
+        answers = filter(lambda answer: "id" in answer and "user_id" in answer, answers)
+        answer_items = [(answer["id"], answer["user_id"]) for answer in answers]
+        drop_dup_answers = drop_dup(answer_items)
+        if tail <= len(drop_dup_answers):
+            answer_ids = [item[0] for item in drop_dup_answers[offset:tail]]
+    if len(answer_ids) == 0:
+        answers = recall_answers(None, offset, size, filters, ANSWER_SORT_TYPE.RECOMMEND, {})
+        answer_ids = [answer['id'] for answer in answers]
+    return {"answer_ids": answer_ids}
+
+@bind('doris/recommend/icon')
+def fetch_icon(device_id, size):
+    try:
+        card_type = "icon"
+        try:
+             que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+             que = IconQueue.objects.last()
+        if not que:
+            return {"icon": []}
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        data = list(islice(cycle(que), cursor, cursor + size))
+        return {card_type: list(map(int, data))}
+    except:
+        logging_exception()
+        return {"icon": []}
+
+@bind('doris/recommend/homepage_polymer')
+def fetch_polymer_ids(device_id, size):
+    try:
+        card_type = "polymer_ids"
+        try:
+            que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+            que = IconQueue.objects.last()
+        if not que:
+            return {"polymer_ids": []}
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        data = list(islice(cycle(que), cursor, cursor + size))
+        return {card_type: list(map(int, data))}
+    except:
+        logging_exception()
+        return {"polymer_ids": []}
+
+@bind('doris/recommend/feed')
+def recommend_feed(device_id, card_type, city_id, size):
+    try:
+        return RecommendFeed.dispatch(device_id, card_type,
+                                      city_id, size)
+    except:
+        logging_exception()
+        return {card_type: []}
+
+
+class RecommendFeed:
+    @classmethod
+    def dispatch(cls, device_id, card_type, city_id, size):
+        data = []
+        if card_type == CARD_TYPE.QA:
+            data = cls.fetch_qa(device_id, card_type, size)
+        elif card_type == CARD_TYPE.ANSWER:
+            data = cls.fetch_answer(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.ARTICLE:
+            data = cls.fetch_article(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.QUESTION:
+            data = cls.fetch_question(device_id, card_type, size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.DIARY:
+            data = cls.fetch_diary(device_id, card_type, city_id, size)
+
+        elif card_type == CARD_TYPE.USERTOPIC:
+            data = cls.fetch_user_topic(device_id,card_type,size)
+        elif card_type == CARD_TYPE.DOCTORTOPIC:
+            data = cls.fetch_doctor_topic(device_id,card_type,size)
+            data = list(map(int, data))
+        elif card_type == CARD_TYPE.ENCYCLOPEDIA:
+            data = cls.fetch_wiki(device_id,card_type,size)
+        return {card_type: data}
+
+    @staticmethod
+    def current_date():
+        return datetime.datetime.now().strftime('%Y-%m-%d')
+
+    @staticmethod
+    def fetch_question(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceQuestionQueue.objects.get(device_id=device_id)
+        except DeviceQuestionQueue.DoesNotExist:
+            que = QuestionQueue.objects.last()
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_icon(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceIconQueue.objects.get(device_id=device_id)
+        except DeviceIconQueue.DoesNotExist:
+            que = IconQueue.objects.last()
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_wiki(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+
+            que = WikiQueue.objects.last()
+            if not que:
+                return []
+            # que = list(filter(None, que.queue.split(',')))
+            que = json.loads(que.queue)
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_answer(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                          card_type=card_type, date=RecommendFeed.current_date())
+            try:
+                que = DeviceAnswerQueue.objects.get(device_id=device_id)
+            except DeviceAnswerQueue.DoesNotExist:
+                que = AnswerQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_qa(device_id, card_type, size):
+        try:
+            def get_after_filter_qa():
+                try:
+                    return json.loads(gmkv.get(after_filter_key))
+                except:
+                    return []
+
+            def write_after_filter_qa(cid_list):
+                try:
+                    if gmkv.exists(after_filter_key):
+                        gmkv.set(after_filter_key, json.dumps(cid_list))
+                    else:
+                        gmkv.set(after_filter_key, json.dumps(cid_list),ex = 6*60*60)
+
+                except:
+                    logging_exception()
+                    logger.error("catch exception,err_log:%s" % traceback.format_exc())
+
+            def filter_qa(device_id,cid_list):
+                try:
+                    key = str(device_id) + "_dislike_qa"
+                    if gmkv.exists(key):
+                        dislike = gmkv.smembers(key)
+                        if len(cid_list) > 0:
+                            if type(cid_list[0]) == int or type(cid_list[0]) == str:
+                                cid_list = [i for i in cid_list if str(i).encode('utf-8') not in dislike]
+                            else:
+                                cid_list = [i for i in cid_list if i not in dislike]
+                        return cid_list
+                    else:
+                        return cid_list
+                except:
+                    return cid_list
+
+            def read_history(cid_list):
+                if redis_client.exists(today_qa_key):
+                    redis_client.sadd(today_qa_key, *cid_list)
+                else:
+                    redis_client.sadd(today_qa_key, *cid_list)
+                    redis_client.expire(today_qa_key, 15 * 24 * 60 * 60)
+                if redis_client.exists(read_qa_key) and redis_client.exists(old_qa_key):
+                    redis_client.sdiffstore(read_qa_key, read_qa_key, old_qa_key)
+                    redis_client.delete(old_qa_key)
+                    redis_client.expire(read_qa_key, time=13 * 24 * 60 * 60)
+
+                redis_client.sadd(read_qa_key, *cid_list)
+
+            def get_gmkv(redis_ip, redis_port, redis_db, redis_password=""):
+                try:
+                    if len(redis_password) == 0:
+                        cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, socket_timeout=2)
+                    else:
+                        cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, password=redis_password,
+                                              socket_timeout=2)
+                    cli_ins.ping()
+
+                    return cli_ins
+                except:
+                    return None
+
+            search_qa_recommend_list = list()
+            read_qa_key = "TS:recommend_answer_set:device_id:" + str(device_id)
+            old_qa_key = "TS:recommend_answer_set:device_id:{}:{}"\
+                .format(device_id,(datetime.date.today() - datetime.timedelta(days=14)).strftime("%Y-%m-%d"))
+            today_qa_key = "TS:recommend_answer_set:device_id:{}:{}"\
+                .format(device_id, datetime.date.today().strftime("%Y-%m-%d"))
+            answer_queue_key = "qa_is_tail:" + str(device_id)
+            after_filter_key = "device_qa_after_filter:device_id:" + str(device_id)
+            gmkv = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gmkv = get_gmkv(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"],
+                                         redis_db=gm_kv_host_item["db"],
+                                         redis_password=gm_kv_host_item["password"])
+                if gmkv:
+                    break
+
+            if device_id != '0':
+                search_qa_recommend_key = "TS:search_recommend_answer_queue:device_id:" + str(device_id)
+                if redis_client.exists(search_qa_recommend_key):
+                    search_qa_recommend_dict = redis_client.hgetall(search_qa_recommend_key)
+                    queue_list = json.loads(search_qa_recommend_dict[b'answer_queue'])
+                    queue_list = filter_qa(device_id, queue_list)
+                    if len(queue_list) == 0:
+                        redis_client.delete(search_qa_recommend_key)
+                    elif len(queue_list) == 1:
+                        size = size - 1
+                        search_qa_recommend_list = queue_list
+                        redis_client.delete(search_qa_recommend_key)
+                    else:
+                        size = size - 1
+                        search_qa_recommend_list.append(queue_list[0])
+                        redis_client.hset(search_qa_recommend_key,"answer_queue",json.dumps(queue_list[1:]))
+
+                if gmkv.exists(answer_queue_key):
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                elif gmkv.exists(after_filter_key):
+                    que = get_after_filter_qa()
+                    que = filter_qa(device_id,que)
+                    if len(que) == 0:
+                        gmkv.set(answer_queue_key,"tail",ex = 6*60*60)
+                        if len(search_qa_recommend_list) > 0:
+                            search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                            read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+                    elif len(que) <= size:
+                        search_qa_recommend_list.extend(que)
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+                    else:
+                        search_qa_recommend_list.extend(que[:size])
+                        write_after_filter_qa(que[size:])
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                        return search_qa_recommend_list
+
+                try:
+                    que = DeviceQAQueue.objects.get(device_id=device_id)
+                except DeviceQAQueue.DoesNotExist:
+                    que = AnswerQueue.objects.last()
+                if not que:
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                qa = list(filter(None, que.queue.split(',')))
+                if device_id != "0":
+                    qa = filter_qa(device_id,qa)
+                if len(qa) == 0:
+                    if device_id != "0":
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                    if len(search_qa_recommend_list) > 0:
+                        search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                elif len(qa) <= size:
+                    search_qa_recommend_list.extend(qa)
+                    search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                    if device_id != "0":
+                        gmkv.set(answer_queue_key, "tail", ex=6 * 60 * 60)
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+                else:
+                    search_qa_recommend_list.extend(qa[:size])
+                    search_qa_recommend_list = list(map(int, search_qa_recommend_list))
+                    if device_id != "0":
+                        write_after_filter_qa(qa[size:])
+                        read_history(search_qa_recommend_list)
+                    return search_qa_recommend_list
+
+            else:
+                key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                              card_type=card_type, date=RecommendFeed.current_date())
+                try:
+                    que = DeviceQAQueue.objects.get(device_id=device_id)
+                except DeviceQAQueue.DoesNotExist:
+                    que = AnswerQueue.objects.last()
+                if not que:
+                    return []
+                que = list(filter(None, que.queue.split(',')))
+
+                # adjust args.
+                cursor = redis_client.get(key) or 0
+                cursor = int(cursor) % len(que)
+                size = min(size, len(que))
+                # redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                data = list(islice(cycle(que), cursor, cursor + size))
+                data = list(map(int, data))
+                if cursor + 2 * size < len(que):
+                    redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                else:
+                    try:
+                        context.request_logger.app(reset_answer_queue=True)
+                        cursor = 0
+                        redis_client.set(key, cursor, ex=24 * 60 * 60)
+                    except:
+                        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+                return data
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_article(device_id, card_type, size):
+        key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                      card_type=card_type, date=RecommendFeed.current_date())
+        try:
+            que = DeviceArticleQueue.objects.get(device_id=device_id)
+        except DeviceArticleQueue.DoesNotExist:
+            que = ArticleQueue.objects.last()
+
+        if not que:
+            return []
+
+        que = list(filter(None, que.queue.split(',')))
+
+        # adjust args.
+        cursor = redis_client.get(key) or 0
+        cursor = int(cursor) % len(que)
+        size = min(size, len(que))
+        redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+        return list(islice(cycle(que), cursor, cursor + size))
+
+    @staticmethod
+    def fetch_user_topic(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id, card_type=card_type,
+                                                          date=RecommendFeed.current_date())
+
+            if (device_id != '0') and size >= 2:
+
+                search_topic_recommend_key = "TS:search_recommend_tractate_queue:device_id:" + str(device_id)
+                search_topic_recommend_list = list()
+                search_cursor_ts = 0
+                if redis_client.exists(search_topic_recommend_key):
+                    search_topic_recommend_dict = redis_client.hgetall(search_topic_recommend_key)
+                    if b'cursor' in search_topic_recommend_dict:
+                        search_cursor_ts = json.loads(search_topic_recommend_dict[b'cursor'])
+                        if search_cursor_ts < 30:
+                            search_topic_recommend_list = json.loads(search_topic_recommend_dict[b'tractate_queue'])
+                            if search_cursor_ts < len(search_topic_recommend_list):
+                                size = size - 2
+
+            try:
+                que = DeviceUserTopicQueue.objects.get(device_id=device_id)
+            except DeviceUserTopicQueue.DoesNotExist:
+                que = UserTopicQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            data = list(islice(cycle(que), cursor, cursor + size))
+            data = list(map(int, data))
+
+            if cursor + 2 * size < len(que):
+                redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            else:
+                try:
+                    context.request_logger.app(reset_queue=True)
+                    cursor = 0
+                    redis_client.set(key, cursor, ex=24 * 60 * 60)
+                except:
+                    redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+
+            if device_id != '0' and size >= 2:
+                if len(search_topic_recommend_list) > 0 and search_cursor_ts < len(search_topic_recommend_list):
+                    queue = search_topic_recommend_list[search_cursor_ts:search_cursor_ts + 2]
+                    queue.extend(data)
+                    data = queue
+                    new_search_cursor = search_cursor_ts + 2
+                    redis_client.hset(search_topic_recommend_key, 'cursor', new_search_cursor)
+                    redis_client.expire(search_topic_recommend_key, 30 * 24 * 60 * 60)
+                read_topic_key = "TS:recommend_tractate_set:device_id:" + str(device_id)
+                if len(data) > 0:
+                    redis_client.sadd(read_topic_key, *data)
+            return data
+
+        except:
+            logging_exception()
+            return []
+
+    @staticmethod
+    def fetch_doctor_topic(device_id, card_type, size):
+        try:
+            key = '{device_id}-{card_type}-{date}'.format(device_id=device_id,
+                                                          card_type=card_type, date=RecommendFeed.current_date())
+            try:
+                que = DeviceDoctorTopicQueue.objects.get(device_id=device_id)
+            except DeviceDoctorTopicQueue.DoesNotExist:
+                que = DoctorTopicQueue.objects.last()
+            if not que:
+                return []
+            que = list(filter(None, que.queue.split(',')))
+            # adjust args.
+            cursor = redis_client.get(key) or 0
+            cursor = int(cursor) % len(que)
+            size = min(size, len(que))
+            redis_client.set(key, cursor + size, ex=24 * 60 * 60)
+            return list(islice(cycle(que), cursor, cursor + size))
+        except:
+            logging_exception()
+            return []
+
+
+    @classmethod
+    def get_gm_kv_ins(cls,redis_ip, redis_port, redis_db, redis_password=""):
+        try:
+            if len(redis_password) == 0:
+                cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, socket_timeout=2)
+            else:
+                cli_ins = redis.Redis(host=redis_ip, port=redis_port, db=redis_db, password=redis_password,
+                                      socket_timeout=2)
+            cli_ins.ping()
+
+            return cli_ins
+        except:
+            return None
+
+    @classmethod
+    def fetch_diary_queue_data(cls, city_id, device_id=None):
+        local = list()
+        nearby = list()
+        nation = list()
+        megacity = list()
+        use_city_id = city_id
+        try:
+            gm_kv_ins = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gm_kv_ins = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"], redis_db=gm_kv_host_item["db"],redis_password=gm_kv_host_item["password"])
+                if gm_kv_ins:
+                    break
+
+            specify_city_id_key = "diary_queue:city_id:" + use_city_id
+            world_city_id_key = "diary_queue:city_id:world"
+            if device_id is not None:
+                specify_city_id_key = "device_diary_queue:device_id:" + device_id + ":city_id:" + use_city_id
+
+            city_val_dict = gm_kv_ins.hgetall(specify_city_id_key)
+            if len(city_val_dict) == 0:
+                city_val_dict = gm_kv_ins.hgetall(world_city_id_key)
+                use_city_id = "world"
+
+            if b"native_queue" in city_val_dict and city_val_dict[b"native_queue"]:
+                local = list(filter(None, city_val_dict[b"native_queue"].split(b",")))
+            if b"nearby_queue" in city_val_dict and city_val_dict[b"nearby_queue"]:
+                nearby = list(filter(None, city_val_dict[b"nearby_queue"].split(b",")))
+            if b"nation_queue" in city_val_dict and city_val_dict[b"nation_queue"]:
+                nation = list(filter(None, city_val_dict[b"nation_queue"].split(b",")))
+            if b"megacity_queue" in city_val_dict and city_val_dict[b"megacity_queue"]:
+                megacity = list(filter(None, city_val_dict[b"megacity_queue"].split(b",")))
+
+            return (local, nearby, nation, megacity, use_city_id)
+
+        except:
+            logging_exception()
+            logger.error("catch exception,err_log:%s" % traceback.format_exc())
+            qs = DiaryQueue.objects.filter(city_id__in=[city_id, 'world'])
+            # Assume that world queue must exist.
+            if len(qs) == 1:
+                obj = qs[0]
+            else:
+                obj = qs[0] if qs[0].city_id == city_id else qs[1]
+
+            if obj.native_queue:
+                local = list(filter(None, obj.native_queue.split(',')))
+            if obj.nearby_queue:
+                nearby = list(filter(None, obj.nearby_queue.split(',')))
+            if obj.nation_queue:
+                nation = list(filter(None, obj.nation_queue.split(',')))
+            if obj.megacity_queue:
+                megacity = list(filter(None, obj.megacity_queue.split(',')))
+
+            use_city_id = obj.city_id if obj else use_city_id
+            return (local, nearby, nation, megacity, use_city_id)
+
+    @classmethod
+    def fetch_device_diary_queue_data(cls, city_id, device_id):
+        local = list()
+        nearby = list()
+        nation = list()
+        megacity = list()
+        use_city_id = city_id
+        try:
+            gm_kv_ins = None
+            for gm_kv_host_item in settings.GM_KV_HOSTS:
+                gm_kv_ins = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"], redis_db=gm_kv_host_item["db"],redis_password=gm_kv_host_item["password"])
+                if gm_kv_ins:
+                    break
+
+            specify_city_id_key = "device_diary_queue:device_id:" + device_id + ":city_id:" + use_city_id
+            city_val_dict = gm_kv_ins.hgetall(specify_city_id_key)
+
+            if b"native_queue" in city_val_dict and city_val_dict[b"native_queue"]:
+                local = list(filter(None, city_val_dict[b"native_queue"].split(b",")))
+            if b"nearby_queue" in city_val_dict and city_val_dict[b"nearby_queue"]:
+                nearby = list(filter(None, city_val_dict[b"nearby_queue"].split(b",")))
+            if b"nation_queue" in city_val_dict and city_val_dict[b"nation_queue"]:
+                nation = list(filter(None, city_val_dict[b"nation_queue"].split(b",")))
+            if b"megacity_queue" in city_val_dict and city_val_dict[b"megacity_queue"]:
+                megacity = list(filter(None, city_val_dict[b"megacity_queue"].split(b",")))
+
+            return (local, nearby, nation, megacity, use_city_id)
+        except:
+            logging_exception()
+            logger.error("catch exception,err_log:%s" % traceback.format_exc())
+            obj = DeviceDiaryQueue.objects.filter(device_id=device_id, city_id=city_id).first()
+            if obj and obj.native_queue:
+                local = list(filter(None, obj.native_queue.split(',')))
+            if obj and obj.nearby_queue:
+                nearby = list(filter(None, obj.nearby_queue.split(',')))
+            if obj and obj.nation_queue:
+                nation = list(filter(None, obj.nation_queue.split(',')))
+            if obj and obj.megacity_queue:
+                megacity = list(filter(None, obj.megacity_queue.split(',')))
+
+            use_city_id = obj.city_id if obj else use_city_id
+            return (local, nearby, nation, megacity, use_city_id)
+
+    @classmethod
+    def fetch_diary(cls, device_id, card_type, city_id, size):
+        try:
+            def read_history(cid_list):
+                if redis_client.exists(today_key):
+                    redis_client.sadd(today_key, *cid_list)
+                else:
+                    redis_client.sadd(today_key, *cid_list)
+                    redis_client.expire(today_key, 15 * 24 * 60 * 60)
+                if redis_client.exists(read_key) and redis_client.exists(old_key):
+                    redis_client.sdiffstore(read_key, read_key, old_key)
+                    redis_client.delete(old_key)
+                    redis_client.expire(read_key, time=13 * 24 * 60 * 60)
+
+                redis_client.sadd(read_key, *cid_list)
+
+            def dislike_cid_filter(device_id, cid_list):
+                try:
+                    key = str(device_id) + "_dislike_diary"
+                    if gmkv.exists(key):
+                        dislike = gmkv.smembers(key)
+                        if len(cid_list) > 0:
+                            if type(cid_list[0]) == int or type(cid_list[0]) == str:
+                                cid_list = [i for i in cid_list if str(i).encode('utf-8') not in dislike]
+                            else:
+                                cid_list = [i for i in cid_list if i not in dislike]
+                    return cid_list
+                except:
+                    return cid_list
+
+            def fetch_after_filter_queue(device_id, city_id, name):
+                local = list()
+                try:
+                    key = "device_diary_queue_after_filter:device_id:" + device_id + ":city_id:" + city_id + name
+                    if gmkv.exists(key):
+                        return json.loads(gmkv.get(key))
+                    else:
+                        return local
+                except:
+
+                    return local
+
+            def write_after_filter_queue(device_id, city_id, cid_list, name):
+                try:
+                    key = "device_diary_queue_after_filter:device_id:" + device_id + ":city_id:" + city_id + name
+                    if gmkv.exists(key):
+                        gmkv.set(key, json.dumps(cid_list))
+                    else:
+                        gmkv.set(key, json.dumps(cid_list), ex=6 * 60 * 60)
+
+                except:
+                    logging_exception()
+                    logger.error("catch exception,err_log:%s" % traceback.format_exc())
+
+            def get_data(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+                nx = int(round(x * 1.0 / (x + y + z + m) * size))
+                ny = int(round(y * 1.0 / (x + y + z + m) * size))
+                nz = int(round(z * 1.0 / (x + y + z + m) * size))
+                nm = int(round(m * 1.0 / (x + y + z + m) * size))
+                nxyz = [nx, ny, nm, nz]
+                xyz = [x, y, m, z]
+                counter = Counter([nx, ny, nm, nz])
+                if counter[0] == 2:
+                    nxyz[nxyz.index(0)] += size - sum(nxyz)
+                else:
+                    nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+                nx, ny, nm, nz = nxyz
+
+                local_filter = dislike_cid_filter(device_id, cx)
+                if len(local_filter) == 0:
+                    local_filter = dislike_cid_filter(device_id, local)
+                slocal = local_filter[:nx]
+                have_x = local_filter[nx:]
+                x_list = list(map(int, have_x))
+                write_after_filter_queue(device_id, city_id, x_list, "native")
+
+                ny += (nx - len(slocal))
+
+                nearby_filter = dislike_cid_filter(device_id, cy)
+                if len(nearby_filter) == 0:
+                    nearby_filter = dislike_cid_filter(device_id, nearby)
+                snearby = nearby_filter[:ny]
+                have_y = nearby_filter[ny:]
+                y_list = list(map(int, have_y))
+                write_after_filter_queue(device_id, city_id, y_list, "nearby")
+
+                nm += (ny - len(snearby))
+
+                megacity_filter = dislike_cid_filter(device_id, cm)
+                if len(megacity_filter) == 0:
+                    megacity_filter = dislike_cid_filter(device_id, megacity)
+                smegacity = megacity_filter[:nm]
+                have_m = megacity_filter[nm:]
+                m_list = list(map(int, have_m))
+                write_after_filter_queue(device_id, city_id, m_list, "megacity")
+
+                nz += (nm - len(smegacity))
+
+                nation_filter = dislike_cid_filter(device_id, cz)
+                if len(nation_filter) == 0:
+                    nation_filter = dislike_cid_filter(device_id, nation)
+                snation = nation_filter[:nz]
+                have_z = snation[nz:]
+                z_list = list(map(int, have_z))
+                write_after_filter_queue(device_id, city_id, z_list, "nation")
+
+                return chain(slocal, snearby, smegacity, snation)
+            if device_id != '0':
+                portrait_list = list()
+                click_diary_size = 1
+                search_diary_size = 4
+                read_key = "TS:recommend_diary_set:device_id:" + str(device_id)
+                old_key = "TS:recommend_diary_set:device_id:{}:{}" \
+                    .format(device_id, (datetime.date.today() - datetime.timedelta(days=14)).strftime("%Y-%m-%d"))
+                today_key = "TS:recommend_diary_set:device_id:{}:{}" \
+                    .format(device_id, datetime.date.today().strftime("%Y-%m-%d"))
+                user_portrait_diary_key = 'user_portrait_recommend_diary_queue:device_id:%s:%s' % \
+                                          (device_id, datetime.datetime.now().strftime('%Y-%m-%d'))
+                gmkv = None
+                for gm_kv_host_item in settings.GM_KV_HOSTS:
+                    gmkv = cls.get_gm_kv_ins(redis_ip=gm_kv_host_item["host"], redis_port=gm_kv_host_item["port"],
+                                                  redis_db=gm_kv_host_item["db"],
+                                                  redis_password=gm_kv_host_item["password"])
+                    if gmkv:
+                        break
+
+                if redis_client.exists(user_portrait_diary_key):
+                    user_portrait_diary_dict = redis_client.hgetall(user_portrait_diary_key)
+                    user_portrait_cursor = str(user_portrait_diary_dict[b'cursor'], encoding='utf-8')
+                    if user_portrait_cursor == '0':
+                        if b'len_cursor' in user_portrait_diary_dict.keys():
+                            user_portrait_diary_list = json.loads(user_portrait_diary_dict[b'diary_queue'])
+                            filter_user_portrait_diary_list = dislike_cid_filter(device_id, user_portrait_diary_list)
+                            if len(filter_user_portrait_diary_list) > size:
+                                portrait_list = filter_user_portrait_diary_list[:size]
+                                redis_client.hset(user_portrait_diary_key, 'diary_queue',
+                                                  json.dumps(filter_user_portrait_diary_list[size:]))
+                                portrait_list = list(map(int, portrait_list))
+                                read_history(portrait_list)
+                                return portrait_list
+                            else:
+                                size = size - len(filter_user_portrait_diary_list)
+                                portrait_list = filter_user_portrait_diary_list
+                                redis_client.delete(user_portrait_diary_key)
+
+                search_diary_recommend_key = "TS:search_recommend_diary_queue:device_id:" + str(device_id)
+                search_list = list()
+
+                if redis_client.exists(search_diary_recommend_key) and size > 3:
+                    search_diary_recommend_dict = redis_client.hgetall(search_diary_recommend_key)
+                    search_diary_recommend_list = json.loads(search_diary_recommend_dict[b'diary_queue'])
+                    search_diary_recommend_list = dislike_cid_filter(device_id, search_diary_recommend_list)
+                    if len(search_diary_recommend_list) == 0:
+                        redis_client.delete(search_diary_recommend_key)
+                    elif len(search_diary_recommend_list) <= search_diary_size:
+                        search_list = search_diary_recommend_list
+                        size = size - len(search_diary_recommend_list)
+                        redis_client.delete(search_diary_recommend_key)
+
+                    else:
+                        search_list = search_diary_recommend_list[:search_diary_size]
+                        size = size - search_diary_size
+                        redis_client.hset(search_diary_recommend_key, 'diary_queue',
+                                          json.dumps(search_diary_recommend_list[search_diary_size:]))
+                    if size <= 0:
+                        portrait_list.extend(search_list)
+                        portrait_list = list(map(int, portrait_list))
+                        read_history(portrait_list)
+                        return portrait_list
+
+                diary_recommend_key = "TS:recommend_diary_queue:device_id:" + str(device_id)
+                ts_recommend_list = list()
+                if redis_client.exists(diary_recommend_key) and size > 0:
+                    diary_recommend_dict = redis_client.hgetall(diary_recommend_key)
+                    diary_recommend_list = json.loads(diary_recommend_dict[b'diary_queue'])
+                    diary_recommend_list = dislike_cid_filter(device_id, diary_recommend_list)
+                    if len(diary_recommend_list) == 0:
+                        redis_client.delete(diary_recommend_key)
+
+                    elif len(diary_recommend_list) <= click_diary_size:
+                        ts_recommend_list = diary_recommend_list
+                        redis_client.delete(diary_recommend_key)
+                        size = size - len(ts_recommend_list)
+
+                    else:
+                        size = size - click_diary_size
+                        ts_recommend_list = diary_recommend_list[:click_diary_size]
+                        diary_recommend_list_json = json.dumps(diary_recommend_list[click_diary_size:])
+                        redis_client.hset(diary_recommend_key, 'diary_queue', diary_recommend_list_json)
+                    if size <= 0:
+                        portrait_list.extend(search_list)
+                        portrait_list.extend(ts_recommend_list)
+                        portrait_list = list(map(int, portrait_list))
+                        read_history(portrait_list)
+                        return portrait_list
+
+                if size > 0:
+                    try:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_device_diary_queue_data(city_id,
+                                                                                                       device_id)
+                        if len(local) == 0 and len(nearby) == 0 and len(nation) == 0 and len(megacity) == 0:
+                            (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+
+                    except:
+                        logging_exception()
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                    x, y, m, z = cls.get_city_scale(city_id)
+                    cx = fetch_after_filter_queue(device_id, city_id, "native")
+                    cy = fetch_after_filter_queue(device_id, city_id, "nearby")
+                    cz = fetch_after_filter_queue(device_id, city_id, "nation")
+                    cm = fetch_after_filter_queue(device_id, city_id, "megacity")
+                    data = get_data(
+                        local, nearby, nation, megacity,
+                        cx, cy, cm, cz,
+                        x, y, z, m, size)
+
+                    portrait_list.extend(search_list)
+                    portrait_list.extend(ts_recommend_list)
+                    portrait_list.extend(data)
+                    if len(portrait_list) == 0:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                        portrait_list = cls.get_queue(local, nearby, nation, megacity,
+                                                      device_id, city_id, size, x, y, z, m)
+                    portrait_list = list(map(int, portrait_list))
+                    if len(portrait_list) != 0:
+                        read_history(portrait_list)
+                    return portrait_list
+            else:
+
+                try:
+                    (local, nearby, nation, megacity, city_id) = cls.fetch_device_diary_queue_data(city_id, device_id)
+                    if len(local) == 0 and len(nearby) == 0 and len(nation) == 0 and len(megacity) == 0:
+                        (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+                except:
+                    logging_exception()
+                    (local, nearby, nation, megacity, city_id) = cls.fetch_diary_queue_data(city_id)
+
+                key = '{device_id}-{city_id}-{date}'.format(device_id=device_id,
+                                                            city_id=city_id, date=RecommendFeed.current_date())
+                # strategy rule: when user refresh over 30 loadings, reset native nearby nation queue cursor.
+                counter_key = key + '-counter_v1'
+                counter = redis_client.incr(counter_key)
+                if counter == 1:
+                    redis_client.expire(counter_key, 24 * 60 * 60)
+                cursor_key = key + '-cursor_v1'
+                cursor = redis_client.get(cursor_key) or b'0-0-0-0'
+                # if counter > 30:
+                #     cursor = b'0-0-0-0'
+                #     redis_client.delete(counter_key)
+                cx, cy, cm, cz = map(int, cursor.split(b'-'))
+
+                x, y, m, z = cls.get_city_scale(city_id)
+                data, ncx, ncy, ncm, ncz = cls.get_scale_data(
+                    local, nearby, nation, megacity,
+                    cx, cy, cm, cz,
+                    x, y, z, m, size
+                )
+
+                if ncx == cx and ncy == cy:  # native queue and nearby queue
+                    logger.info("diary queue reach end,cx:%d,cy:%d,cm:%d,cz:%d", cx, cy, cm, cz)
+                    # redis_client.delete(counter_key)
+                    # data, ncx, ncy, ncm, ncz = cls.get_scale_data(
+                    #     local, nearby, nation, megacity,
+                    #     0, 0, 0, 0,
+                    #     x, y, z, m, size
+                    # )
+                    ncx = ncy = ncm = ncz = 0
+
+                val = '-'.join(map(str, [ncx, ncy, ncm, ncz]))
+                redis_client.set(cursor_key, val, ex=24 * 60 * 60)
+                data = list(map(int, data))
+
+                return data
+        except:
+            logging_exception()
+            return []
+
+
+    @classmethod
+    def get_queue(cls,local, nearby, nation, megacity, device_id,city_id,size,x,y,z,m):
+        key = '{device_id}-{city_id}-{date}'.format(device_id=device_id,
+                                                    city_id=city_id, date=RecommendFeed.current_date())
+        counter_key = key + '-counter_v1'
+        counter = redis_client.incr(counter_key)
+        if counter == 1:
+            redis_client.expire(counter_key, 24 * 60 * 60)
+        cursor_key = key + '-cursor_v1'
+        cursor = redis_client.get(cursor_key) or b'0-0-0-0'
+        cx, cy, cm, cz = map(int, cursor.split(b'-'))
+
+        def get_scale(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+            nx = int(round(x * 1.0 / (x + y + z + m) * size))
+            ny = int(round(y * 1.0 / (x + y + z + m) * size))
+            nz = int(round(z * 1.0 / (x + y + z + m) * size))
+            nm = int(round(m * 1.0 / (x + y + z + m) * size))
+            nxyz = [nx, ny, nm, nz]
+            xyz = [x, y, m, z]
+            counter = Counter([nx, ny, nm, nz])
+            if counter[0] == 2:
+                nxyz[nxyz.index(0)] += size - sum(nxyz)
+            else:
+                nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+            nx, ny, nm, nz = nxyz
+
+            slocal = local[cx:cx + nx]
+            cx = min(cx + nx, len(local))
+            ny += (nx - len(slocal))
+
+            snearby = nearby[cy:cy + ny]
+            cy = min(cy + ny, len(nearby))
+            nm += (ny - len(snearby))
+
+            smegacity = megacity[cm: cm + nm]
+            cm = min(cm + nm, len(megacity))
+            nz += (nm - len(smegacity))
+
+            snation = nation[cz:cz + nz]
+            cz = min(cz + nz, len(nation))
+            return chain(slocal, snearby, smegacity, snation), cx, cy, cm, cz
+
+        data, ncx, ncy, ncm, ncz = get_scale(
+            local, nearby, nation, megacity,
+            cx, cy, cm, cz,
+            x, y, z, m, size)
+
+        if ncx == cx and ncy == cy:  # native queue and nearby queue
+            logger.info("diary queue reach end,cx:%d,cy:%d,cm:%d,cz:%d", cx, cy, cm, cz)
+
+            ncx = ncy = ncm = ncz = 0
+
+        val = '-'.join(map(str, [ncx, ncy, ncm, ncz]))
+        redis_client.set(cursor_key, val, ex=24 * 60 * 60)
+        return list(map(int, data))
+
+    @staticmethod
+    def get_city_scale(city_id):
+        try:
+            c = CityScale.objects.get(city_id=city_id)
+            x, y, z, m = c.native, c.nearby, c.nation, c.megacity
+        except CityScale.DoesNotExist:
+            try:
+                c = City.objects.get(id=city_id)
+                if c.level in (CITY_LEVEL.SUPER, CITY_LEVEL.ONE):
+                    x, y, m, z = 4, 3, 0, 3
+                elif c.level == CITY_LEVEL.TWO:
+                    x, y, m, z = 3, 3, 0, 3
+                elif c.level == CITY_LEVEL.THREE:
+                    x, y, m, z = 1, 4, 0, 5
+                else:
+                    x, y, m, z = 0, 0, 0, 10
+            except City.DoesNotExist:
+                x, y, m, z = 0, 0, 0, 10
+
+        return x, y, m, z
+
+    @staticmethod
+    def get_scale_data(local, nearby, nation, megacity, cx, cy, cm, cz, x, y, z, m, size):
+        """
+
+        :param local:           local diary queue
+        :param nearby:          nearby diary queue
+        :param nation:          nation diary queue
+        :param megacity:        megacity diary queue
+        :param cx:              seen local diary offset
+        :param cy:              seen nearby diary offset
+        :param cz:              seen nation diary offset
+        :param cm:              seen megacity diary offset
+        :param x:               local diary scale factor
+        :param y:               nearby diary scale factor
+        :param z:               nation diary scale factor
+        :param m:               megacity diary scale factor
+        :param size:            nubmer of diary
+        :return:
+        """
+        # 本地 临近 特大城市 全国 四个层级 都按照的是四舍五入取得方式
+        # 针对出现的问题，本次相应的优化是：
+        # 1、如果出现两个层级为零，且有剩余坑位时，则按照本地 临近 全国的优先级，先给优先级高且为零的层级一个坑位。
+        # 2、如果所有层级都非零，且有剩余坑位时，则优先给权重占比大的层级一个坑位。
+        # 3、如果只有一个层级为零，且有剩余坑位时，则优先填充权重占比大的层级一个坑位。
+        nx = int(round(x * 1.0 / (x + y + z + m) * size))
+        ny = int(round(y * 1.0 / (x + y + z + m) * size))
+        nz = int(round(z * 1.0 / (x + y + z + m) * size))
+        nm = int(round(m * 1.0 / (x + y + z + m) * size))
+        nxyz = [nx, ny, nm, nz]
+        xyz = [x, y, m, z]
+        counter = Counter([nx, ny, nm, nz])
+        if counter[0] == 2:
+            nxyz[nxyz.index(0)] += size - sum(nxyz)
+        else:
+            nxyz[xyz.index(max(xyz))] += size - sum(nxyz)
+        nx, ny, nm, nz = nxyz
+
+        slocal = local[cx:cx + nx]
+        cx = min(cx + nx, len(local))
+        ny += (nx - len(slocal))
+
+        snearby = nearby[cy:cy + ny]
+        cy = min(cy + ny, len(nearby))
+        nm += (ny - len(snearby))
+
+        smegacity = megacity[cm: cm + nm]
+        cm = min(cm + nm, len(megacity))
+        nz += (nm - len(smegacity))
+
+        snation = nation[cz:cz + nz]
+        cz = min(cz + nz, len(nation))
+
+        return chain(slocal, snearby, smegacity, snation), cx, cy, cm, cz
\ No newline at end of file