Merge branch 'master' into hk

libs/tools.py

@@ -25,10 +25,10 @@ def tzlc(dt, truncate_to_sec=True):
 
 def get_have_read_topic_id_list(device_id,user_id,query_type):
     try:
-        if user_id==-1:
-            redis_key = "physical:home_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(query_type)
-        else:
+        if user_id and user_id>0:
             redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
+        else:
+            redis_key = "physical:home_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(query_type)
 
 
         have_read_topic_id_list = list()

linucb/views/collect_data.py

@@ -72,10 +72,8 @@ class CollectData(object):
                 # Todo:设置过期时间，调研set是否支持
                 redis_client.expire(tag_recommend_redis_key, 7*24*60*60)
 
-                have_read_topic_id_list = list()
-                have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,-1,TopicPageType.HOME_RECOMMEND)
-                promote_recommend_topic_id_list = list()
-                promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(is_online=1).values_list("topic_id")
+                have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,user_id,TopicPageType.HOME_RECOMMEND)
+                promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(is_online=1).values_list("topic_id",flat=True)
                 have_read_topic_id_list.extend(promote_recommend_topic_id_list)
 
                 recommend_topic_id_list = list()

search/utils/topic.py

@@ -124,7 +124,6 @@ class TopicUtils(object):
                                 filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
                                 user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[]):
         """
-        :需增加打散逻辑
         :remark:获取首页推荐帖子列表
         :param user_id:
         :param offset:

search/views/tag.py

@@ -10,6 +10,8 @@ from search.utils.topic import TopicUtils
 from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.common import *
+from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow
+import time
 
 
 def get_highlight(fields=[]):
@@ -95,9 +97,52 @@ def query_by_tag_type(tag_type_id,offset,size):
 
 
 @bind("physical/search/choice_push_tag")
-def choice_push_tag(device_id,user_id):
+def choice_push_tag(device_id, user_id):
+    """
+    :remark 挑选push标签
+    :param device_id:
+    :param user_id:
+    :return:
+    """
     try:
-        pass
+        redis_key_prefix = "physical:push_tag:user_id:"
+        redis_push_tag_key = redis_key_prefix + str(user_id)
+
+        redis_push_tag_data = redis_client.get(redis_push_tag_key)
+        redis_push_tag_dict = json.loads(redis_push_tag_data) if redis_push_tag_data else {}
+
+        now_sec = int(time.time())
+        valid_time = 8*7*24*60*60
+        for tag_id in redis_push_tag_dict:
+            if now_sec - redis_push_tag_dict[tag_id] >= valid_time:
+                redis_push_tag_dict.popitem(tag_id)
+
+        redis_push_tag_list = list(redis_push_tag_dict.keys())
+        account_user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True)
+        community_tag_follow_list = CommunityTagFollow.objects.filter(user_id=user_id).values_list("tag_id",flat=True)
+
+        linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
+        tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
+        linucb_recommend_tag_data = redis_client.get(tag_recommend_redis_key)
+        linucb_recommend_tag_list = json.loads(linucb_recommend_tag_data) if linucb_recommend_tag_data else []
+
+        account_user_tag_list.extend(community_tag_follow_list)
+        account_user_tag_list.extend(linucb_recommend_tag_list)
+        unread_tag_list = list(set(account_user_tag_list) - set(redis_push_tag_list))
+
+        ret_tag_set = set()
+        if len(unread_tag_list)>0:
+            for tag_id in unread_tag_list:
+                valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id,is_online=True).count()
+                if valid_tag_topic_num>100:
+                    ret_tag_set.add(tag_id)
+                    redis_push_tag_dict[tag_id] = now_sec
+                    if len(ret_tag_set)>=2:
+                        break
+
+        redis_client.set(redis_push_tag_key, json.dumps(redis_push_tag_dict))
+
+        return {"tag_list": list(ret_tag_set)}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"tag_list": []}

search/views/topic.py

@@ -49,26 +49,21 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
                                  query_type=TopicPageType.HOME_RECOMMEND):
     try:
         if query is None:
-            # redis_key = "physical:home_recommend" + ":user_id:" + str(
-            #     user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
-            redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
+            if user_id>0:
+                redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
+            else:
+                redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
         else:
-            # redis_key = "physical:home_query" + ":user_id:" + str(
-            #     user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
-            redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
+            if user_id>0:
+                redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
+            else:
+                redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
 
 
         redis_field_list = [b'have_read_topic_list']
         redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
 
         topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
-        # click_recommend_redis_key = "physical:click_recommend:device_id:" + str(device_id)
-
-
-        # recommend_tag_dict = dict()
-        # tag_recommend_val = redis_client.get(tag_recommend_redis_key)
-        # if tag_recommend_val:
-        #     recommend_tag_dict = json.loads(str(tag_recommend_val, encoding="utf-8"))
 
         recommend_topic_list=list()
         recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
@@ -80,20 +75,6 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
                 recommend_topic_list = recommend_topic_id_list[cursor:newcursor]
                 redis_client.hset(topic_recommend_redis_key,"cursor",newcursor)
 
-        # click_recommend_topic_id_list = list()
-        # click_recommend_topic_list = list()
-        #
-        # click_recommend_topic_dict = redis_client.hgetall(click_recommend_redis_key)
-        # if b"data" in click_recommend_topic_dict:
-        #     click_recommend_topic_id_list = json.loads(click_recommend_topic_dict[b"data"])
-        #     cursor = int(str(click_recommend_topic_dict[b"cursor"], encoding="utf-8"))
-        #     newcursor = cursor + 2
-        #     if newcursor < 4 and len(click_recommend_topic_id_list) ==2:
-        #         for i in range(0,2):
-        #           click_recommend_topic_list.append(click_recommend_topic_id_list[i])
-        #         redis_client.hset(click_recommend_redis_key, "cursor", newcursor)
-        # combine_recommend_topic_list_tmp = click_recommend_topic_list.extend(recommend_topic_list)
-        # combine_recommend_topic_list = combine_recommend_topic_list_tmp[0:5]
         have_read_topic_id_list = list()
 
         if redis_field_val_list[0]:
@@ -119,12 +100,12 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         have_read_topic_id_list_add_promote = list()
         have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
         promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
-            is_online=1).values_list("topic_id")
-        if len(have_read_topic_id_list)>0:
-          for topic_id in promote_recommend_topic_id_list:
-             have_read_topic_id_list_add_promote.extend(topic_id)
-        topic_id_list = list()
+            is_online=1).values_list("topic_id",flat=True)
+
+        for topic_id in promote_recommend_topic_id_list:
+            have_read_topic_id_list_add_promote.append(topic_id)
 
+        topic_id_list = list()
         rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
                                                            single_size=size,query=query, query_type=query_type,
                                                            filter_topic_id_list=have_read_topic_id_list_add_promote,
@@ -134,60 +115,19 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         if (len(recommend_topic_list) == 6):
             if (size < 11):
                 topic_id_list.extend(rank_topic_id_list[0:3])
-                topic_id_list.extend(recommend_topic_id_list[0:3])
+                topic_id_list.extend(recommend_topic_list[0:3])
                 topic_id_list.extend(rank_topic_id_list[3:size])
-                topic_id_list.extend(recommend_topic_id_list[3:6])
+                topic_id_list.extend(recommend_topic_list[3:6])
             else:
                 topic_id_list.extend(rank_topic_id_list[0:size - 7])
-                topic_id_list.extend(recommend_topic_id_list[0:3])
+                topic_id_list.extend(recommend_topic_list[0:3])
                 topic_id_list.extend(rank_topic_id_list[size - 7:size])
-                topic_id_list.extend(recommend_topic_id_list[3:6])
+                topic_id_list.extend(recommend_topic_list[3:6])
         else:
             topic_id_list.extend(rank_topic_id_list)
 
-        have_read_group_id_set = set()
-        have_read_user_id_set = set()
-        unread_topic_id_dict = dict()
         logging.info("attention_tag_list:%s"%(str(topic_id_list)))
 
-        # # 当前页小组数量
-        # cur_page_group_num = 0
-        # # 当前页用户数量
-        # cur_page_user_num = 0
-        #
-        # for topic_id in topic_id_dict:
-        #     if topic_id_dict[topic_id][0] in have_read_group_id_set or topic_id_dict[topic_id][
-        #         1] in have_read_user_id_set:
-        #         unread_topic_id_dict[topic_id] = topic_id_dict[topic_id]
-        #     else:
-        #         if isinstance(topic_id_dict[topic_id][0], int) and topic_id_dict[topic_id][
-        #             0] > 0 and cur_page_group_num < (size * 0.9):
-        #             have_read_group_id_set.add(topic_id_dict[topic_id][0])
-        #             have_read_user_id_set.add(topic_id_dict[topic_id][1])
-        #             have_read_topic_id_list.append(topic_id)
-        #             cur_page_group_num += 1
-        #             recommend_topic_ids.append(topic_id)
-        #         elif topic_id_dict[topic_id] and cur_page_user_num < (size * 0.1):
-        #             have_read_user_id_set.add(topic_id_dict[topic_id][1])
-        #             cur_page_user_num += 1
-        #             recommend_topic_ids.append(topic_id)
-        #             have_read_topic_id_list.append(topic_id)
-        #         else:
-        #             unread_topic_id_dict[topic_id] = topic_id_dict[topic_id]
-        #
-        #         if len(recommend_topic_ids) >= size:
-        #             break
-
-        # if len(recommend_topic_ids) < size and len(unread_topic_id_dict) > 0:
-        #     for unread_topic_id in unread_topic_id_dict:
-        #         if len(recommend_topic_ids) < size:
-        #             recommend_topic_ids.append(unread_topic_id)
-        #             have_read_topic_id_list.append(unread_topic_id)
-        #         else:
-        #             break
-
-        # topic_id_list.extend(recommend_topic_list)
-        # recommend_topic_list.extend(topic_id_list)
         have_read_topic_id_list.extend(topic_id_list)
         if len(have_read_topic_id_list) > 30000:
             cut_len = len(have_read_topic_id_list)-30000
@@ -196,7 +136,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
             "have_read_topic_list": json.dumps(have_read_topic_id_list),
         }
         redis_client.hmset(redis_key, redis_dict)
-        # 每个session key保存15分钟
+        # 每个session key保存30天
         redis_client.expire(redis_key, 60 * 60 * 24 * 30)
 
         return topic_id_list

trans2es/models/tag.py

@@ -22,7 +22,6 @@ class TopicTag(models.Model):
     is_collection = models.IntegerField(verbose_name=u"是否编辑标签")
 
 
-
 class AccountUserTag(models.Model):
     class Meta:
         verbose_name=u"用户标签"