Merge branch 'master' into hk

libs/es.py

@@ -155,7 +155,9 @@ class ESPerform(object):
 
             bulk_actions = []
 
-            if sub_index_name=="topic" or sub_index_name=="topic-star-routing":
+            if sub_index_name=="topic" or \
+                    sub_index_name=="topic-star-routing" or \
+                    sub_index_name=="topic-high-star":
                 for data in data_list:
                     if data:
                         bulk_actions.append({

linucb/utils/register_user_tag.py

@@ -29,8 +29,12 @@ class RegisterUserTag(object):
     linucb_device_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:device_id:"
     linucb_user_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:user_id:"
 
+    # 用户关注标签
     linucb_register_user_tag_key = "physical:linucb:register_user_tag_info"
-
+    # # 用户关注标签
+    # linucb_user_attention_tag_key = "physical:linucb:user_attention_tag_info"
+    # # 用户喜好标签（注册勾选 + 关注）
+    # linucb_user_like_tag_key = "physical:linucb:user_like_tag_info"
     @classmethod
     def get_user_attention_tag(cls, pk_list):
         """
@@ -45,14 +49,17 @@ class RegisterUserTag(object):
                 tag_id = item.tag_id
                 user_id = item.user_id
 
-                user_tag_list = CommunityTagFollow.objects.filter(user=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True)
+                user_tag_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True))
                 user_id_dict[user_id] = user_tag_list
 
+                user_register_tag_list = list(AccountUserTag.objects.filter(user=user_id,is_deleted=False).values_list("tag_id", flat=True))
+                user_id_dict[user_id].extend(user_register_tag_list)
+
             for user_id in user_id_dict:
-                redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
-                redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
-                redis_user_tag_id_list.extend(user_id_dict[user_id])
-                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(redis_user_tag_id_list))))
+                # redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
+                # redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
+                # redis_user_tag_id_list.extend(user_id_dict[user_id])
+                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(user_id_dict[user_id]))))
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
@@ -65,60 +72,22 @@ class RegisterUserTag(object):
         :return:
         """
         try:
-            # user_id_set = set()
             user_id_dict = dict()
             query_results = AccountUserTag.objects.filter(pk__in=pk_list)
             for item in query_results:
                 tag_id = item.tag_id
                 user_id = item.user
-                user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id", flat=True)
+                user_tag_list = list(AccountUserTag.objects.filter(user=user_id).values_list("tag_id", flat=True))
                 user_id_dict[user_id] = user_tag_list
 
+                user_follow_tag_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True))
+                user_id_dict[user_id].extend(user_follow_tag_list)
+
             for user_id in user_id_dict:
-                redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
-                redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
-                redis_user_tag_id_list.extend(user_id_dict[user_id])
-                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(redis_user_tag_id_list))))
+                # redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
+                # redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
+                # redis_user_tag_id_list.extend(user_id_dict[user_id])
+                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(user_id_dict[user_id]))))
 
-        #         if user_id not in user_id_set:
-        #             user_id_set.add(user_id)
-        #
-        #             user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True)
-        #
-        #             have_read_topic_id_list = Tools.get_have_read_topic_id_list(-1, user_id,
-        #                                                                         TopicPageType.HOME_RECOMMEND)
-        #             recommend_topic_id_list = list()
-        #             cycle_num = int(10000/len(user_tag_list))
-        #             for index in range(0,cycle_num):
-        #                 for tag_id in user_tag_list:
-        #                     redis_tag_id_key = cls.tag_topic_id_redis_prefix + str(tag_id)
-        #                     redis_tag_id_data = redis_client.get(redis_tag_id_key)
-        #                     tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
-        #                     if not redis_tag_id_data:
-        #                         tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
-        #                         redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
-        #                         redis_client.expire(redis_tag_id_key,1*24*60*60)
-        #
-        #                     if len(tag_topic_id_list)>index:
-        #                         for topic_id in tag_topic_id_list[index:]:
-        #                             if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list:
-        #                                 recommend_topic_id_list.append(topic_id)
-        #                                 break
-        #
-        #             redis_register_tag_topic_data = {
-        #                 "data": json.dumps(recommend_topic_id_list),
-        #                 "cursor": 0
-        #             }
-        #             redis_client.hmset(cls.linucb_user_id_register_tag_topic_id_prefix,redis_register_tag_topic_data)
-        #             redis_client.expire(cls.linucb_user_id_register_tag_topic_id_prefix,30*24*60*60)
-        #
-        #             topic_recommend_redis_key = cls.linucb_user_id_recommend_topic_id_prefix + str(user_id)
-        #             redis_data_dict = {
-        #                 "data": json.dumps(recommend_topic_id_list),
-        #                 "cursor":0
-        #             }
-        #             redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
-        #             redis_client.expire(topic_recommend_redis_key,30*24*60*60)
-        #
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

search/views/tag.py

@@ -10,7 +10,7 @@ from search.utils.topic import TopicUtils
 from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.common import *
-from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow
+from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow,Tag
 import time
 
 
@@ -113,13 +113,16 @@ def choice_push_tag(device_id, user_id):
 
         now_sec = int(time.time())
         valid_time = 8*7*24*60*60
-        for tag_id in redis_push_tag_dict:
+
+        ori_key_list = list(redis_push_tag_dict.keys())
+        for tag_id in ori_key_list:
             if now_sec - redis_push_tag_dict[tag_id] >= valid_time:
-                redis_push_tag_dict.popitem(tag_id)
+                redis_push_tag_dict.pop(tag_id)
 
         redis_push_tag_list = list(redis_push_tag_dict.keys())
-        account_user_tag_list = list(AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True))
-        community_tag_follow_list = list(CommunityTagFollow.objects.filter(user_id=user_id).values_list("tag_id",flat=True))
+        redis_push_tag_list = [int(item) for item in redis_push_tag_list]
+        account_user_tag_list = list(AccountUserTag.objects.filter(user=user_id,is_deleted=False).values_list("tag_id",flat=True))
+        community_tag_follow_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_online=True,is_deleted=False).values_list("tag_id",flat=True))
 
         linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
         tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
@@ -130,11 +133,12 @@ def choice_push_tag(device_id, user_id):
         account_user_tag_list.extend(linucb_recommend_tag_list)
         unread_tag_list = list(set(account_user_tag_list) - set(redis_push_tag_list))
 
+        unread_tag_list = list(Tag.objects.filter(id__in=unread_tag_list, is_online=True, is_deleted=False).values_list("id",flat=True))
         ret_tag_set = set()
         if len(unread_tag_list)>0:
             for tag_id in unread_tag_list:
                 valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id,is_online=True).count()
-                if valid_tag_topic_num>0:
+                if valid_tag_topic_num>100:
                     ret_tag_set.add(tag_id)
                     redis_push_tag_dict[tag_id] = now_sec
                     if len(ret_tag_set)>=1:

trans2es/mapping/topic-high-star.json

@@ -52,6 +52,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic-star-routing.json

@@ -52,6 +52,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/mapping/topic.json

@@ -52,6 +52,7 @@
         "search_analyzer": "gm_default_index"
     },
     "is_excellent":{"type": "long"},
-    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
+    "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
+    "is_history": {"type": "boolean"} //是否历史数据
   }
 }

trans2es/type_info.py

@@ -84,7 +84,7 @@ class TypeInfo(object):
 
     def bulk_get_data(self, instance_iterable):
         data_list = []
-        # 4，5星帖子单独索引
+        # 4星以上帖子单独索引
         topic_data_high_star_list = list()
 
         if self.batch_get_data_func:
@@ -154,13 +154,16 @@ class TypeInfo(object):
                                     old_data["is_online"] = False
                                     old_data["is_deleted"] = True
                                     old_data["content_level"] = int_ori_topic_star
+                                    old_data["is_history"] = True
 
                                     data_list.append(old_data)
+                                    if int_ori_topic_star>=4:
+                                        topic_data_high_star_list.append(old_data)
                                     redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
-                        data_list.append(data)
 
-                    # if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
-                    #     topic_data_high_star_list.append(data)
+                            if data["content_level"] and int(data["content_level"])>=4:
+                                topic_data_high_star_list.append(data)
+                        data_list.append(data)
 
         return (data_list,topic_data_high_star_list)
 
@@ -234,13 +237,13 @@ class TypeInfo(object):
                 es=es,
             )
 
-        # # 同时写4星及以上的帖子
-        # if len(topic_data_high_star_list)>0:
-        #     self.elasticsearch_bulk_insert_data(
-        #         sub_index_name="topic-high-star",
-        #         data_list=topic_data_high_star_list,
-        #         es=es,
-        #     )
+        # 同时写4星及以上的帖子
+        if len(topic_data_high_star_list)>0:
+            self.elasticsearch_bulk_insert_data(
+                sub_index_name="topic-high-star",
+                data_list=topic_data_high_star_list,
+                es=es,
+            )
 
         end = time.time()
         time3=end-begin