Merge branch 'master' into 'test'

Master

See merge request !289

injection/data_sync/tasks.py

@@ -20,6 +20,8 @@ def write_to_es(es_type, pk_list, use_batch_query_set=False):
 
         if es_type == "register_user_tag":
             RegisterUserTag.get_register_user_tag(pk_list)
+        elif es_type == "attention_user_tag":
+            RegisterUserTag.get_user_attention_tag(pk_list)
         else:
             type_info_map = get_type_info_map()
             type_info = type_info_map[es_type]

libs/es.py

@@ -264,7 +264,7 @@ class ESPerform(object):
             return True
 
     @classmethod
-    def get_tag_topic_list(cls,tag_id,have_read_topic_id_list):
+    def get_tag_topic_list(cls,tag_id,have_read_topic_id_list,size=100):
         try:
             functions_list = list()
             for id in tag_id:
@@ -274,13 +274,27 @@ class ESPerform(object):
                         "weight": 1
                     }
                 )
+            functions_list += [
+                {
+                    "filter": {"term": {"content_level": 6}},
+                    "weight": 6000
+                },
+                {
+                    "filter": {"term": {"content_level": 5}},
+                    "weight": 5000
+                },
+                {
+                    "filter": {"term": {"content_level": 4}},
+                    "weight": 4000
+                }
+            ]
             q = {
                 "query":{
                     "function_score":{
                         "query": {
                             "bool": {
                                 "must": [
-                                    {"range": {"content_level": {"gte": 3, "lte": 5}}},
+                                    {"range": {"content_level": {"gte": 4, "lte": 6}}},
                                     {"term": {"is_online": True}},
                                     {"term": {"is_deleted": False}},
                                     {"terms": {"tag_list": tag_id}}
@@ -308,7 +322,7 @@ class ESPerform(object):
                     }
                 }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
-                                                       offset=0, size=100,routing="3,4,5")
+                                                       offset=0, size=size,routing="4,5,6")
 
             topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
             logging.info("topic_id_list:%s"%str(topic_id_list))

libs/tools.py

@@ -25,10 +25,10 @@ def tzlc(dt, truncate_to_sec=True):
 
 def get_have_read_topic_id_list(device_id,user_id,query_type):
     try:
-        if user_id==-1:
-            redis_key = "physical:home_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(query_type)
-        else:
+        if user_id and int(user_id)>0:
             redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
+        else:
+            redis_key = "physical:home_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(query_type)
 
 
         have_read_topic_id_list = list()

linucb/utils/register_user_tag.py

@@ -9,7 +9,7 @@ import traceback
 import json
 import pickle
 from django.conf import settings
-from trans2es.models.tag import AccountUserTag
+from trans2es.models.tag import AccountUserTag,CommunityTagFollow
 from libs.es import ESPerform
 import libs.tools as Tools
 from search.utils.common import *
@@ -30,8 +30,40 @@ class RegisterUserTag(object):
     linucb_user_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:user_id:"
 
     linucb_register_user_tag_key = "physical:linucb:register_user_tag_info"
+
+    @classmethod
+    def get_user_attention_tag(cls, pk_list):
+        """
+        :remark 获取用户关注标签
+        :param pk_list:
+        :return:
+        """
+        try:
+            user_id_dict = dict()
+            query_results = CommunityTagFollow.objects.filter(pk__in=pk_list,is_deleted=False,is_online=True)
+            for item in query_results:
+                tag_id = item.tag_id
+                user_id = item.user_id
+
+                user_tag_list = CommunityTagFollow.objects.filter(user=user_id,is_deleted=False,is_online=True).values_list("tag_id", flat=True)
+                user_id_dict[user_id] = user_tag_list
+
+            for user_id in user_id_dict:
+                redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
+                redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
+                redis_user_tag_id_list.extend(user_id_dict[user_id])
+                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(redis_user_tag_id_list))))
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+
+
     @classmethod
     def get_register_user_tag(cls,pk_list):
+        """
+        :remark 用户注册时选的标签
+        :param pk_list:
+        :return:
+        """
         try:
             # user_id_set = set()
             user_id_dict = dict()
@@ -43,7 +75,10 @@ class RegisterUserTag(object):
                 user_id_dict[user_id] = user_tag_list
 
             for user_id in user_id_dict:
-                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(user_id_dict[user_id])))
+                redis_user_tag_id_data = redis_client.hget(cls.linucb_register_user_tag_key, user_id)
+                redis_user_tag_id_list = json.loads(redis_user_tag_id_data) if redis_user_tag_id_data else []
+                redis_user_tag_id_list.extend(user_id_dict[user_id])
+                redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(set(redis_user_tag_id_list))))
 
         #         if user_id not in user_id_set:
         #             user_id_set.add(user_id)

search/utils/topic.py

@@ -124,7 +124,6 @@ class TopicUtils(object):
                                 filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
                                 user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[]):
         """
-        :需增加打散逻辑
         :remark:获取首页推荐帖子列表
         :param user_id:
         :param offset:
@@ -197,25 +196,6 @@ class TopicUtils(object):
                         "weight": 30,
                     }
                 )
-            # if len(pick_user_id_list) > 0:
-            #     functions_list.append(
-            #         {
-            #             "filter": {"bool": {
-            #                 "should": {"terms": {"user_id": pick_user_id_list}}}},
-            #             "weight": 2
-            #         }
-            #     )
-
-            # if len(same_pictorial_id_list) > 0:
-            #     functions_list.append(
-            #         {
-            #             "filter": {"bool": {
-            #                 "should": {"terms": {"user_id": same_pictorial_id_list}}}},
-            #             "weight": 1
-            #         }
-            #     )
-
-            # query_tag_term_list = cls.___get_should_term_list(user_tag_list)
             if len(attention_tag_list) > 0:
                 functions_list.append(
                     {
@@ -224,40 +204,11 @@ class TopicUtils(object):
                         "weight": 100
                     }
                 )
-            # if len(recommend_tag_list)>0:
-            #     if len(recommend_tag_list)>1:
-            #         functions_list += [
-            #             {
-            #                 "filter": {"term": {"tag_list": recommend_tag_list[0]}},
-            #                 "weight": 4
-            #             },
-            #             {
-            #                 "filter": {"terms": {"tag_list": recommend_tag_list[1:]}},
-            #                 "weight": 3
-            #             }
-            #         ]
-            #     else:
-            #         functions_list.append(
-            #             {
-            #                 "filter": {"terms": {"tag_list": recommend_tag_list}},
-            #                 "weight": 3
-            #             }
-            #         )
-
-            # for tag_id in recommend_tag_dict:
-            #     functions_list.append(
-            #         {
-            #             "filter": {"term": {"tag_list": tag_id}},
-            #             "weight": recommend_tag_dict[tag_id]
-            #         }
-            #     )
-
-            # low_content_level = 4 if query_type == TopicPageType.FIND_PAGE else 3
             query_function_score = {
                 "query": {
                     "bool": {
                         "filter": [
-                            {"range": {"content_level": {"gte": 4, "lte": 5}}},
+                            {"range": {"content_level": {"gte": 4, "lte": 6}}},
                             # {"term": {"has_image":True}},
                             {"term": {"is_online": True}},
                             {"term": {"is_deleted": False}}
@@ -292,11 +243,9 @@ class TopicUtils(object):
                     }
                 }
             if len(filter_topic_id_list) > 0:
-                query_function_score["query"]["bool"]["must_not"] = {
-                    "terms": {
-                        "id": filter_topic_id_list
-                    }
-                }
+                query_function_score["query"]["bool"]["must_not"] = [
+                    {"terms":{"id":filter_topic_id_list}}
+                ]
 
             if query is not None:  # 搜索帖子
                 multi_fields = {
@@ -318,7 +267,15 @@ class TopicUtils(object):
                     {"term": {"tag_list": tag_id}}
                 ]
                 query_function_score["query"]["bool"]["minimum_should_match"] = 1
-
+            else:
+                if "must_not" in query_function_score["query"]["bool"]:
+                    query_function_score["query"]["bool"]["must_not"] += [
+                        {"term": {"is_operation_home_recommend": True}}
+                    ]
+                else:
+                    query_function_score["query"]["bool"]["must_not"] = [
+                        {"term": {"is_operation_home_recommend": True}}
+                    ]
             q["query"]["function_score"] = query_function_score
             q["collapse"] = {
                 "field": "user_id"
@@ -350,37 +307,9 @@ class TopicUtils(object):
                                                        offset=offset, size=size,routing=routing)
 
             topic_id_list = list()
-            same_group_id_set = set()
-            same_user_id_set = set()
 
             for item in result_dict["hits"]:
                 topic_id_list.append(item["_source"]["id"])
-            # for item in result_dict["hits"]:
-            #     if item["_source"]["group_id"]>0 and  item["_source"]["group_id"] not in same_group_id_set:
-            #         same_group_id_set.add(item["_source"]["id"])
-            #         topic_id_list.append(item["_source"]["id"])
-            #     else:
-            #         same_group_id_set.add(item["_source"]["id"])
-            #
-            #     if item["_source"]["user_id"] not in same_user_id_set:
-            #         same_user_id_set.add(item["_source"]["id"])
-            #         topic_id_list.append(item["_source"]["id"])
-            #     else:
-            #         same_user_id_set.add(item["_source"]["id"])
-            #
-            #     if len(topic_id_list) >= single_size:
-            #         break
-            #
-            # if len(topic_id_list) < single_size:
-            #     for topic_id in same_group_id_set:
-            #         topic_id_list.append(topic_id)
-            #         if len(topic_id_list)>=single_size:
-            #             break
-            #     for topic_id in same_user_id_set:
-            #         topic_id_list.append(topic_id)
-            #         if len(topic_id_list)>=single_size:
-            #             break
-            logging.info("topic_id_list:%s,attention_tag_list%s" % (str(topic_id_list),str(attention_tag_list)))
             return topic_id_list
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

search/views/tag.py

@@ -134,10 +134,10 @@ def choice_push_tag(device_id, user_id):
         if len(unread_tag_list)>0:
             for tag_id in unread_tag_list:
                 valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id,is_online=True).count()
-                if valid_tag_topic_num>100:
+                if valid_tag_topic_num>0:
                     ret_tag_set.add(tag_id)
                     redis_push_tag_dict[tag_id] = now_sec
-                    if len(ret_tag_set)>=2:
+                    if len(ret_tag_set)>=1:
                         break
 
         redis_client.set(redis_push_tag_key, json.dumps(redis_push_tag_dict))

trans2es/mapping/topic-high-star.json

@@ -51,6 +51,7 @@
         "analyzer": "gm_default_index",
         "search_analyzer": "gm_default_index"
     },
-    "is_excellent":{"type": "long"}
+    "is_excellent":{"type": "long"},
+    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
   }
 }

trans2es/mapping/topic-star-routing.json

@@ -51,6 +51,7 @@
         "analyzer": "gm_default_index",
         "search_analyzer": "gm_default_index"
     },
-    "is_excellent":{"type": "long"}
+    "is_excellent":{"type": "long"},
+    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
   }
 }

trans2es/mapping/topic-v1.json

 {
   "dynamic":"strict",
+  "_routing": {"required": true},
   "properties": {
     "id":{"type":"long"},
     "is_online":{"type":"boolean"},//上线
     "is_deleted":{"type":"boolean"},
     "vote_num":{"type":"long"},
-    "total_vote_num":{"type":"long","default":0},
+    "total_vote_num":{"type":"long"},
     "reply_num":{"type":"long"},
     "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
     "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
@@ -50,6 +51,7 @@
         "analyzer": "gm_default_index",
         "search_analyzer": "gm_default_index"
     },
-    "is_excellent":{"type": "long"}
+    "is_excellent":{"type": "long"},
+    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
   }
 }

trans2es/mapping/topic.json

@@ -51,6 +51,7 @@
         "analyzer": "gm_default_index",
         "search_analyzer": "gm_default_index"
     },
-    "is_excellent":{"type": "long"}
+    "is_excellent":{"type": "long"},
+    "is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
   }
 }

trans2es/models/topic.py

@@ -197,8 +197,8 @@ class Topic(models.Model):
                 offline_score += 6.0
             elif self.content_level == '4':
                 offline_score += 5.0
-            elif self.content_level == '3':
-                offline_score += 2.0
+            elif self.content_level == '6':
+                offline_score += 100.0
 
             # exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
             # click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
@@ -289,4 +289,5 @@ class TopicHomeRecommend(models.Model):
         db_table = "topic_home_recommend"
     id = models.IntegerField(verbose_name=u"id",primary_key=True)
     topic_id = models.IntegerField(verbose_name=u"帖子ID")
-    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_online = models.BooleanField(verbose_name=u'是否上线')
\ No newline at end of file
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')

trans2es/utils/topic_transfer.py

@@ -9,7 +9,7 @@ import time
 import re
 import datetime
 from trans2es.models.user import User
-from trans2es.models.topic import ExcellentTopic
+from trans2es.models.topic import ExcellentTopic,TopicHomeRecommend
 
 class TopicTransfer(object):
 
@@ -121,6 +121,11 @@ class TopicTransfer(object):
                 else:
                     res["is_excellent"] = 0
 
+            res["is_operation_home_recommend"] = False
+            operation_home_recommend = TopicHomeRecommend.objects.filter(topic_id=instance.id).first()
+            if operation_home_recommend and operation_home_recommend.is_online and not operation_home_recommend.is_deleted:
+                res["is_operation_home_recommend"] = True
+
             logging.info("test topic transfer time cost,time0:%d,time1:%d,time2:%d,time3:%d,time4:%d" % (time0,time1,time2,time3,time4))
             return res
         except: