Merge branch 'master' into 'test'

Master

See merge request !99

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*~
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.vscode
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+.idea/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+
+# config
+fabfile.py
+settings.online.py
+settings_local.py
+media/
+log/
+crawldata/
+conf/
+/static
+.vagrant/
+Vagrantfile
+
+*.DS_Store
+dump.rdb

requirements.txt

@@ -11,6 +11,7 @@ PyMySQL==0.9.2
 
 gunicorn==19.9.0
 gevent==1.3.7
+pypinyin==0.34.1
 
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git@master
 git+ssh://git@git.wanmeizhensuo.com/backend/helios.git@master
@@ -19,5 +20,6 @@ git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git@v0.1.3#egg=gm-config==
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-protocol.git@master
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-upload.git@master
 git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
-git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@dev
+git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git@master
+

search/utils/common.py

@@ -26,7 +26,12 @@ class TopicDocumentField(object):
     IS_ONLINE = "is_online",
     TAG_LIST = "tag_list"
 
+class TopicPageType(object):
+    # 首页推荐
+    HOME_RECOMMEND=1
 
+    # 发现页面
+    FIND_PAGE=2
 
 def time_consuming_decorator(func):
     def time_consuming(*args, **kwargs):

search/utils/group.py

@@ -38,7 +38,7 @@ class GroupUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
 
             return ESPerform.get_search_results(ESPerform.get_cli(), "group", q, offset, size)
@@ -47,8 +47,11 @@ class GroupUtils(object):
             return {"total_count":0, "hits":[]}
 
     @classmethod
-    def get_hot_group_recommend_result_list(cls,offset,size):
+    def get_hot_group_recommend_result_list(cls,offset,size,es_cli_obj=None):
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = {
                 "bool":{
@@ -62,10 +65,10 @@ class GroupUtils(object):
                 {"high_quality_topic_num":{"order":"desc"}}
             ]
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"group",q,offset,size)
+            result_dict = ESPerform.get_search_results(es_cli_obj,"group",q,offset,size)
 
             group_ids_list = []
             if len(result_dict["hits"]) > 0:
@@ -77,12 +80,15 @@ class GroupUtils(object):
             return []
 
     @classmethod
-    def get_user_attention_group_list(cls,user_id,offset=0,size=10):
+    def get_user_attention_group_list(cls,user_id,offset=0,size=10,es_cli_obj=None):
         """
         :remark: 获取用户关注小组列表
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = {
                 "bool":{
@@ -94,10 +100,10 @@ class GroupUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["attention_group_id_list"]
+                "includes":["attention_group_id_list"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"user",q,offset,size)
+            result_dict = ESPerform.get_search_results(es_cli_obj,"user",q,offset,size)
             if len(result_dict["hits"])>0:
                 return result_dict["hits"][0]["_source"]["attention_group_id_list"]
             else:
@@ -107,13 +113,16 @@ class GroupUtils(object):
             return []
 
     @classmethod
-    def get_group_ids_by_aggs(cls,group_id_list):
+    def get_group_ids_by_aggs(cls,group_id_list,es_cli_obj=None):
         """
         :remark:聚合查询获取小组列表
         :param group_id_list:
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["size"]=0
             q["query"] = {
@@ -136,7 +145,7 @@ class GroupUtils(object):
                 }
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"topic",q,aggregations_query=True)
+            result_dict = ESPerform.get_search_results(es_cli_obj,"topic",q,aggregations_query=True)
             buckets_list = result_dict["aggregations"]["group_ids"]["buckets"]
 
             sorted_buckets_list = sorted(buckets_list,key=lambda item:item["max_date"]["value"],reverse=True)

search/utils/pick.py

@@ -77,7 +77,7 @@ class PickUtils(object):
                 }
             }
             q["_source"] = {
-                "include": ["id"]
+                "includes": ["id"]
             }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
             recommend_pick_celebrity_list = list()
@@ -123,7 +123,7 @@ class PickUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
 
@@ -153,7 +153,7 @@ class PickUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
 

search/utils/topic.py

@@ -4,8 +4,13 @@
 import logging
 import traceback
 import json
+
+from alpha_types.venus import TOPIC_SEARCH_SORT
+
 from libs.es import ESPerform
 from .common import TopicDocumentField
+from search.utils.common import *
+
 
 
 class TopicUtils(object):
@@ -87,7 +92,7 @@ class TopicUtils(object):
             return []
 
     @classmethod
-    def get_recommend_topic_ids(cls,user_id,offset,size,query=None):
+    def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[]):
         """
         :需增加打散逻辑
         :remark:获取首页推荐帖子列表
@@ -130,9 +135,19 @@ class TopicUtils(object):
                     "gauss": {
                         "update_time": {
                             "scale": "1d",
-                            "decay": 0.5
+                            "decay": 0.9
                         }
+                    },
+                    "weight": 5
+                },
+                {
+                    "gauss": {
+                        "create_time": {
+                            "scale": "1d",
+                            "decay": 0.9
                         }
+                    },
+                    "weight": 10
                 }
             ]
 
@@ -171,18 +186,28 @@ class TopicUtils(object):
                     }
                 )
 
+            low_content_level = 4 if query_type==TopicPageType.FIND_PAGE else 3
             query_function_score = {
                 "query": {
                     "bool": {
-                        "must": {
-                            "range": {"content_level": {"gte": 3, "lte": 5}}
-                        }
+                        "filter": [
+                            {"range": {"content_level": {"gte": low_content_level, "lte": 5}}},
+                            {"term": {"has_image":True}},
+                            {"term": {"is_online": True}},
+                            {"term": {"is_deleted": False}}
+                        ]
                     }
                 },
                 "score_mode": "sum",
                 "boost_mode": "sum",
                 "functions": functions_list
             }
+            if query_type==TopicPageType.FIND_PAGE and len(filter_topic_id_list)>0:
+                query_function_score["query"]["bool"]["must_not"] = {
+                    "terms":{
+                        "id": filter_topic_id_list
+                    }
+                }
 
             if query is not None:#搜索帖子
                 multi_fields = {
@@ -205,33 +230,37 @@ class TopicUtils(object):
 
             q["query"]["function_score"] = query_function_score
             q["_source"] = {
-                "include":["id","group_id","offline_score"]
+                "includes":["id","group_id","offline_score"]
             }
             q["sort"] = [
                 {
                     "_script":{
                         "type":"number",
                         "script":{
-                            "lang":"painless",
-                            "source":"_score+params._source.offline_score"
+                            "lang": "expression",
+                            "source": "_score*doc['offline_score']"
+                            # "lang":"painless",
+                            # "source":"_score+params._source.offline_score"
                         },
                         "order":"desc"
                     }
-                }
+                },
+                "_score"
             ]
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
                                                        offset=offset, size=size)
 
-            if len(result_dict["hits"])>0:
-                return [item["_source"] for item in result_dict["hits"]]
-            else:
-                return []
+            topic_id_dict = dict()
+            for item in result_dict["hits"]:
+                topic_id_dict[item["_source"]["id"]] = item["_source"]["group_id"]
+
+            return topic_id_dict
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return []
+            return dict()
 
     @classmethod
-    def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,offset,size):
+    def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size,es_cli_obj=None):
         """
         :remark 帖子详情页推荐列表,缺少按时间衰减
         :param user_id:
@@ -243,6 +272,9 @@ class TopicUtils(object):
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = dict()
 
@@ -274,9 +306,9 @@ class TopicUtils(object):
             query_function_score = {
                 "query":{
                     "bool":{
-                        "must": {
-                            "range": {"content_level": {"gte": 3, "lte": 5}}
-                        },
+                        "must": [
+                            {"range": {"content_level": {"gte": 3, "lte": 5}}}
+                        ],
                         "must_not":{
                             "term":{
                                 "id":topic_id
@@ -289,6 +321,8 @@ class TopicUtils(object):
                 "functions": functions_list
             }
 
+            if filter_topic_user_id:
+                query_function_score["query"]["bool"]["must"].append({"term": {"user_id": topic_user_id}})
             if len(topic_tag_list)>0:
                 query_function_score["query"]["bool"]["should"]={
                     "terms":{
@@ -297,10 +331,10 @@ class TopicUtils(object):
                 }
             q["query"]["function_score"] = query_function_score
             q["_source"] = {
-                "include":["id","group_id","user_id","_score"]
+                "includes":["id","group_id","user_id","_score"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
+            result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="topic", query_body=q,
                                                        offset=offset, size=size)
 
             return result_dict["hits"]
@@ -309,13 +343,16 @@ class TopicUtils(object):
             return []
 
     @classmethod
-    def get_topic_tag_id_list(cls,topic_id):
+    def get_topic_tag_id_list(cls,topic_id,es_cli_obj=None):
         """
         :remark 获取帖子标签列表
         :param topic_id:
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = {
                 "term":{
@@ -323,10 +360,10 @@ class TopicUtils(object):
                 }
             }
             q["_source"] = {
-                "include":[TopicDocumentField.TAG_LIST]
+                "includes":[TopicDocumentField.TAG_LIST]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,size=1)
+            result_dict = ESPerform.get_search_results(es_cli_obj,sub_index_name="topic",query_body=q,size=1)
 
             tag_id_list = []
             if len(result_dict["hits"])>0:
@@ -406,15 +443,17 @@ class TopicUtils(object):
                 "function_score":query_function_score
             }
             q["_source"] = {
-                "include":["id","group_id","user_id","_score","offline_score","manual_score"]
+                "includes":["id","group_id","user_id","_score","offline_score","manual_score"]
             }
             q["sort"] = [
                 {
                     "_script":{
                         "type":"number",
                         "script":{
-                            "lang":"painless",
-                            "source":"_score+params._source.offline_score+params._source.manual_score"
+                            "lang": "expression",
+                            "source": "_score+doc['offline_score']+doc['manual_score']"
+                            # "lang":"painless",
+                            # "source":"_score+params._source.offline_score+params._source.manual_score"
                         },
                         "order":"desc"
                     }
@@ -428,3 +467,81 @@ class TopicUtils(object):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return list()
+
+    @classmethod
+    def process_filters(cls, filters):
+        """处理过滤器部分。"""
+
+        f = [
+            {"term": {"is_online": True}},
+            {"term": {"is_deleted": False}},
+        ]
+        if not filters:
+            return f
+
+        for k, v in filters.items():
+            if k == "group_id":
+                f.append({
+                    "term": {"group_id": v},
+                })
+
+        return f
+
+    @classmethod
+    def process_nfilters(cls, nfilters):
+        """处理过滤器部分。"""
+
+        nf = []
+        if not nfilters:
+            return nf
+
+        for k, v in nfilters.items():
+            pass
+
+        return nf
+
+    @classmethod
+    def process_sort(cls, sorts_by):
+        """处理排序部分。"""
+
+        sort_rule = []
+        if sorts_by == TOPIC_SEARCH_SORT.VOTE_NUM:
+            sort_rule.append({
+                "vote_num":{
+                    "order":"desc"
+                },
+                "update_time":{
+                    "order":"desc"
+                },
+            })
+
+        return sort_rule
+
+    @classmethod
+    def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10):
+
+        q = {
+            "query": {
+                "bool": {
+                    "must": cls.process_filters(filters),
+                    "must_not": cls.process_nfilters(nfilters),
+                }
+            },
+            "_source": {
+                "includes":["id"]
+            },
+            "sort": [],
+        }
+
+        if sorts_by:
+            sorts = cls.process_sort(sorts_by)
+            q["sort"] = sorts
+
+        try:
+            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
+                                                       query_body=q, offset=offset, size=size)
+
+            return result_dict["hits"]
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return []

search/utils/user.py

@@ -5,18 +5,72 @@ import logging
 import traceback
 import json
 import time
+import copy
 from libs.es import ESPerform
 
 
 class UserUtils(object):
     @classmethod
-    def get_attention_user_list(cls,user_id_list,self_user_id):
+    def get_batch_attention_user_list(cls,user_id_list,self_user_id,es_cli_obj=None):
+        """
+        :remark 批量用户 关注的 用户列表
+        :param user_id_list:
+        :param self_user_id:
+        :return:
+        """
+        try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
+            user_id_list.append(self_user_id)
+
+            q = dict()
+            q["query"] = {
+                "terms":{
+                    "user_id":user_id_list
+                }
+            }
+            q["_source"] = {
+                "includes":["attention_user_id_list","user_id"]
+            }
+
+            result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=len(user_id_list))
+
+            self_attention_user_id_list = []
+            attention_user_dict_list = list()
+            ret_attention_user_id_list = list()
+            for hit_item in result_dict["hits"]:
+
+                attention_user_dict = dict()
+                user_id = hit_item["_source"]["user_id"]
+
+                if user_id == self_user_id:
+                    self_attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
+                else:
+                    attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
+
+                    attention_user_dict[user_id] = attention_user_id_list
+                    attention_user_dict_list.append(attention_user_dict)
+                    ret_attention_user_id_list.append(user_id)
+
+            return [self_attention_user_id_list,attention_user_dict_list,ret_attention_user_id_list]
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return ([],[],[])
+
+
+    @classmethod
+    def get_attention_user_list(cls,user_id_list,self_user_id,es_cli_obj=None):
         """
         :remark 获取指定用户列表 关注的 用户列表
         :param user_id:
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
+
             q = dict()
             q["query"] = {
                 "terms":{
@@ -24,10 +78,10 @@ class UserUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["attention_user_id_list","user_id"]
+                "includes":["attention_user_id_list","user_id"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset=0, size=len(user_id_list))
+            result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=len(user_id_list))
 
             self_attention_user_id_list = []
             recursion_attention_user_id_list = []
@@ -59,7 +113,7 @@ class UserUtils(object):
             return []
 
     @classmethod
-    def get_recommend_user_list(cls,self_attention_user_id_list,recursion_attention_user_id_list,offset,size):
+    def get_recommend_user_list(cls,self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj=None):
         """
         :remark 获取推荐用户列表
         :param attention_user_id_list:
@@ -67,6 +121,9 @@ class UserUtils(object):
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = dict()
 
@@ -118,9 +175,9 @@ class UserUtils(object):
             }
             q["query"]["function_score"] = query_function_score
             q["_source"] = {
-                "include":["user_id"]
+                "includes":["user_id"]
             }
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="user", query_body=q,
+            result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="user", query_body=q,
                                                        offset=offset, size=size)
 
             recommend_user_list = list()
@@ -131,3 +188,109 @@ class UserUtils(object):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return []
+
+
+    @classmethod
+    def get_batch_recommend_user_dict(cls,need_filter_attention_user_id_list,attention_user_id_list,attention_user_dict_list,self_user_id,offset,size,es_cli_obj=None):
+        """
+        :remark 获取批量推荐用户
+        :param need_filter_attention_user_id_list:
+        :param attention_user_dict:
+        :param self_user_id:
+        :param offset:
+        :param size:
+        :return:
+        """
+        try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
+            batch_query_list = list()
+            for interesting_user_item_dict in attention_user_dict_list:
+                for interesting_user_id in interesting_user_item_dict:
+                    if interesting_user_id != self_user_id:
+                        logging.info("duan add,interesting_user_id:%d" % interesting_user_id)
+                        filter_user_id_list = copy.deepcopy(need_filter_attention_user_id_list)
+                        filter_user_id_list.append(interesting_user_id)
+
+                        q = dict()
+                        q["from"] = offset
+                        q["size"] = size
+                        q["query"] = dict()
+
+                        functions_list = [
+                            {
+                                "gauss": {
+                                    "latest_topic_time_val": {
+                                        "origin": int(time.time()),
+                                        "scale": "600",
+                                        "decay": 0.1
+                                    }
+                                }
+                            }
+                        ]
+
+                        if len(interesting_user_item_dict[interesting_user_id]) > 0:
+                            functions_list.append(
+                                {
+                                    "filter":{
+                                        "bool":{
+                                            "should":{"terms":{"user_id":interesting_user_item_dict[interesting_user_id]}}
+                                        }
+                                    },
+                                    "weight":10
+                                }
+                            )
+
+                        query_function_score = {
+                            "query": {
+                                "bool": {
+                                    "must": [
+                                        {"term": {"is_recommend": True}},
+                                        {"term": {"is_online": True}},
+                                        {"term": {"is_deleted": False}},
+                                        {"term": {"is_shadow": False}}
+                                    ],
+                                    "must_not":{
+                                        "terms":{
+                                            "user_id":filter_user_id_list
+                                        }
+                                    }
+                                }
+                            },
+                            "score_mode": "sum",
+                            "boost_mode": "sum",
+                            "functions": functions_list
+                        }
+                        q["query"]["function_score"] = query_function_score
+                        q["_source"] = {
+                            "includes":["user_id"]
+                        }
+
+                        batch_query_list.append(q)
+
+            index_name = ESPerform.get_official_index_name("user","read")
+            search_header_dict = {'index': index_name, 'type': "_doc"}
+            query_body = ""
+            for query_item in batch_query_list:
+                query_body += "{}\n{}\n".format(json.dumps(search_header_dict),json.dumps(query_item))
+
+            result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="user", query_body=query_body,
+                                                       batch_search=True)
+            ret_dict = dict()
+            user_index = 0
+            for res_item in result_dict["responses"]:
+                recommend_user_list = list()
+                for item in res_item["hits"]["hits"]:
+                    recommend_user_list.append(item["_source"]["user_id"])
+
+                logging.info("duan add,attention_user_id_list:%s,user_index:%d" % (str(attention_user_id_list), user_index))
+                ret_dict[str(attention_user_id_list[user_index])] = recommend_user_list
+                user_index += 1
+
+
+            logging.info("duan add,ret_dict:%s" % str(ret_dict))
+            return ret_dict
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return dict()

search/views/group.py

@@ -9,6 +9,7 @@ from libs.cache import redis_client
 from libs.es import ESPerform
 from search.utils.group import GroupUtils
 from search.utils.common import GroupSortTypes
+from libs.es import ESPerform
 
 
 @bind("physical/search/query_group")
@@ -46,18 +47,21 @@ def group_sort(user_id=-1,sort_type=GroupSortTypes.HOT_RECOMMEND,offset=0,size=1
         if not isinstance(user_id,int):
             user_id = -1
 
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
         if sort_type==GroupSortTypes.HOT_RECOMMEND:
-            group_ids_list = GroupUtils.get_hot_group_recommend_result_list(offset,size)
+            group_ids_list = GroupUtils.get_hot_group_recommend_result_list(offset,size,es_cli_obj)
 
             return {"group_recommend_ids":group_ids_list}
 
         elif sort_type==GroupSortTypes.ATTENTION_RECOMMEND:
-            attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset=0,size=1)
+            attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset=0,size=1,es_cli_obj=es_cli_obj)
             if len(attention_group_list)==0:
                 return {"group_recommend_ids": []}
             else:
                 attention_group_id_list = [item["group_id"] for item in attention_group_list]
-                sorted_group_ids_list = GroupUtils.get_group_ids_by_aggs(attention_group_id_list)
+                sorted_group_ids_list = GroupUtils.get_group_ids_by_aggs(attention_group_id_list,es_cli_obj)
 
                 group_recommend_ids_list = sorted_group_ids_list
                 #if len(group_recommend_ids_list) < size and len(group_recommend_ids_list)<len(attention_group_list):

search/views/pick.py

@@ -113,7 +113,7 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
             "_score"
         ]
         q["_source"] = {
-            "include":["id"]
+            "includes":["id"]
         }
         result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
 

search/views/tag.py

@@ -46,7 +46,7 @@ def query_tag(query,offset,size):
                 {'_score': {"order": "desc"}}
             ],
             "_source": {
-                "include": ["id", "name"]
+                "includes": ["id", "name"]
             }
         }
         q["highlight"] = get_highlight(["name"])            
@@ -107,7 +107,7 @@ def query_by_tag_type(tag_type_id,offset,size):
                 {"near_new_topic_num":{"order":"desc"}}
             ],
             "_source": {
-                "include": ["id", "name"]
+                "includes": ["id", "name"]
             }
         }
 

search/views/topic.py

@@ -10,14 +10,50 @@ from search.utils.topic import TopicUtils
 from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.common import *
+from libs.es import ESPerform
+
+
+def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE):
+    try:
+        if user_id == -1:
+            redis_key = "physical:discover_page" + ":user_id:" + str(user_id) + ":device_id:" + device_id
+        else:
+            redis_key = "physical:discover_page" + ":user_id:" + str(user_id)
+
+        redis_field_list = [b'have_read_topic_id']
+        redis_field_val_list = redis_client.hmget(redis_key,redis_field_list)
+
+        have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else []
+
+        recommend_topic_ids = []
+        topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id=user_id, offset=0, size=size,query_type=query_type,filter_topic_id_list=have_read_topic_id_list)
+
+        for topic_id in topic_id_dict:
+            recommend_topic_ids.append(topic_id)
+
+        have_read_topic_id_list.extend(recommend_topic_ids)
+        redis_dict = {
+            "have_read_topic_id": json.dumps(have_read_topic_id_list)
+        }
+        redis_client.hmset(redis_key,redis_dict)
 
+        return recommend_topic_ids
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return []
 
-def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
+def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_type=TopicPageType.HOME_RECOMMEND):
     try:
         if query is None:
-            redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":session_id:" + session_id
+            if user_id == -1:
+                redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
+            else:
+                redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
+        else:
+            if user_id == -1:
+                redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
             else:
-            redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":session_id:" + session_id + ":query:" + str(query)
+                redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
 
         redis_field_list = [b'last_offset_num', b'unread_topic_id']
         for page_id in range(0,offset,size):
@@ -27,17 +63,16 @@ def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
         last_offset_num = int(redis_field_val_list[0]) if redis_field_val_list[0] else -1
         recommend_topic_ids = []
 
-        topic_id_list = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query)
+        topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query,query_type=query_type)
         have_read_group_id_set = set()
-        unread_topic_id_list = list()
+        unread_topic_id_dict = dict()
         have_read_topic_id_set = set()
 
-        ori_unread_topic_id_list = list()
-
         if redis_field_val_list[1] and offset>0:
             if (user_id>0 and offset==last_offset_num) or user_id==-1:
-                ori_unread_topic_id_list = json.loads(redis_field_val_list[1])
-                topic_id_list = ori_unread_topic_id_list + topic_id_list
+                ori_unread_topic_id_dict = json.loads(redis_field_val_list[1])
+                if len(ori_unread_topic_id_dict) > 0:
+                    topic_id_dict.update(ori_unread_topic_id_dict)
 
                 for have_read_item in redis_field_val_list[2:]:
                     if have_read_item:
@@ -48,51 +83,50 @@ def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
         # 当前页用户数量
         cur_page_user_num = 0
 
-        for item in topic_id_list:
-            if item["group_id"] in have_read_group_id_set:
-                unread_topic_id_list.append(item)
+        for topic_id in topic_id_dict:
+            if topic_id_dict[topic_id] in have_read_group_id_set:
+                unread_topic_id_dict[topic_id] = topic_id_dict[topic_id]
             else:
-                if item["id"] not in have_read_topic_id_set:
-                    if isinstance(item["group_id"],int) and item["group_id"]>0 and cur_page_group_num<(size*0.9):
-                        have_read_group_id_set.add(item["group_id"])
-                        have_read_topic_id_set.add(item["id"])
+                if topic_id not in have_read_topic_id_set:
+                    if isinstance(topic_id_dict[topic_id],int) and topic_id_dict[topic_id]>0 and cur_page_group_num<(size*0.9):
+                        have_read_group_id_set.add(topic_id_dict[topic_id])
+                        have_read_topic_id_set.add(topic_id)
                         cur_page_group_num += 1
-                        recommend_topic_ids.append(item["id"])
-                    elif item["group_id"]<=0 and cur_page_user_num<(size*0.1):
+                        recommend_topic_ids.append(topic_id)
+                    elif topic_id_dict[topic_id] and cur_page_user_num<(size*0.1):
                         cur_page_user_num += 1
-                        recommend_topic_ids.append(item["id"])
+                        recommend_topic_ids.append(topic_id)
                     else:
-                        unread_topic_id_list.append(item)
+                        unread_topic_id_dict[topic_id] = topic_id_dict[topic_id]
 
                     if len(recommend_topic_ids) >= size:
                         break
 
-        if len(recommend_topic_ids) < size and len(unread_topic_id_list)>0:
-            recommend_len = len(recommend_topic_ids)
-            offi_unread_topic_id = [item["id"] for item in unread_topic_id_list[:(size-recommend_len)]]
-            recommend_topic_ids = recommend_topic_ids + offi_unread_topic_id
-            unread_topic_id_list = unread_topic_id_list[(size-recommend_len):]
 
+        offi_unread_topic_id_dict = dict()
+        if len(recommend_topic_ids) < size and len(unread_topic_id_dict)>0:
+            for unread_topic_id in unread_topic_id_dict:
+                if len(recommend_topic_ids)<size:
+                    recommend_topic_ids.append(unread_topic_id)
+                else:
+                    offi_unread_topic_id_dict[unread_topic_id] = unread_topic_id_dict[unread_topic_id]
 
-        if user_id==-1:
-            unread_topic_id_list = ori_unread_topic_id_list + unread_topic_id_list
         redis_dict = {
-            "unread_topic_id":json.dumps(unread_topic_id_list),
+            "unread_topic_id":json.dumps(offi_unread_topic_id_dict),
             "last_offset_num":offset+size,
             offset: json.dumps(recommend_topic_ids)
         }
         redis_client.hmset(redis_key,redis_dict)
         # 每个session key保存15分钟
-        redis_client.expire(redis_key,15*60*60)
+        redis_client.expire(redis_key,15*60)
 
         return recommend_topic_ids
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return []
 
-
 @bind("physical/search/home_recommend")
-def home_recommend(session_id="",user_id=-1,offset=0,size=10):
+def home_recommend(device_id="",user_id=-1,offset=0,size=10,query_type=TopicPageType.HOME_RECOMMEND):
     """
     :remark:首页推荐，目前只推荐日记
     :param session_id:
@@ -104,10 +138,34 @@ def home_recommend(session_id="",user_id=-1,offset=0,size=10):
     try:
         if not user_id:
             user_id=-1
-        if not isinstance(session_id,str):
-            session_id = ""
+        if not isinstance(device_id,str):
+            device_id = ""
 
-        recommend_topic_ids = get_home_recommend_topic_ids(user_id,session_id,offset,size)
+        recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query_type=query_type)
+
+        return {"recommend_topic_ids":recommend_topic_ids}
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"recommend_topic_ids": []}
+
+
+@bind("physical/search/discover_page")
+def discover_page(device_id="",user_id=-1,size=10):
+    """
+    :remark:首页推荐，目前只推荐日记
+    :param session_id:
+    :param user_id:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        if not user_id:
+            user_id=-1
+        if not isinstance(device_id,str):
+            device_id = ""
+
+        recommend_topic_ids = get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE)
 
         return {"recommend_topic_ids":recommend_topic_ids}
     except:
@@ -116,7 +174,7 @@ def home_recommend(session_id="",user_id=-1,offset=0,size=10):
 
 
 @bind("physical/search/home_query")
-def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
+def home_query(device_id="",user_id=-1,query="",offset=0,size=10):
     """
     :remark:首页搜索，目前只推荐日记
     :param session_id:
@@ -129,10 +187,10 @@ def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
     try:
         if not user_id:
             user_id=-1
-        if not isinstance(session_id,str):
-            session_id = ""
+        if not isinstance(device_id,str):
+            device_id = ""
 
-        recommend_topic_ids = get_home_recommend_topic_ids(user_id,session_id,offset,size,query)
+        recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query)
         return {"recommend_topic_ids":recommend_topic_ids}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
@@ -140,7 +198,7 @@ def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
 
 
 @bind("physical/search/topic_detail_page_recommend")
-def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,offset=0,size=10):
+def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,filter_topic_user_id=False,offset=0,size=10):
     """
     :remark:帖子详情页推荐策略，缺少第一个卡片策略
     :param user_id:
@@ -153,10 +211,13 @@ def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_u
         if not isinstance(user_id,int):
             user_id = -1
 
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
         # 获取帖子标签列表
-        topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id)
+        topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id,es_cli_obj)
 
-        result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,offset,size)
+        result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size,es_cli_obj)
         recommend_topic_ids_list = list()
         if len(result_list)>0:
             recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
@@ -193,3 +254,15 @@ def query_topic_by_tag_aggregation(user_id,tag_id, offset, size):
         return {"recommend_topic_id": []}
 
 
+@bind("physical/search/topic")
+def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
+    """帖子搜索。"""
+
+    try:
+        result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
+                                                sorts_by=sorts_by, offset=offset, size=size)
+        topic_ids = [item["_source"]["id"] for item in result_list]
+        return {"topic_ids": topic_ids}
+    except:
+        logging.error("catch exception, err_msg:%s" % traceback.format_exc())
+        return {"topic_ids": []}

search/views/user.py

@@ -10,6 +10,7 @@ from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.user import UserUtils
 from search.utils.common import *
+from libs.es import ESPerform
 
 
 @bind("physical/search/recommend_user")
@@ -26,16 +27,51 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
         if not isinstance(self_user_id,int):
             self_user_id = -1
 
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
         #获取关注用户列表
-        (self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id)
+        (self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id,es_cli_obj)
 
         #去除自身及感兴趣的用户ID
         self_attention_user_id_list.append(self_user_id)
         self_attention_user_id_list.append(interesting_user_id)
 
-        recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size)
+        recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj)
 
         return recommend_user_list
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-        return {"recommend_user_ids":[]}
\ No newline at end of file
+        return []
+
+@bind("physical/search/batch_recommend_user")
+def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10):
+    """
+    :remark 点关注推荐用户 - 批量接口
+    :param self_user_id:
+    :param interesting_user_id:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        if not isinstance(self_user_id,int):
+            self_user_id = -1
+
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
+        #获取关注用户列表
+        (need_filter_attention_user_id_list, attention_user_dict_list,attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list,self_user_id,es_cli_obj)
+
+
+        #去除自身及感兴趣的用户ID
+        need_filter_attention_user_id_list.append(self_user_id)
+
+        recommend_user_dict = UserUtils.get_batch_recommend_user_dict(need_filter_attention_user_id_list=need_filter_attention_user_id_list,attention_user_id_list=attention_user_id_list,attention_user_dict_list=attention_user_dict_list,self_user_id=self_user_id,offset=offset,size=size,es_cli_obj=es_cli_obj)
+
+        logging.info("duan add,recommend_user_dict:%s" % str(recommend_user_dict))
+        return recommend_user_dict
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {}
\ No newline at end of file

trans2es/mapping/topic.json

@@ -18,6 +18,7 @@
     "pick_id_list":{"type":"long"},
     "offline_score":{"type":"double"},//离线算分
     "manual_score":{"type":"double"},//人工赋分
+    "has_image":{"type":"boolean"},//是否有图
     "create_time":{"type":"date", "format":"date_time_no_millis"},
     "update_time":{"type":"date", "format":"date_time_no_millis"},
     "create_time_val":{"type":"long"},

trans2es/models/topic.py

@@ -27,32 +27,58 @@ class ActionSumAboutTopic(models.Model):
     data_type = models.IntegerField(verbose_name=u"动作类型")
     data_value = models.BigIntegerField(verbose_name=u"值")
 
+class TopicImage(models.Model):
+    class Meta:
+        verbose_name = u'日记图片'
+        db_table = 'topic_image'
+
+    id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
+    topic_id = models.IntegerField(verbose_name=u'日记ID')
+    url = models.CharField(verbose_name=u'图片URL',max_length=300)
+    is_online = models.BooleanField(verbose_name='是否上线')
+    is_deleted = models.BooleanField(verbose_name='是否删除')
+    create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+
 
 class Topic(models.Model):
     class Meta:
         verbose_name = u'日记'
         db_table = 'topic'
 
-    id = models.IntegerField(verbose_name='日记ID',primary_key=True)
-    name = models.CharField(verbose_name='日记名称',max_length=100)
+    id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
+    name = models.CharField(verbose_name=u'日记名称',max_length=100)
     #group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
     group = models.ForeignKey(
         Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
-    user_id = models.IntegerField(verbose_name='用户ID')
+    user_id = models.IntegerField(verbose_name=u'用户ID')
     drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
-    description = models.CharField(verbose_name='日记本描述',max_length=200)
-    content = models.CharField(verbose_name='日记本内容',max_length=1000)
+    description = models.CharField(verbose_name=u'日记本描述',max_length=200)
+    content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
     share_num = models.IntegerField(verbose_name='')
-    vote_num = models.IntegerField(verbose_name='点赞数')
-    reply_num = models.IntegerField(verbose_name='回复数')
+    vote_num = models.IntegerField(verbose_name=u'点赞数')
+    reply_num = models.IntegerField(verbose_name=u'回复数')
     cover = models.CharField(verbose_name='',max_length=200)
-    is_online = models.BooleanField(verbose_name='是否上线')
-    is_deleted = models.BooleanField(verbose_name='是否删除')
-    content_level = models.CharField(verbose_name='内容等级',max_length=3)
+    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
     create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
 
 
+    def topic_has_image(self):
+        try:
+            has_image = False
+
+            query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
+            if len(query_list)>0:
+                has_image=True
+
+            return has_image
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return False
+
     def get_pick_id_info(self):
         try:
             pick_id_list = list()

trans2es/utils/topic_transfer.py

@@ -38,6 +38,7 @@ class TopicTransfer(object):
 
             res["offline_score"] = instance.get_topic_offline_score()
             res["manual_score"] = instance.drop_score
+            res["has_image"] = instance.topic_has_image()
 
             create_time = instance.create_time
             tzlc_create_time = tzlc(create_time)