Merge branch 'master' into 'test'

Master

See merge request !99

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*~
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.vscode
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+.idea/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+
+# config
+fabfile.py
+settings.online.py
+settings_local.py
+media/
+log/
+crawldata/
+conf/
+/static
+.vagrant/
+Vagrantfile
+
+*.DS_Store
+dump.rdb

requirements.txt

@@ -11,6 +11,7 @@ PyMySQL==0.9.2
 
 gunicorn==19.9.0
 gevent==1.3.7
+pypinyin==0.34.1
 
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git@master
 git+ssh://git@git.wanmeizhensuo.com/backend/helios.git@master
@@ -19,5 +20,6 @@ git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git@v0.1.3#egg=gm-config==
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-protocol.git@master
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-upload.git@master
 git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
-git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@dev
+git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
 git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git@master
+

search/utils/common.py

@@ -26,7 +26,12 @@ class TopicDocumentField(object):
     IS_ONLINE = "is_online",
     TAG_LIST = "tag_list"
 
+class TopicPageType(object):
+    # 首页推荐
+    HOME_RECOMMEND=1
 
+    # 发现页面
+    FIND_PAGE=2
 
 def time_consuming_decorator(func):
     def time_consuming(*args, **kwargs):

search/utils/group.py

@@ -38,7 +38,7 @@ class GroupUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
 
             return ESPerform.get_search_results(ESPerform.get_cli(), "group", q, offset, size)
@@ -47,8 +47,11 @@ class GroupUtils(object):
             return {"total_count":0, "hits":[]}
 
     @classmethod
-    def get_hot_group_recommend_result_list(cls,offset,size):
+    def get_hot_group_recommend_result_list(cls,offset,size,es_cli_obj=None):
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = {
                 "bool":{
@@ -62,10 +65,10 @@ class GroupUtils(object):
                 {"high_quality_topic_num":{"order":"desc"}}
             ]
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"group",q,offset,size)
+            result_dict = ESPerform.get_search_results(es_cli_obj,"group",q,offset,size)
 
             group_ids_list = []
             if len(result_dict["hits"]) > 0:
@@ -77,12 +80,15 @@ class GroupUtils(object):
             return []
 
     @classmethod
-    def get_user_attention_group_list(cls,user_id,offset=0,size=10):
+    def get_user_attention_group_list(cls,user_id,offset=0,size=10,es_cli_obj=None):
         """
         :remark: 获取用户关注小组列表
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = {
                 "bool":{
@@ -94,10 +100,10 @@ class GroupUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["attention_group_id_list"]
+                "includes":["attention_group_id_list"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"user",q,offset,size)
+            result_dict = ESPerform.get_search_results(es_cli_obj,"user",q,offset,size)
             if len(result_dict["hits"])>0:
                 return result_dict["hits"][0]["_source"]["attention_group_id_list"]
             else:
@@ -107,13 +113,16 @@ class GroupUtils(object):
             return []
 
     @classmethod
-    def get_group_ids_by_aggs(cls,group_id_list):
+    def get_group_ids_by_aggs(cls,group_id_list,es_cli_obj=None):
         """
         :remark:聚合查询获取小组列表
         :param group_id_list:
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["size"]=0
             q["query"] = {
@@ -136,7 +145,7 @@ class GroupUtils(object):
                 }
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"topic",q,aggregations_query=True)
+            result_dict = ESPerform.get_search_results(es_cli_obj,"topic",q,aggregations_query=True)
             buckets_list = result_dict["aggregations"]["group_ids"]["buckets"]
 
             sorted_buckets_list = sorted(buckets_list,key=lambda item:item["max_date"]["value"],reverse=True)

search/utils/pick.py

@@ -77,7 +77,7 @@ class PickUtils(object):
                 }
             }
             q["_source"] = {
-                "include": ["id"]
+                "includes": ["id"]
             }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
             recommend_pick_celebrity_list = list()
@@ -123,7 +123,7 @@ class PickUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
 
@@ -153,7 +153,7 @@ class PickUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["id"]
+                "includes":["id"]
             }
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
 

search/utils/user.py

@@ -5,18 +5,72 @@ import logging
 import traceback
 import json
 import time
+import copy
 from libs.es import ESPerform
 
 
 class UserUtils(object):
     @classmethod
-    def get_attention_user_list(cls,user_id_list,self_user_id):
+    def get_batch_attention_user_list(cls,user_id_list,self_user_id,es_cli_obj=None):
+        """
+        :remark 批量用户 关注的 用户列表
+        :param user_id_list:
+        :param self_user_id:
+        :return:
+        """
+        try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
+            user_id_list.append(self_user_id)
+
+            q = dict()
+            q["query"] = {
+                "terms":{
+                    "user_id":user_id_list
+                }
+            }
+            q["_source"] = {
+                "includes":["attention_user_id_list","user_id"]
+            }
+
+            result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=len(user_id_list))
+
+            self_attention_user_id_list = []
+            attention_user_dict_list = list()
+            ret_attention_user_id_list = list()
+            for hit_item in result_dict["hits"]:
+
+                attention_user_dict = dict()
+                user_id = hit_item["_source"]["user_id"]
+
+                if user_id == self_user_id:
+                    self_attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
+                else:
+                    attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
+
+                    attention_user_dict[user_id] = attention_user_id_list
+                    attention_user_dict_list.append(attention_user_dict)
+                    ret_attention_user_id_list.append(user_id)
+
+            return [self_attention_user_id_list,attention_user_dict_list,ret_attention_user_id_list]
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return ([],[],[])
+
+
+    @classmethod
+    def get_attention_user_list(cls,user_id_list,self_user_id,es_cli_obj=None):
         """
         :remark 获取指定用户列表 关注的 用户列表
         :param user_id:
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
+
             q = dict()
             q["query"] = {
                 "terms":{
@@ -24,10 +78,10 @@ class UserUtils(object):
                 }
             }
             q["_source"] = {
-                "include":["attention_user_id_list","user_id"]
+                "includes":["attention_user_id_list","user_id"]
             }
 
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset=0, size=len(user_id_list))
+            result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=len(user_id_list))
 
             self_attention_user_id_list = []
             recursion_attention_user_id_list = []
@@ -59,7 +113,7 @@ class UserUtils(object):
             return []
 
     @classmethod
-    def get_recommend_user_list(cls,self_attention_user_id_list,recursion_attention_user_id_list,offset,size):
+    def get_recommend_user_list(cls,self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj=None):
         """
         :remark 获取推荐用户列表
         :param attention_user_id_list:
@@ -67,6 +121,9 @@ class UserUtils(object):
         :return:
         """
         try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
             q = dict()
             q["query"] = dict()
 
@@ -118,9 +175,9 @@ class UserUtils(object):
             }
             q["query"]["function_score"] = query_function_score
             q["_source"] = {
-                "include":["user_id"]
+                "includes":["user_id"]
             }
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="user", query_body=q,
+            result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="user", query_body=q,
                                                        offset=offset, size=size)
 
             recommend_user_list = list()
@@ -130,4 +187,110 @@ class UserUtils(object):
             return recommend_user_list
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return []
\ No newline at end of file
+            return []
+
+
+    @classmethod
+    def get_batch_recommend_user_dict(cls,need_filter_attention_user_id_list,attention_user_id_list,attention_user_dict_list,self_user_id,offset,size,es_cli_obj=None):
+        """
+        :remark 获取批量推荐用户
+        :param need_filter_attention_user_id_list:
+        :param attention_user_dict:
+        :param self_user_id:
+        :param offset:
+        :param size:
+        :return:
+        """
+        try:
+            if not es_cli_obj:
+                es_cli_obj = ESPerform.get_cli()
+
+            batch_query_list = list()
+            for interesting_user_item_dict in attention_user_dict_list:
+                for interesting_user_id in interesting_user_item_dict:
+                    if interesting_user_id != self_user_id:
+                        logging.info("duan add,interesting_user_id:%d" % interesting_user_id)
+                        filter_user_id_list = copy.deepcopy(need_filter_attention_user_id_list)
+                        filter_user_id_list.append(interesting_user_id)
+
+                        q = dict()
+                        q["from"] = offset
+                        q["size"] = size
+                        q["query"] = dict()
+
+                        functions_list = [
+                            {
+                                "gauss": {
+                                    "latest_topic_time_val": {
+                                        "origin": int(time.time()),
+                                        "scale": "600",
+                                        "decay": 0.1
+                                    }
+                                }
+                            }
+                        ]
+
+                        if len(interesting_user_item_dict[interesting_user_id]) > 0:
+                            functions_list.append(
+                                {
+                                    "filter":{
+                                        "bool":{
+                                            "should":{"terms":{"user_id":interesting_user_item_dict[interesting_user_id]}}
+                                        }
+                                    },
+                                    "weight":10
+                                }
+                            )
+
+                        query_function_score = {
+                            "query": {
+                                "bool": {
+                                    "must": [
+                                        {"term": {"is_recommend": True}},
+                                        {"term": {"is_online": True}},
+                                        {"term": {"is_deleted": False}},
+                                        {"term": {"is_shadow": False}}
+                                    ],
+                                    "must_not":{
+                                        "terms":{
+                                            "user_id":filter_user_id_list
+                                        }
+                                    }
+                                }
+                            },
+                            "score_mode": "sum",
+                            "boost_mode": "sum",
+                            "functions": functions_list
+                        }
+                        q["query"]["function_score"] = query_function_score
+                        q["_source"] = {
+                            "includes":["user_id"]
+                        }
+
+                        batch_query_list.append(q)
+
+            index_name = ESPerform.get_official_index_name("user","read")
+            search_header_dict = {'index': index_name, 'type': "_doc"}
+            query_body = ""
+            for query_item in batch_query_list:
+                query_body += "{}\n{}\n".format(json.dumps(search_header_dict),json.dumps(query_item))
+
+            result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="user", query_body=query_body,
+                                                       batch_search=True)
+            ret_dict = dict()
+            user_index = 0
+            for res_item in result_dict["responses"]:
+                recommend_user_list = list()
+                for item in res_item["hits"]["hits"]:
+                    recommend_user_list.append(item["_source"]["user_id"])
+
+                logging.info("duan add,attention_user_id_list:%s,user_index:%d" % (str(attention_user_id_list), user_index))
+                ret_dict[str(attention_user_id_list[user_index])] = recommend_user_list
+                user_index += 1
+
+
+            logging.info("duan add,ret_dict:%s" % str(ret_dict))
+            return ret_dict
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return dict()

search/views/group.py

@@ -9,6 +9,7 @@ from libs.cache import redis_client
 from libs.es import ESPerform
 from search.utils.group import GroupUtils
 from search.utils.common import GroupSortTypes
+from libs.es import ESPerform
 
 
 @bind("physical/search/query_group")
@@ -46,18 +47,21 @@ def group_sort(user_id=-1,sort_type=GroupSortTypes.HOT_RECOMMEND,offset=0,size=1
         if not isinstance(user_id,int):
             user_id = -1
 
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
         if sort_type==GroupSortTypes.HOT_RECOMMEND:
-            group_ids_list = GroupUtils.get_hot_group_recommend_result_list(offset,size)
+            group_ids_list = GroupUtils.get_hot_group_recommend_result_list(offset,size,es_cli_obj)
 
             return {"group_recommend_ids":group_ids_list}
 
         elif sort_type==GroupSortTypes.ATTENTION_RECOMMEND:
-            attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset=0,size=1)
+            attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset=0,size=1,es_cli_obj=es_cli_obj)
             if len(attention_group_list)==0:
                 return {"group_recommend_ids": []}
             else:
                 attention_group_id_list = [item["group_id"] for item in attention_group_list]
-                sorted_group_ids_list = GroupUtils.get_group_ids_by_aggs(attention_group_id_list)
+                sorted_group_ids_list = GroupUtils.get_group_ids_by_aggs(attention_group_id_list,es_cli_obj)
 
                 group_recommend_ids_list = sorted_group_ids_list
                 #if len(group_recommend_ids_list) < size and len(group_recommend_ids_list)<len(attention_group_list):

search/views/pick.py

@@ -113,7 +113,7 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
             "_score"
         ]
         q["_source"] = {
-            "include":["id"]
+            "includes":["id"]
         }
         result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
 

search/views/tag.py

@@ -46,7 +46,7 @@ def query_tag(query,offset,size):
                 {'_score': {"order": "desc"}}
             ],
             "_source": {
-                "include": ["id", "name"]
+                "includes": ["id", "name"]
             }
         }
         q["highlight"] = get_highlight(["name"])            
@@ -107,7 +107,7 @@ def query_by_tag_type(tag_type_id,offset,size):
                 {"near_new_topic_num":{"order":"desc"}}
             ],
             "_source": {
-                "include": ["id", "name"]
+                "includes": ["id", "name"]
             }
         }
 

search/views/user.py

@@ -10,6 +10,7 @@ from libs.es import ESPerform
 from libs.cache import redis_client
 from search.utils.user import UserUtils
 from search.utils.common import *
+from libs.es import ESPerform
 
 
 @bind("physical/search/recommend_user")
@@ -26,16 +27,51 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
         if not isinstance(self_user_id,int):
             self_user_id = -1
 
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
         #获取关注用户列表
-        (self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id)
+        (self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id,es_cli_obj)
 
         #去除自身及感兴趣的用户ID
         self_attention_user_id_list.append(self_user_id)
         self_attention_user_id_list.append(interesting_user_id)
 
-        recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size)
+        recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj)
 
         return recommend_user_list
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-        return {"recommend_user_ids":[]}
\ No newline at end of file
+        return []
+
+@bind("physical/search/batch_recommend_user")
+def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10):
+    """
+    :remark 点关注推荐用户 - 批量接口
+    :param self_user_id:
+    :param interesting_user_id:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        if not isinstance(self_user_id,int):
+            self_user_id = -1
+
+        #获取es链接对象
+        es_cli_obj = ESPerform.get_cli()
+
+        #获取关注用户列表
+        (need_filter_attention_user_id_list, attention_user_dict_list,attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list,self_user_id,es_cli_obj)
+
+
+        #去除自身及感兴趣的用户ID
+        need_filter_attention_user_id_list.append(self_user_id)
+
+        recommend_user_dict = UserUtils.get_batch_recommend_user_dict(need_filter_attention_user_id_list=need_filter_attention_user_id_list,attention_user_id_list=attention_user_id_list,attention_user_dict_list=attention_user_dict_list,self_user_id=self_user_id,offset=offset,size=size,es_cli_obj=es_cli_obj)
+
+        logging.info("duan add,recommend_user_dict:%s" % str(recommend_user_dict))
+        return recommend_user_dict
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {}
\ No newline at end of file

trans2es/mapping/topic.json

@@ -18,6 +18,7 @@
     "pick_id_list":{"type":"long"},
     "offline_score":{"type":"double"},//离线算分
     "manual_score":{"type":"double"},//人工赋分
+    "has_image":{"type":"boolean"},//是否有图
     "create_time":{"type":"date", "format":"date_time_no_millis"},
     "update_time":{"type":"date", "format":"date_time_no_millis"},
     "create_time_val":{"type":"long"},

trans2es/models/topic.py

@@ -27,32 +27,58 @@ class ActionSumAboutTopic(models.Model):
     data_type = models.IntegerField(verbose_name=u"动作类型")
     data_value = models.BigIntegerField(verbose_name=u"值")
 
+class TopicImage(models.Model):
+    class Meta:
+        verbose_name = u'日记图片'
+        db_table = 'topic_image'
+
+    id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
+    topic_id = models.IntegerField(verbose_name=u'日记ID')
+    url = models.CharField(verbose_name=u'图片URL',max_length=300)
+    is_online = models.BooleanField(verbose_name='是否上线')
+    is_deleted = models.BooleanField(verbose_name='是否删除')
+    create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+
 
 class Topic(models.Model):
     class Meta:
         verbose_name = u'日记'
         db_table = 'topic'
 
-    id = models.IntegerField(verbose_name='日记ID',primary_key=True)
-    name = models.CharField(verbose_name='日记名称',max_length=100)
+    id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
+    name = models.CharField(verbose_name=u'日记名称',max_length=100)
     #group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
     group = models.ForeignKey(
         Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
-    user_id = models.IntegerField(verbose_name='用户ID')
+    user_id = models.IntegerField(verbose_name=u'用户ID')
     drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
-    description = models.CharField(verbose_name='日记本描述',max_length=200)
-    content = models.CharField(verbose_name='日记本内容',max_length=1000)
+    description = models.CharField(verbose_name=u'日记本描述',max_length=200)
+    content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
     share_num = models.IntegerField(verbose_name='')
-    vote_num = models.IntegerField(verbose_name='点赞数')
-    reply_num = models.IntegerField(verbose_name='回复数')
+    vote_num = models.IntegerField(verbose_name=u'点赞数')
+    reply_num = models.IntegerField(verbose_name=u'回复数')
     cover = models.CharField(verbose_name='',max_length=200)
-    is_online = models.BooleanField(verbose_name='是否上线')
-    is_deleted = models.BooleanField(verbose_name='是否删除')
-    content_level = models.CharField(verbose_name='内容等级',max_length=3)
+    is_online = models.BooleanField(verbose_name=u'是否上线')
+    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+    content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
     create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
 
 
+    def topic_has_image(self):
+        try:
+            has_image = False
+
+            query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
+            if len(query_list)>0:
+                has_image=True
+
+            return has_image
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return False
+
     def get_pick_id_info(self):
         try:
             pick_id_list = list()

trans2es/utils/topic_transfer.py

@@ -38,6 +38,7 @@ class TopicTransfer(object):
 
             res["offline_score"] = instance.get_topic_offline_score()
             res["manual_score"] = instance.drop_score
+            res["has_image"] = instance.topic_has_image()
 
             create_time = instance.create_time
             tzlc_create_time = tzlc(create_time)