modify

search/utils/common.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+class GroupSortTypes(object):
+    # 热度排序
+    HOT_RECOMMEND=0
+
+    # 关注排序
+    ATTENTION_RECOMMEND=1
\ No newline at end of file

search/utils/group.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+import traceback
+from libs.es import ESPerform
+
+
+class GroupUtils(object):
+
+    @classmethod
+    def get_group_query_result(cls,query,offset,size):
+        try:
+            q = dict()
+            q["filter"] = {
+                "bool":{
+                    "must":{
+                        "is_online":True
+                    }
+                }
+            }
+
+            multi_fields = {
+                'description': 2,
+                'name': 4,
+            }
+            query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
+            multi_match = {
+                'query': query,
+                'type': 'cross_fields',
+                'operator': 'and',
+                'fields': query_fields,
+            }
+
+            q['query'] = {
+                'bool': {
+                    "should": [
+                        {'multi_match': multi_match}
+                    ],
+                    "minimum_should_match": 1
+                }
+            }
+            q["_source"] = {
+                "include":["id"]
+            }
+
+            return ESPerform.get_search_results(ESPerform.get_cli(), "group", q, offset, size)
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return {"total_count":0, "hits":[]}
+
+    @classmethod
+    def get_hot_group_recommend_results(cls,offset,size):
+        try:
+            q = dict()
+            q["filter"] = {
+                "bool":{
+                    "must":{
+                        "is_online":True
+                    }
+                }
+            }
+            q["sort"] = [
+                {"high_quality_topic_num":{"order":"desc"}}
+            ]
+            q["_source"] = {
+                "include":["id"]
+            }
+
+            return ESPerform.get_search_results(ESPerform.get_cli(),"group",q,offset,size)
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return {"total_count":0, "hits":[]}
+
+    @classmethod
+    def get_user_attention_group_list(cls,user_id,offset,size):
+        """
+        :remark: 获取用户关注小组列表
+        :return:
+        """
+        try:
+            q = dict()
+            q["filter"] = {
+                "bool":{
+                    "must":{
+                        "is_online":True,
+                        "user_id":user_id
+                    }
+                }
+            }
+            q["_source"] = {
+                "include":["attention_group_id_list"]
+            }
+
+            result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"user",q,offset,size)
+            if len(result_dict["hits"])>0:
+                return result_dict["hits"][0]["attention_group_id_list"]
+            else:
+                return []
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return []

search/utils/topic.py

@@ -68,7 +68,7 @@ class TopicUtils(object):
             return False
 
     @classmethod
-    def get_recommend_topic_ids(cls,user_id,offset,size,is_first_time,group_topic_ids_index,not_group_topic_ids_index):
+    def get_recommend_topic_ids(cls,user_id,offset,size,is_first_time,group_topic_ids_index,not_group_topic_ids_index,query=None):
         """
         :需增加打散逻辑
         :remark:获取首页推荐帖子列表
@@ -156,11 +156,30 @@ class TopicUtils(object):
             q["query"] = {
                 "function_score": query_function_score
             }
+
+            if query is not None:#搜索帖子
+                multi_fields = {
+                    'description': 2,
+                    'name': 4,
+                }
+                query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
+                multi_match = {
+                    'query': query,
+                    'type': 'cross_fields',
+                    'operator': 'and',
+                    'fields': query_fields,
+                }
+                q["query"]["bool"] = {
+                    "should": [
+                        {'multi_match': multi_match}
+                    ],
+                    "minimum_should_match": 1
+                }
             q["_source"] = {
                 "include":["id","group_id"]
             }
 
-            logging.info("duan add,home_recommend es query:%s" % str(q).encode("utf-8"))
+            logging.info("duan add,es query:%s" % str(q).encode("utf-8"))
             result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
                                                        offset=offset, size=size)
             group_topic_ids = list()

search/views/group.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from libs.cache import redis_client
+from libs.es import ESPerform
+from search.utils.group import GroupUtils
+from search.utils.common import GroupSortTypes
+
+
+@bind("physical/search/query_group")
+def query_group(query="",offset=0,size=10):
+    """
+    :remark:小组搜索排序策略，缺少排序策略
+    :param query:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        result_dict = GroupUtils.get_group_query_result(query,offset,size)
+        group_ids_list = [item.id for item in result_dict["hits"]]
+
+        return {"group_ids": group_ids_list}
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"group_ids":[]}
+
+@bind("physical/search/group_sort")
+def group_sort(user_id=-1,sort_type=GroupSortTypes.HOT_RECOMMEND,offset=0,size=10):
+    """
+    :remark 小组排序,缺少：前1天发评论人数＊x
+    :param user_id:
+    :param sort_type:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        if sort_type==GroupSortTypes.HOT_RECOMMEND:
+            result_dict = GroupUtils.get_hot_group_recommend_results(offset,size)
+            group_ids_list = [item.id for item in result_dict["hits"]]
+
+            return {"group_recommend_ids":group_ids_list}
+        elif sort_type==GroupSortTypes.ATTENTION_RECOMMEND:
+            attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset,size)
+            if len(attention_group_list)==0:
+                return {"group_recommend_ids": []}
+
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"group_recommend_ids":[]}

search/views/topic.py

@@ -13,7 +13,7 @@ from libs.cache import redis_client
 @bind("physical/search/home_recommend")
 def home_recommend(user_id=-1,offset=0,size=10):
     """
-    :remark:首页推荐，目前只推荐日记
+    :remark:首页推荐，目前只推荐日记,缺少小组打散策略
     :return:
     """
     try:
@@ -47,7 +47,6 @@ def home_recommend(user_id=-1,offset=0,size=10):
             TopicUtils.refresh_redis_hash_data(redis_client,redis_key,redis_hash_dict)
 
             return {"recommend_topic_ids":recommend_topic_ids}
-
         else:
             (group_topic_ids,not_group_topic_ids) = TopicUtils.get_recommend_topic_ids(user_id,offset,size,True,0,0)
             recommend_topic_ids = group_topic_ids[:(size - 1)] + not_group_topic_ids[:1]
@@ -58,8 +57,65 @@ def home_recommend(user_id=-1,offset=0,size=10):
                 "not_group_topic_ids_index": 0
             }
             TopicUtils.refresh_redis_hash_data(redis_client, redis_key, redis_hash_dict)
+
             return {"recommend_topic_ids": recommend_topic_ids}
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"recommend_topic_ids": []}
+
+
+@bind("physical/search/home_query")
+def home_query(user_id=-1,query="",offset=0,size=10):
+    """
+    :remark 首页搜索排序,没有做打散,搜索逻辑优化
+    :param query:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        redis_key = "physical:home_query:" + "user_id:" + str(user_id)
+        redis_val_dict = redis_client.hgetall(redis_key)
+        if len(redis_val_dict) > 0:
+            group_topic_ids = json.loads(redis_val_dict["group_topic_ids"])
+            not_group_topic_ids = json.loads(redis_val_dict["not_group_topic_ids"])
+            group_topic_ids_index = redis_val_dict["group_topic_ids_index"]
+            not_group_topic_ids_index = redis_val_dict["not_group_topic_ids_index"]
+
+            if len(group_topic_ids) < (size-1) or len(not_group_topic_ids) < 1:
+                if len(group_topic_ids) < (size-1):
+                    group_topic_ids_index += 1
+                    (new_group_topic_ids, new_not_group_topic_ids) = TopicUtils.get_recommend_topic_ids(user_id, offset, size,
+                                                                                                False, group_topic_ids_index, not_group_topic_ids_index,query)
+                    group_topic_ids += new_group_topic_ids
+                else:
+                    not_group_topic_ids_index += 1
+                    (new_group_topic_ids, new_not_group_topic_ids) = TopicUtils.get_recommend_topic_ids(user_id, offset, size,
+                                                                                                False, group_topic_ids_index, not_group_topic_ids_index,query)
+                    not_group_topic_ids += new_not_group_topic_ids
+
+            recommend_topic_ids = group_topic_ids[:(size-1)] + not_group_topic_ids[:1]
+            redis_hash_dict = {
+                "group_topic_ids": group_topic_ids[(size-1):],
+                "not_group_topic_ids": not_group_topic_ids[1:],
+                "group_topic_ids_index":group_topic_ids_index,
+                "not_group_topic_ids_index":not_group_topic_ids_index
+            }
+            TopicUtils.refresh_redis_hash_data(redis_client,redis_key,redis_hash_dict)
 
+            return {"recommend_topic_ids":recommend_topic_ids}
+        else:
+            (group_topic_ids,not_group_topic_ids) = TopicUtils.get_recommend_topic_ids(user_id,offset,size,True,0,0,query)
+            recommend_topic_ids = group_topic_ids[:(size - 1)] + not_group_topic_ids[:1]
+            redis_hash_dict = {
+                "group_topic_ids": group_topic_ids[(size - 1):],
+                "not_group_topic_ids": not_group_topic_ids[1:],
+                "group_topic_ids_index": 0,
+                "not_group_topic_ids_index": 0
+            }
+            TopicUtils.refresh_redis_hash_data(redis_client, redis_key, redis_hash_dict)
+
+            return {"recommend_topic_ids": recommend_topic_ids}
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
         return {"recommend_topic_ids": []}

trans2es/mapping/group.json

@@ -11,6 +11,7 @@
     "topic_num":{"type":"long"},
     "creator_id":{"type":"long"},
     "icon":{"type":"text"},
+    "high_quality_topic_num":{"type":"long"},//前一天该小组4&5星帖子数量
     "create_time":{"type":"date", "format":"date_time_no_millis"},
     "update_time":{"type":"date", "format":"date_time_no_millis"}
   }

trans2es/mapping/user.json

@@ -31,6 +31,13 @@
         "country_id":{"type":"long"}
       }
     },
+    "attention_group_id_list":{//关注小组列表
+      "type":"nested",
+      "properties":{
+        "group_id":{"type":"long"},
+        "update_time":{"type":"date", "format":"date_time_no_millis"}
+      }
+    },
     "create_time":{"type":"date", "format":"date_time_no_millis"},
     "update_time":{"type":"date", "format":"date_time_no_millis"}
   }

trans2es/models/group.py

@@ -8,6 +8,7 @@ import logging
 from libs.es import ESPerform
 from django.db import models
 import datetime
+from .topic import Topic
 
 class Group(models.Model):
     class Meta:
@@ -26,6 +27,14 @@ class Group(models.Model):
     create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
 
+    #获取前一天4，5星发帖数
+    def get_high_quality_topic_num(self):
+        yesterday = datetime.datetime.now()-datetime.timedelta(days=1)
+        yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day)
+        yesterday_end_time = "%s-%s-%s 23:59:59" % (yesterday.year, yesterday.month, yesterday.day)
+
+        return Topic.objects.filter(content_level__in=("4", "5"), group_id=self.id,create_time__gte=yesterday_begin_time,create_time__lte=yesterday_end_time).count()
+
     def detail(self):
         result = {
             'id': self.id,

trans2es/models/group_user_role.py

@@ -16,8 +16,11 @@ class GroupUserRole(models.Model):
         db_table = 'group_user_role'
 
     user_id = models.BigIntegerField(verbose_name=u'用户ID')
+    group_id = models.BigIntegerField(verbose_name=u'组ID')
     group = models.ForeignKey(
         Group, verbose_name=u"关联的小组", null=True, blank=True, default=None, on_delete=models.CASCADE)
     is_online = models.BooleanField(verbose_name=u"是否有效", default=True, db_index=True)
     role_id = models.SmallIntegerField(verbose_name=u'角色ID')
-    invite_num = models.IntegerField(verbose_name=u'邀请数量', default=0)
\ No newline at end of file
+    invite_num = models.IntegerField(verbose_name=u'邀请数量', default=0)
+    create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))

trans2es/models/user.py

@@ -10,6 +10,7 @@ from django.db import models
 import datetime
 from .group_user_role import GroupUserRole
 from .tag import AccountUserTag
+from libs.tools import tzlc
 
 class User(models.Model):
     class Meta:
@@ -45,6 +46,22 @@ class User(models.Model):
 
         return follow_user_detail_list
 
+    def get_attention_group_id_list(self):
+        try:
+            attention_group_id_list = list()
+            query_results = GroupUserRole.objects.filter(is_online=True,user_id=self.user_id)
+            for item in query_results:
+                item_dict = {
+                    "group_id": item.group_id,
+                    "update_time":tzlc(item.update_time)
+                }
+                attention_group_id_list.append(item_dict)
+
+            return attention_group_id_list
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return []
+
     def get_pick_user_id_list(self):
         pick_user_id_list = list()
         user_picks = self.user_pick.filter(is_deleted=False)

trans2es/type_info.py

@@ -12,10 +12,10 @@ import elasticsearch
 import elasticsearch.helpers
 import sys
 
-from trans2es.models import topic,user,pick_celebrity
-from trans2es.utils.topic_transfer import TopicTransfer
+from trans2es.models import topic,user,pick_celebrity,group
 from trans2es.utils.user_transfer import UserTransfer
 from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
+from trans2es.utils.group_transfer import GroupTransfer
 
 from libs.es import ESPerform
 
@@ -266,6 +266,17 @@ def get_type_info_map():
             bulk_insert_chunk_size=100,
             round_insert_chunk_size=5,
             round_insert_period=2,
+        ),
+        TypeInfo(
+            name="group",  # 小组
+            type="group",
+            model=group.Group,
+            # query_deferred=lambda:user.User.objects.all().query,
+            query_deferred=lambda: group.Group.objects.all().query,
+            get_data_func=GroupTransfer.get_group_data,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
         )
     ]
 

trans2es/utils/group_transfer.py

@@ -26,6 +26,7 @@ class GroupTransfer(object):
             res["icon"] = instance.icon
             res["create_time"] = instance.create_time
             res["update_time"] = instance.update_time
+            res["high_quality_topic_num"] = instance.get_high_quality_topic_num()
 
             return res
         except:

trans2es/utils/topic_transfer.py

@@ -30,8 +30,12 @@ class TopicTransfer(object):
             res["offline_score"] = instance.get_topic_offline_score()
 
             create_time = instance.create_time
+            tzlc_create_time = tzlc(create_time)
+
+            print("duan add,type:%s,dir:%s" % (type(tzlc_create_time),dir(tzlc_create_time)))
             res["create_time"] = tzlc(create_time)
 
+
             update_time = instance.update_time
             res["update_time"] = tzlc(update_time)
 

trans2es/utils/user_transfer.py

@@ -26,6 +26,7 @@ class UserTransfer(object):
 
             res["tag_list"] = instance.get_user_tag_id_list()
             res["attention_user_id_list"] = instance.get_follow_user_id_list()
+            res["attention_group_id_list"] = instance.get_attention_group_id_list()
             res["pick_user_id_list"] = instance.get_pick_user_id_list()
             res["same_group_user_id_list"] = instance.get_same_group_user_id_list()