Merge branch 'feature/sun_topic2es' into 'test'

Feature/sun topic2es

See merge request !181

app_conf.xml

@@ -9,6 +9,7 @@
     <config name="initializer_list">
         <element value="physical.django_init"/>
         <element value="search.views.topic"/>
+        <element value="search.views.business_topic"/>
         <element value="search.views.pick"/>
         <element value="search.views.group"/>
         <element value="search.views.user"/>

search/utils/topic.py

@@ -555,21 +555,43 @@ class TopicUtils(object):
             return list()
 
     @classmethod
-    def process_filters(cls, filters):
+    def process_filters(cls, filters, filter_online=True):
         """处理过滤器部分。"""
 
         f = [
-            {"term": {"is_online": True}},
             {"term": {"is_deleted": False}},
         ]
         if not filters:
             return f
 
         for k, v in filters.items():
-            if k == "group_id":
+
+            if k in ["create_time_gte", "create_time_lte"]:
+
+                if k == "create_time_gte":
+                    op = "gte"
+                elif k == "create_time_lte":
+                    op = "lte"
+
+                f.append({
+                    "range": {
+                        "create_time_val": {
+                            op: v,
+                        }
+                    }
+                })
+            else:
+                if isinstance(k, list):
                     f.append({
-                    "term": {"group_id": v},
+                        "terms": {k: v},
                     })
+                else:
+                    f.append({
+                        "term": {k: v},
+                    })
+
+        if filter_online:
+            f.append({"term": {"is_online": True}})
 
         return f
 
@@ -601,22 +623,29 @@ class TopicUtils(object):
                 },
             })
 
+        elif sorts_by == TOPIC_SEARCH_SORT.VOTE_NUM:
+            pass
+
+        elif sorts_by == '2':
+            sort_rule.append({
+                "id":{
+                    "order":"desc"
+                },
+            })
+
         return sort_rule
 
     @classmethod
-    def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10):
+    def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
 
+        must = cls.process_filters(filters, filter_online=True)
         q = {
             "query": {
                 "bool": {
-                    "must": cls.process_filters(filters),
+                    "must": must,
                     "must_not": cls.process_nfilters(nfilters),
                 }
-            },
-            "_source": {
-                "includes":["id"]
-            },
-            "sort": [],
+            }
         }
 
         if sorts_by:
@@ -624,10 +653,18 @@ class TopicUtils(object):
             q["sort"] = sorts
 
         try:
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
-                                                       query_body=q, offset=offset, size=size)
+            result_dict = ESPerform.get_search_results(
+                ESPerform.get_cli(), sub_index_name=index_name,
+                query_body=q, offset=offset, size=size
+            )
 
-            return result_dict["hits"]
+            return {
+                "hits": result_dict["hits"],
+                "total_count": result_dict["total_count"]
+            }
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return []
+            return {
+                "hits": [],
+                "total_count": 0
+            }

search/views/business_topic.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals, absolute_import, print_function
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from search.utils.topic import TopicUtils
+from libs.es import ESPerform
+from libs.cache import redis_client
+from search.utils.common import *
+from libs.es import ESPerform
+
+
+@bind("physical/search/business/topic")
+def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
+    """帖子搜索。"""
+
+    try:
+        result_list = TopicUtils.list_topic_ids(
+            filters=filters, nfilters=nfilters, sorts_by=sorts_by,
+            offset=offset, size=size, filter_online=False,
+            index_name="business_topic"
+        )
+        topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
+        return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
+    except:
+        logging.error("catch exception, err_msg:%s" % traceback.format_exc())
+        return {"topic_ids": [], "total_count": 0}

search/views/topic.py

@@ -75,7 +75,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
         recommend_topic_ids = []
         have_read_topic_id_list = list()
 
-        if redis_field_val_list[0]:
+        if redis_field_val_list[0] and query is None:
             have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
 
         user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
@@ -303,11 +303,14 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
     try:
         result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
                                                 sorts_by=sorts_by, offset=offset, size=size)
-        topic_ids = [item["_source"]["id"] for item in result_list]
-        return {"topic_ids": topic_ids}
+        topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
+        return {
+            "topic_ids": topic_ids,
+            "total_count": result_list["total_count"]
+        }
     except:
         logging.error("catch exception, err_msg:%s" % traceback.format_exc())
-        return {"topic_ids": []}
+        return {"topic_ids": [], "total_count": 0}
 
 
 @bind("physical/search/query_topic_by_user_similarity")

trans2es/mapping/topic.json

@@ -25,6 +25,29 @@
     "update_time":{"type":"date", "format":"date_time_no_millis"},
     "create_time_val":{"type":"long"},
     "update_time_val":{"type":"long"},
-    "language_type":{"type":"long"}
+    "language_type":{"type":"long"},
+
+    "is_shadow": {"type": "boolean"},
+    "is_recommend": {"type": "boolean"},
+    "is_complaint": {"type": "boolean"},  // 是否被举报
+
+    "virtual_content_level":{"type": "text"},
+    "like_num_crawl": {"type": "long"}, // 爬取点赞数
+    "comment_num_crawl": {"type": "long"}, // 爬取评论数
+
+    "is_crawl": {"type": "boolean"},
+    "platform": {"type": "long"},
+    "platform_id": {"type": "long"},
+
+    "drop_score":{"type": "double"},  // 人工降分
+    "sort_score":{"type": "double"},  // 排序分
+
+    "pictorial_id":{"type": "long"}, //所在组ID
+    "pictorial_name":{  // 所在组名称
+        "type": "text",
+        "analyzer": "gm_default_index",
+        "search_analyzer": "gm_default_index"
+    }
+
   }
 }
\ No newline at end of file

trans2es/models/__init__.py

@@ -13,3 +13,4 @@ from .tag import AccountUserTag
 from .user import User
 from .group import Group
 from .topic import Topic
+# from .pictorial import PictorialTopic

trans2es/models/celebrity.py

@@ -30,7 +30,7 @@ class Celebrity(models.Model):
     def get_pick_id_list(self):
         try:
             pick_id_list = list()
-            query_results = PickCelebrity.objects.filter(celebrity_id=self.id,is_deleted=False)
+            query_results = PickCelebrity.objects.using(settings.SLAVE_DB_NAME).filter(celebrity_id=self.id,is_deleted=False)
             for data_item in query_results:
                 pick_id_list.append(data_item.pick_id)
 

trans2es/models/pictorial.py

+# from django.db import models
+
+
+# class PictorialTopic(models.Model):
+#     """画报帖子关系"""
+
+#     class Meta:
+#         verbose_name = u'画报帖子关系'
+#         app_label = 'community'
+#         db_table = 'community_pictorial_topic'
+
+#     id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
+#     pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
+#     topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
+#     user_id = models.BigIntegerField(verbose_name=u'用户ID')
+#     is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
+#     is_online = models.BooleanField(verbose_name=u'是否上线')
+#     is_deleted = models.BooleanField(verbose_name=u'是否删除')

trans2es/models/topic.py

@@ -10,6 +10,8 @@ from libs.es import ESPerform
 from django.db import models
 import datetime
 
+from alpha_types.venus import GRAP_PLATFORM
+
 from .pick_topic import PickTopic
 from .tag import TopicTag, Tag
 from .user_extra import UserExtra
@@ -70,11 +72,31 @@ class Topic(models.Model):
     create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
 
+    is_crawl = models.BooleanField(verbose_name="是否是爬回的数据", default=False)
+    has_image = models.BooleanField(verbose_name=u"是否有图片", default=False)
+    virtual_content_level = models.CharField(verbose_name=u'模拟内容等级', max_length=100, default='')
+    like_num_crawl = models.IntegerField(verbose_name=u'爬取点赞数', default=0)
+    comment_num_crawl = models.IntegerField(verbose_name=u'爬取评论数', default=0)
+    is_recommend = models.IntegerField(verbose_name=u'是否推荐 0 非推荐 1 推荐', default=0)
+    sort_score = models.FloatField(verbose_name='排序分', default=0)
+    is_shadow = models.BooleanField(verbose_name="是否是马甲账户", default=False)
+    platform = models.IntegerField(verbose_name=u'平台来源', choices=GRAP_PLATFORM, default=GRAP_PLATFORM.ALPHA)
+    platform_id = models.BigIntegerField(verbose_name='用平台ID', null=True)
+
+    @property
+    def is_complaint(self):
+        """是否被举报"""
+
+        if TopicComplaint.objects.filter(topic_id=self.id, is_online=True).exists():
+            return True
+
+        return False
+
     def topic_has_image(self):
         try:
             has_image = False
 
-            query_list = TopicImage.objects.filter(topic_id=self.id, is_deleted=False, is_online=True)
+            query_list = TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_deleted=False, is_online=True)
             if len(query_list) > 0:
                 has_image = True
 
@@ -86,7 +108,7 @@ class Topic(models.Model):
     def get_pick_id_info(self):
         try:
             pick_id_list = list()
-            query_list = PickTopic.objects.filter(topic_id=self.id, is_deleted=False)
+            query_list = PickTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_deleted=False)
             for item in query_list:
                 pick_id_list.append(item.pick_id)
 
@@ -100,8 +122,8 @@ class Topic(models.Model):
             topic_tag_id_list = list()
             edit_tag_id_list = list()
 
-            tag_id_list = TopicTag.objects.filter(topic_id=self.id).values_list("tag_id", flat=True)
-            tag_query_results = Tag.objects.filter(id__in=tag_id_list)
+            tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id).values_list("tag_id", flat=True)
+            tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=tag_id_list)
             for tag_item in tag_query_results:
                 is_online=tag_item.is_online
                 is_deleted=tag_item.is_deleted
@@ -122,7 +144,7 @@ class Topic(models.Model):
             tag_name_list = list()
             logging.info("get tag_id_list :%s" % tag_id_list)
             for i in range(0, len(tag_name_list), 1000):
-                query_results = Tag.objects.filter(id__in=tag_id_list[i:i + 1000])
+                query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=tag_id_list[i:i + 1000])
                 for item in query_results:
                     tag_name_list.append(item)
             return tag_name_list
@@ -136,7 +158,7 @@ class Topic(models.Model):
             user_is_shadow = False
 
             # 是否官方推荐用户
-            user_query_results = UserExtra.objects.filter(user_id=self.user_id)
+            user_query_results = UserExtra.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id)
             if user_query_results.count() > 0:
                 if user_query_results[0].is_recommend:
                     offline_score += 2.0
@@ -155,9 +177,9 @@ class Topic(models.Model):
             elif self.content_level == '3':
                 offline_score += 2.0
 
-            exposure_count = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=1).count()
-            click_count = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=2).count()
-            uv_num = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=3).count()
+            exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
+            click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
+            uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
 
             if exposure_count > 0:
                 offline_score += click_count / exposure_count
@@ -173,3 +195,19 @@ class Topic(models.Model):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return 0.0
+
+
+class TopicComplaint(models.Model):
+    """帖子举报"""
+
+    class Meta:
+        verbose_name = u'帖子举报'
+        app_label = 'community'
+        db_table = 'topic_complaint'
+
+    id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
+    user_id = models.BigIntegerField(verbose_name=u'用户ID', db_index=True)
+    topic = models.ForeignKey(
+        Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE, related_name='complaints')
+
+    is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
\ No newline at end of file

trans2es/models/user.py

@@ -37,7 +37,7 @@ class User(models.Model):
     def get_is_recommend_flag(self):
         is_shadow = False
         is_recommend = False
-        query_sql = UserExtra.objects.filter(user_id=self.user_id, is_deleted=False, is_online=True)
+        query_sql = UserExtra.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id, is_deleted=False, is_online=True)
         for record in query_sql:
             is_recommend = record.is_recommend
             is_shadow = record.is_shadow
@@ -48,7 +48,7 @@ class User(models.Model):
         latest_topic_time_val = -1
 
         # 获取该用户最新发帖时间
-        topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",
+        topic_records = Topic.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",
                                                                                                         flat=True).first()
         if topic_records:
             tzlc_topic_update_time = tzlc(topic_records)
@@ -65,7 +65,7 @@ class User(models.Model):
         logging.info("get follow_user_id_list :%s" % follow_user_id_list)
         for i in range(0, len(follow_user_id_list), 1000):
             logging.info("get follow_user_id_list :%s" % follow_user_id_list[i:i + 1000])
-            sql_data_list = User.objects.filter(user_id__in=follow_user_id_list[i:i + 1000])
+            sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=follow_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {
                     "user_id": detail_data.user_id,
@@ -78,7 +78,7 @@ class User(models.Model):
     def get_attention_group_id_list(self):
         try:
             attention_group_id_list = list()
-            query_results = GroupUserRole.objects.filter(is_online=True, user_id=self.user_id)
+            query_results = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True, user_id=self.user_id)
             for item in query_results:
                 item_dict = {
                     "group_id": item.group_id,
@@ -93,13 +93,13 @@ class User(models.Model):
 
     def get_pick_user_id_list(self):
         pick_topic_id_list = list()
-        user_picks = self.user_pick.filter(is_deleted=False, is_pick=True)
+        user_picks = self.user_pick.using(settings.SLAVE_DB_NAME).filter(is_deleted=False, is_pick=True)
         for user_pick in user_picks:
             pick_topic_id_list.append(user_pick.picktopic_id)
 
         pick_user_id_list = []
         for i in range(0, len(pick_topic_id_list), 1000):
-            topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list[i:i + 1000])
+            topic_sql_list = Topic.objects.using(settings.SLAVE_DB_NAME).filter(id__in=pick_topic_id_list[i:i + 1000])
             for topic_data in topic_sql_list:
                 pick_user_id_list.append(topic_data.user_id)
 
@@ -107,7 +107,7 @@ class User(models.Model):
         pick_user_detail_list = list()
 
         for i in range(0, len(pick_user_id_list), 1000):
-            sql_data_list = User.objects.filter(user_id__in=pick_user_id_list[i:i + 1000])
+            sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=pick_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {
                     "user_id": detail_data.user_id,
@@ -119,16 +119,16 @@ class User(models.Model):
 
     def get_same_group_user_id_list(self):
         same_group_user_id_list = list()
-        group_items_list = GroupUserRole.objects.filter(user_id=self.user_id)
+        group_items_list = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id)
         for group_item in group_items_list:
             group_id = group_item.group_id
-            user_items_list = GroupUserRole.objects.filter(group_id=group_id)
+            user_items_list = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(group_id=group_id)
             for user_items_list in user_items_list:
                 same_group_user_id_list.append(user_items_list.user_id)
 
         same_group_detail_list = list()
         for i in range(0, len(same_group_user_id_list), 1000):
-            sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list[i:i + 1000])
+            sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=same_group_user_id_list[i:i + 1000])
             for detail_data in sql_data_list:
                 item = {
                     "user_id": detail_data.user_id,
@@ -142,7 +142,7 @@ class User(models.Model):
         try:
             user_tag_id_list = list()
 
-            query_results = AccountUserTag.objects.filter(user=self.user_id, is_deleted=False)
+            query_results = AccountUserTag.objects.using(settings.SLAVE_DB_NAME).filter(user=self.user_id, is_deleted=False)
             for item in query_results:
                 user_tag_id_list.append(item.tag_id)