sun后台帖子迁移到es

search/utils/topic.py

@@ -550,11 +550,10 @@ class TopicUtils(object):
             return list()
 
     @classmethod
-    def process_filters(cls, filters):
+    def process_filters(cls, filters, filter_online=True):
         """处理过滤器部分。"""
 
         f = [
-            {"term": {"is_online": True}},
             {"term": {"is_deleted": False}},
         ]
         if not filters:
@@ -566,6 +565,9 @@ class TopicUtils(object):
                     "term": {"group_id": v},
                 })
 
+        if filter_online:
+            f.append({"term": {"is_online": True}})
+
         return f
 
     @classmethod
@@ -599,12 +601,12 @@ class TopicUtils(object):
         return sort_rule
 
     @classmethod
-    def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10):
+    def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
 
         q = {
             "query": {
                 "bool": {
-                    "must": cls.process_filters(filters),
+                    "must": cls.process_filters(filters, filter_online=True),
                     "must_not": cls.process_nfilters(nfilters),
                 }
             },
@@ -619,8 +621,10 @@ class TopicUtils(object):
             q["sort"] = sorts
 
         try:
-            result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
-                                                       query_body=q, offset=offset, size=size)
+            result_dict = ESPerform.get_search_results(
+                ESPerform.get_cli(), sub_index_name=index_name,
+                query_body=q, offset=offset, size=size
+            )
 
             return result_dict["hits"]
         except:

search/views/business_topic.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals, absolute_import, print_function
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from search.utils.topic import TopicUtils
+from libs.es import ESPerform
+from libs.cache import redis_client
+from search.utils.common import *
+from libs.es import ESPerform
+
+
+@bind("physical/search/business/topic")
+def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
+    """帖子搜索。"""
+
+    try:
+        result_list = TopicUtils.list_topic_ids(
+            filters=filters, nfilters=nfilters, sorts_by=sorts_by,
+            offset=offset, size=size, filter_online=False
+        )
+        topic_ids = [item["_source"]["id"] for item in result_list]
+        return {"topic_ids": topic_ids}
+    except:
+        logging.error("catch exception, err_msg:%s" % traceback.format_exc())
+        return {"topic_ids": []}

trans2es/mapping/business_topic.json

+// 业务端帖子存储表做宽表查询
+{
+    "dynamic": "strict",
+    "properties":{
+        "id":{"type": "long"},
+        "user_id":{"type": "long"},
+        "is_shadow": {"type": "boolean"},
+        "is_online":{"type": "boolean"},
+        "is_deleted":{"type": "boolean"},
+        "has_image":{"type": "boolean"}, //是否有图
+        "has_video":{"type": "boolean"}, //是否是视频
+        "is_recommend": {"type": "boolean"},
+        "is_complaint": {"type": "boolean"},  // 是否被举报
+
+        "vote_num":{"type": "long"},
+        "reply_num":{"type": "long"},
+        "share_num": {"type": "long"},
+
+        "name":{
+            "type": "text",
+            "analyzer": "gm_default_index",
+            "search_analyzer": "gm_default_index"
+        },
+        "description":{
+            "type": "text",
+            "analyzer": "gm_default_index",
+            "search_analyzer": "gm_default_index"
+        },
+        "content":{
+            "type": "text",
+            "analyzer": "gm_default_index",
+            "search_analyzer": "gm_default_index"
+        },
+        "content_level":{"type": "text"},
+
+        "virtual_content_level":{"type": "text"},
+        "language_type": {"type": "long"},
+        "like_num_crawl": {"type": "long"}, // 爬取点赞数
+        "comment_num_crawl": {"type": "long"}, // 爬取评论数
+
+        "is_crawl": {"type": "long"},
+        "platform": {"type": "long"},
+        "platform_id": {"type": "long"},
+
+        "drop_score":{"type": "double"},  // 人工降分
+        "sort_score":{"type": "double"},  // 排序分
+
+        "group_id":{"type": "long"}, //所在组ID
+        "group_name":{"type": "long"}, //所在组名称
+
+        "create_time":{"type": "date", "format": "date_time_no_millis"},
+        "update_time":{"type": "date", "format": "date_time_no_millis"},
+        "create_time_val":{"type": "long"},
+        "update_time_val":{"type": "long"},
+
+        "tag_list":{"type": "long"}, // 标签属性
+        "edit_tag_list":{"type": "long"}, //编辑标签
+        "tag_name_list":{
+            "type": "text",
+            "analyzer": "gm_default_index",
+            "search_analyzer": "gm_default_index"
+        }
+    }
+}
\ No newline at end of file

trans2es/models/topic.py

@@ -70,6 +70,15 @@ class Topic(models.Model):
     create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
 
+    @property
+    def is_complaint(self):
+        """是否被举报"""
+
+        if TopicComplaint.objects.filter(topic_id=self.id, is_online=True).exists():
+            return True
+
+        return False
+
     def topic_has_image(self):
         try:
             has_image = False
@@ -173,3 +182,19 @@ class Topic(models.Model):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return 0.0
+
+
+class TopicComplaint(models.Model):
+    """帖子举报"""
+
+    class Meta:
+        verbose_name = u'帖子举报'
+        app_label = 'community'
+        db_table = 'topic_complaint'
+
+    id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
+    user_id = models.BigIntegerField(verbose_name=u'用户ID', db_index=True)
+    topic = models.ForeignKey(
+        Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE, related_name='complaints')
+
+    is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
\ No newline at end of file

trans2es/type_info.py

@@ -265,6 +265,16 @@ def get_type_info_map():
             round_insert_chunk_size=5,
             round_insert_period=2,
         ),
+        TypeInfo(
+            name='business_topic',  # 运营后台
+            type='business_topic',
+            model=topic.Topic,
+            query_deferred=lambda: topic.Topic.objects.all().query,
+            get_data_func=TopicTransfer.get_business_topic_data,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
+        ),
         TypeInfo(
             name = "user", #用户
             type="user",

trans2es/utils/topic_transfer.py

@@ -94,4 +94,68 @@ class TopicTransfer(object):
             return res
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return None
\ No newline at end of file
+            return None
+
+    @classmethod
+    def get_business_topic_data(cls,instance):
+        try:
+            res = {
+                "id": instance.id,
+                "user_id": instance.user_id,
+                "is_shadow": instance.is_shadow,
+                "is_online": instance.is_online,
+                "is_deleted": instance.is_deleted,
+                "has_image": instance.topic_has_image(),
+                "has_video": instance.has_video,
+                "is_recommend": instance.is_recommend,
+                "is_complaint": instance.is_complaint,
+
+                "vote_num": instance.vote_num,
+                "reply_num": instance.reply_num,
+                "share_num": instance.share_num,
+
+                "name": instance.name,
+                "description": instance.description,
+                "content": instance.content,
+                "content_level": instance.content_level,
+
+                "virtual_content_level": instance.virtual_content_level,
+                "language_type": instance.language_type,
+                "like_num_crawl": instance.like_num_crawl,
+                "comment_num_crawl": instance.comment_num_crawl,
+
+                "is_crawl": instance.is_crawl,
+                "platform": instance.platform,
+                "platform_id": instance.platform_id,
+
+                "drop_score": instance.drop_score,
+                "sort_score": instance.sort_score,
+            }
+
+            if instance.group:
+                res["group_id"] = instance.group.id
+                res["group_name"] = instance.group.name
+            else:
+                res["group_id"] = -1
+                res["group_name"] = ""
+
+            (topic_tag_id_list, edit_tag_id_list) = instance.get_topic_tag_id_list()
+            res["tag_list"] = topic_tag_id_list
+            res["edit_tag_list"] = edit_tag_id_list
+            res["tag_name_list"] = instance.get_tag_name_list(res["tag_list"]),
+
+            create_time = instance.create_time
+            tzlc_create_time = tzlc(create_time)
+            res["create_time"] = tzlc_create_time
+            res["create_time_val"] = int(time.mktime(tzlc_create_time.timetuple()))
+
+            update_time = instance.update_time
+            tzlc_update_time = tzlc(update_time)
+            res["update_time"] = tzlc_update_time
+            res["update_time_val"] = int(time.mktime(tzlc_update_time.timetuple()))
+
+            return res
+
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return None