帖子同步到es

app_conf.xml

@@ -9,6 +9,7 @@
     <config name="initializer_list">
         <element value="physical.django_init"/>
         <element value="search.views.topic"/>
+        <element value="search.views.business_topic"/>
         <element value="search.views.pick"/>
         <element value="search.views.group"/>
         <element value="search.views.user"/>

search/utils/topic.py

@@ -560,10 +560,30 @@ class TopicUtils(object):
             return f
 
         for k, v in filters.items():
-            if k == "group_id":
+
+            if k in ["create_time_gte", "create_time_lte"]:
+
+                if k == "create_time_gte":
+                    op = "gte"
+                elif k == "create_time_lte":
+                    op = "lte"
+
                 f.append({
-                    "term": {"group_id": v},
+                    "range": {
+                        "create_time_val": {
+                            op: v,
+                        }
+                    }
                 })
+            else:
+                if isinstance(k, list):
+                    f.append({
+                        "terms": {k: v},
+                    })
+                else:
+                    f.append({
+                        "term": {k: v},
+                    })
 
         if filter_online:
             f.append({"term": {"is_online": True}})
@@ -598,22 +618,29 @@ class TopicUtils(object):
                 },
             })
 
+        elif sorts_by == TOPIC_SEARCH_SORT.VOTE_NUM:
+            pass
+
+        elif sorts_by == '2':
+            sort_rule.append({
+                "id":{
+                    "order":"desc"
+                },
+            })
+
         return sort_rule
 
     @classmethod
     def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
 
+        must = cls.process_filters(filters, filter_online=True)
         q = {
             "query": {
                 "bool": {
-                    "must": cls.process_filters(filters, filter_online=True),
+                    "must": must,
                     "must_not": cls.process_nfilters(nfilters),
                 }
-            },
-            "_source": {
-                "includes":["id"]
-            },
-            "sort": [],
+            }
         }
 
         if sorts_by:
@@ -626,7 +653,13 @@ class TopicUtils(object):
                 query_body=q, offset=offset, size=size
             )
 
-            return result_dict["hits"]
+            return {
+                "hits": result_dict["hits"],
+                "total_count": result_dict["total_count"]
+            }
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return []
+            return {
+                "hits": [],
+                "total_count": 0
+            }

search/views/business_topic.py

@@ -14,16 +14,17 @@ from libs.es import ESPerform
 
 
 @bind("physical/search/business/topic")
-def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
+def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
     """帖子搜索。"""
 
     try:
         result_list = TopicUtils.list_topic_ids(
             filters=filters, nfilters=nfilters, sorts_by=sorts_by,
-            offset=offset, size=size, filter_online=False
+            offset=offset, size=size, filter_online=False,
+            index_name="business_topic"
         )
-        topic_ids = [item["_source"]["id"] for item in result_list]
-        return {"topic_ids": topic_ids}
+        topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
+        return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
     except:
         logging.error("catch exception, err_msg:%s" % traceback.format_exc())
-        return {"topic_ids": []}
+        return {"topic_ids": [], "total_count": 0}

search/views/topic.py

@@ -285,11 +285,14 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
     try:
         result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
                                                 sorts_by=sorts_by, offset=offset, size=size)
-        topic_ids = [item["_source"]["id"] for item in result_list]
-        return {"topic_ids": topic_ids}
+        topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
+        return {
+            "topic_ids": topic_ids,
+            "total_count": result_list["total_count"]
+        }
     except:
         logging.error("catch exception, err_msg:%s" % traceback.format_exc())
-        return {"topic_ids": []}
+        return {"topic_ids": [], "total_count": 0}
 
 
 @bind("physical/search/query_topic_by_user_similarity")

trans2es/mapping/business_topic.json

-// 业务端帖子存储表做宽表查询
-{
-    "dynamic": "strict",
-    "properties":{
-        "id":{"type": "long"},
-        "user_id":{"type": "long"},
-        "is_shadow": {"type": "boolean"},
-        "is_online":{"type": "boolean"},
-        "is_deleted":{"type": "boolean"},
-        "has_image":{"type": "boolean"}, //是否有图
-        "has_video":{"type": "boolean"}, //是否是视频
-        "is_recommend": {"type": "boolean"},
-        "is_complaint": {"type": "boolean"},  // 是否被举报
-
-        "vote_num":{"type": "long"},
-        "reply_num":{"type": "long"},
-        "share_num": {"type": "long"},
-
-        "name":{
-            "type": "text",
-            "analyzer": "gm_default_index",
-            "search_analyzer": "gm_default_index"
-        },
-        "description":{
-            "type": "text",
-            "analyzer": "gm_default_index",
-            "search_analyzer": "gm_default_index"
-        },
-        "content":{
-            "type": "text",
-            "analyzer": "gm_default_index",
-            "search_analyzer": "gm_default_index"
-        },
-        "content_level":{"type": "text"},
-
-        "virtual_content_level":{"type": "text"},
-        "language_type": {"type": "long"},
-        "like_num_crawl": {"type": "long"}, // 爬取点赞数
-        "comment_num_crawl": {"type": "long"}, // 爬取评论数
-
-        "is_crawl": {"type": "boolean"},
-        "platform": {"type": "long"},
-        "platform_id": {"type": "long"},
-
-        "drop_score":{"type": "double"},  // 人工降分
-        "sort_score":{"type": "double"},  // 排序分
-
-        "pictorial_id":{"type": "long"}, //所在组ID
-        "pictorial_name":{  // 所在组名称
-            "type": "text",
-            "analyzer": "gm_default_index",
-            "search_analyzer": "gm_default_index"
-        },
-
-        "create_time":{"type": "date", "format": "date_time_no_millis"},
-        "update_time":{"type": "date", "format": "date_time_no_millis"},
-        "create_time_val":{"type": "long"},
-        "update_time_val":{"type": "long"},
-
-        "tag_list":{"type": "long"}, // 标签属性
-        "edit_tag_list":{"type": "long"}, //编辑标签
-        "tag_name_list":{
-            "type": "text",
-            "analyzer": "gm_default_index",
-            "search_analyzer": "gm_default_index"
-        }
-    }
-}
\ No newline at end of file

trans2es/mapping/topic.json

@@ -25,6 +25,29 @@
     "update_time":{"type":"date", "format":"date_time_no_millis"},
     "create_time_val":{"type":"long"},
     "update_time_val":{"type":"long"},
-    "language_type":{"type":"long"}
+    "language_type":{"type":"long"},
+
+    "is_shadow": {"type": "boolean"},
+    "is_recommend": {"type": "boolean"},
+    "is_complaint": {"type": "boolean"},  // 是否被举报
+
+    "virtual_content_level":{"type": "text"},
+    "like_num_crawl": {"type": "long"}, // 爬取点赞数
+    "comment_num_crawl": {"type": "long"}, // 爬取评论数
+
+    "is_crawl": {"type": "boolean"},
+    "platform": {"type": "long"},
+    "platform_id": {"type": "long"},
+
+    "drop_score":{"type": "double"},  // 人工降分
+    "sort_score":{"type": "double"},  // 排序分
+
+    "pictorial_id":{"type": "long"}, //所在组ID
+    "pictorial_name":{  // 所在组名称
+        "type": "text",
+        "analyzer": "gm_default_index",
+        "search_analyzer": "gm_default_index"
+    }
+
   }
 }
\ No newline at end of file

trans2es/models/__init__.py

@@ -13,4 +13,4 @@ from .tag import AccountUserTag
 from .user import User
 from .group import Group
 from .topic import Topic
-from .pictorial import PictorialTopic
+# from .pictorial import PictorialTopic

trans2es/models/pictorial.py

-from django.db import models
+# from django.db import models
 
 
-class PictorialTopic(models.Model):
-    """画报帖子关系"""
+# class PictorialTopic(models.Model):
+#     """画报帖子关系"""
 
-    class Meta:
-        verbose_name = u'画报帖子关系'
-        app_label = 'community'
-        db_table = 'community_pictorial_topic'
+#     class Meta:
+#         verbose_name = u'画报帖子关系'
+#         app_label = 'community'
+#         db_table = 'community_pictorial_topic'
 
-    id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
-    pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
-    topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
-    user_id = models.BigIntegerField(verbose_name=u'用户ID')
-    is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
-    is_online = models.BooleanField(verbose_name=u'是否上线')
-    is_deleted = models.BooleanField(verbose_name=u'是否删除')
+#     id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
+#     pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
+#     topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
+#     user_id = models.BigIntegerField(verbose_name=u'用户ID')
+#     is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
+#     is_online = models.BooleanField(verbose_name=u'是否上线')
+#     is_deleted = models.BooleanField(verbose_name=u'是否删除')

trans2es/type_info.py

@@ -265,16 +265,6 @@ def get_type_info_map():
             round_insert_chunk_size=5,
             round_insert_period=2,
         ),
-        TypeInfo(
-            name='business_topic',  # 运营后台
-            type='business_topic',
-            model=topic.Topic,
-            query_deferred=lambda: topic.Topic.objects.all().query,
-            get_data_func=TopicTransfer.get_business_topic_data,
-            bulk_insert_chunk_size=100,
-            round_insert_chunk_size=5,
-            round_insert_period=2,
-        ),
         TypeInfo(
             name = "user", #用户
             type="user",

trans2es/utils/topic_transfer.py

@@ -94,61 +94,4 @@ class TopicTransfer(object):
             return res
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return None
-
+            return None
\ No newline at end of file
-    @classmethod
-    def get_business_topic_data(cls,instance):
-        try:
-            res = {
-                "id": instance.id,
-                "user_id": instance.user_id,
-                "is_shadow": instance.is_shadow,
-                "is_online": instance.is_online,
-                "is_deleted": instance.is_deleted,
-                "has_image": instance.topic_has_image(),
-                "has_video": instance.has_video,
-                "is_recommend": True if instance.is_recommend else False,
-                "is_complaint": instance.is_complaint,
-
-                "vote_num": instance.vote_num,
-                "reply_num": instance.reply_num,
-                "share_num": instance.share_num,
-
-                "name": instance.name,
-                "description": instance.description,
-                "content": instance.content,
-                "content_level": instance.content_level,
-
-                "virtual_content_level": instance.virtual_content_level,
-                "language_type": instance.language_type,
-                "like_num_crawl": instance.like_num_crawl,
-                "comment_num_crawl": instance.comment_num_crawl,
-
-                "is_crawl": instance.is_crawl,
-                "platform": instance.platform,
-                "platform_id": instance.platform_id,
-
-                "drop_score": instance.drop_score,
-                "sort_score": instance.sort_score,
-            }
-
-            (topic_tag_id_list, edit_tag_id_list) = instance.get_topic_tag_id_list()
-            res["tag_list"] = topic_tag_id_list
-            res["edit_tag_list"] = edit_tag_id_list
-            res["tag_name_list"] = instance.get_tag_name_list(res["tag_list"]),
-
-            create_time = instance.create_time
-            tzlc_create_time = tzlc(create_time)
-            res["create_time"] = tzlc_create_time
-            res["create_time_val"] = int(time.mktime(tzlc_create_time.timetuple()))
-
-            update_time = instance.update_time
-            tzlc_update_time = tzlc(update_time)
-            res["update_time"] = tzlc_update_time
-            res["update_time_val"] = int(time.mktime(tzlc_update_time.timetuple()))
-
-            return res
-
-        except:
-            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return None