modify

trans2es/mapping/topic-high-star.json

+{
+  "dynamic":"strict",
+  "properties": {
+    "id":{"type":"long"},
+    "is_online":{"type":"boolean"},//上线
+    "is_deleted":{"type":"boolean"},
+    "vote_num":{"type":"long"},
+    "reply_num":{"type":"long"},
+    "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "content_level":{"type":"text"},
+    "user_id":{"type":"long"},
+    "group_id":{"type":"long"}, //所在组ID
+    "tag_list":{"type":"long"},//标签属性
+    "edit_tag_list":{"type":"long"},//编辑标签
+    "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
+    "share_num":{"type":"long"},
+    "pick_id_list":{"type":"long"},
+    "offline_score":{"type":"double"},//离线算分
+    "manual_score":{"type":"double"},//人工赋分
+    "has_image":{"type":"boolean"},//是否有图
+    "has_video":{"type":"boolean"},//是否是视频
+    "create_time":{"type":"date", "format":"date_time_no_millis"},
+    "update_time":{"type":"date", "format":"date_time_no_millis"},
+    "create_time_val":{"type":"long"},
+    "update_time_val":{"type":"long"},
+    "language_type":{"type":"long"},
+
+    "is_shadow": {"type": "boolean"},
+    "is_recommend": {"type": "boolean"},
+    "is_complaint": {"type": "boolean"},  // 是否被举报
+
+    "virtual_content_level":{"type": "text"},
+    "like_num_crawl": {"type": "long"}, // 爬取点赞数
+    "comment_num_crawl": {"type": "long"}, // 爬取评论数
+
+    "is_crawl": {"type": "boolean"},
+    "platform": {"type": "long"},
+    "platform_id": {"type": "long"},
+
+    "drop_score":{"type": "double"},  // 人工降分
+    "sort_score":{"type": "double"},  // 排序分
+
+    "pictorial_id":{"type": "long"}, //所在组ID
+    "pictorial_name":{  // 所在组名称
+        "type": "text",
+        "analyzer": "gm_default_index",
+        "search_analyzer": "gm_default_index"
+    }
+
+  }
+}
\ No newline at end of file

trans2es/type_info.py

@@ -80,6 +80,8 @@ class TypeInfo(object):
 
     def bulk_get_data(self, instance_iterable):
         data_list = []
+        # 3，4，5星帖子单独索引
+        topic_data_high_star_list = list()
         if self.batch_get_data_func:
             _pk_list = [getattr(instance, 'pk', None) for instance in instance_iterable]
             not_found_pk_list = []
@@ -136,7 +138,9 @@ class TypeInfo(object):
                     ))
                 else:
                     data_list.append(data)
-        return data_list
+                    if self.type=="topic" and instance.content_level>=3:
+                        topic_data_high_star_list.append(data)
+        return (data_list,topic_data_high_star_list)
 
     def elasticsearch_bulk_insert_data(self, sub_index_name, data_list, es=None):
 
@@ -164,7 +168,7 @@ class TypeInfo(object):
         return ESPerform.es_helpers_bulk(es, data_list, sub_index_name, True)
 
     def elasticsearch_bulk_insert(self, sub_index_name, instance_iterable, es=None):
-        data_list = self.bulk_get_data(instance_iterable)
+        data_list, topic_data_high_star_list = self.bulk_get_data(instance_iterable)
         return self.elasticsearch_bulk_insert_data(
             sub_index_name=sub_index_name,
             data_list=data_list,
@@ -187,7 +191,7 @@ class TypeInfo(object):
         time1=end-begin
 
         begin = time.time()
-        data_list = self.bulk_get_data(instance_list)
+        data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
         end = time.time()
         time2=end-begin
 
@@ -197,6 +201,14 @@ class TypeInfo(object):
             data_list=data_list,
             es=es,
         )
+
+        # 同时写3星及以上的帖子
+        if len(topic_data_high_star_list)>0:
+            self.elasticsearch_bulk_insert_data(
+                sub_index_name="topic-high-star",
+                data_list=topic_data_high_star_list,
+                es=es,
+            )
         end = time.time()
         time3=end-begin
         logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0,time1,time2,time3))
@@ -211,7 +223,7 @@ class TypeInfo(object):
 
             stage_1_time = time.time()
 
-            data_list = self.bulk_get_data(instance_list)
+            data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
 
             stage_2_time = time.time()
 
@@ -256,6 +268,16 @@ def get_type_info_map():
         return _get_type_info_map_result
 
     type_info_list = [
+        TypeInfo(
+            name='topic-high-star',  # 高星日记
+            type='topic-high-star',
+            model=topic.Topic,
+            query_deferred=lambda: topic.Topic.objects.all().query,
+            get_data_func=TopicTransfer.get_topic_data,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
+        ),
         TypeInfo(
             name='topic',  # 日记
             type='topic',