merge

trans2es/type_info.py

@@ -81,6 +81,9 @@ class TypeInfo(object):
 
     def bulk_get_data(self, instance_iterable):
         data_list = []
+        # 4，5星帖子单独索引
+        topic_data_high_star_list = list()
+
         if self.batch_get_data_func:
             _pk_list = [getattr(instance, 'pk', None) for instance in instance_iterable]
             not_found_pk_list = []
@@ -137,7 +140,10 @@ class TypeInfo(object):
                     ))
                 else:
                     data_list.append(data)
-        return data_list
+                    if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
+                        topic_data_high_star_list.append(data)
+
+        return (data_list,topic_data_high_star_list)
 
     def elasticsearch_bulk_insert_data(self, sub_index_name, data_list, es=None):
 
@@ -162,10 +168,10 @@ class TypeInfo(object):
         #             traceback.print_exc()
         #             es_result = 'error'
 
-        return ESPerform.es_helpers_bulk(es, data_list, sub_index_name, True)
+        return ESPerform.es_helpers_bulk(es, data_list, sub_index_name)
 
     def elasticsearch_bulk_insert(self, sub_index_name, instance_iterable, es=None):
-        data_list = self.bulk_get_data(instance_iterable)
+        data_list, topic_data_high_star_list = self.bulk_get_data(instance_iterable)
         return self.elasticsearch_bulk_insert_data(
             sub_index_name=sub_index_name,
             data_list=data_list,
@@ -188,7 +194,7 @@ class TypeInfo(object):
         time1=end-begin
 
         begin = time.time()
-        data_list = self.bulk_get_data(instance_list)
+        data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
         end = time.time()
         time2=end-begin
 
@@ -201,6 +207,14 @@ class TypeInfo(object):
             data_list=data_list,
             es=es,
         )
+        # 同时写4星及以上的帖子
+        if len(topic_data_high_star_list)>0:
+            self.elasticsearch_bulk_insert_data(
+                sub_index_name="topic-high-star",
+                data_list=topic_data_high_star_list,
+                es=es,
+            )
+
         end = time.time()
         time3=end-begin
         logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0,time1,time2,time3))
@@ -215,7 +229,7 @@ class TypeInfo(object):
 
             stage_1_time = time.time()
 
-            data_list = self.bulk_get_data(instance_list)
+            data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
 
             stage_2_time = time.time()
 
@@ -260,7 +274,16 @@ def get_type_info_map():
         return _get_type_info_map_result
 
     type_info_list = [
-
+        TypeInfo(
+            name='topic-high-star',  # >=4星日记
+            type='topic-high-star',
+            model=topic.Topic,
+            query_deferred=lambda: topic.Topic.objects.all().query,
+            get_data_func=TopicTransfer.get_topic_data,
+            bulk_insert_chunk_size=100,
+            round_insert_chunk_size=5,
+            round_insert_period=2,
+        ),
         TypeInfo(
             name='topic',  # 日记
             type='topic',