modify

libs/es.py

@@ -118,6 +118,26 @@ class ESPerform(object):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return False
 
+    @classmethod
+    def put_indices_template(cls,es_cli,template_file_name, template_name):
+        """
+        :remark put index template
+        :param es_cli:
+        :param template_file_name:
+        :param template_name:
+        :return:
+        """
+        try:
+            assert (es_cli is not None)
+
+            mapping_dict = cls.__load_mapping(template_file_name)
+            es_cli.indices.put_template(name=template_name,body=mapping_dict)
+
+            return True
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+            return False
+
     @classmethod
     def es_helpers_bulk(cls,es_cli,data_list,sub_index_name,auto_create_index=False,doc_type="_doc"):
         try:

search/views/user.py

@@ -26,6 +26,10 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
         #获取关注用户列表
         (self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id)
 
+        #去除自身及感兴趣的用户ID
+        self_attention_user_id_list.append(self_user_id)
+        self_attention_user_id_list.append(interesting_user_id)
+        
         recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size)
 
         return recommend_user_list

trans2es/management/commands/trans2es_data2es_parallel.py

@@ -58,15 +58,8 @@ class Command(BaseCommand):
         make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True),
     )
 
-    def handle(self, *args, **options):
+    def __sync_data_by_type(self, type_name):
         try:
-            type_name_list = get_type_info_map().keys()
-            for type_name in type_name_list:
-
-                if len(options["type"]) and type_name!=options["type"]:
-                    logging.warning("type_name:%s can not need to execute!" % type_name)
-                    continue
-
             type_info = get_type_info_map()[type_name]
             query_set = type_info.queryset
 
@@ -78,6 +71,18 @@ class Command(BaseCommand):
                     chunk=chunk,
                 )
                 job()
+        except:
+            logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+
+    def handle(self, *args, **options):
+        try:
+            type_name_list = get_type_info_map().keys()
+            for type_name in type_name_list:
+
+                if len(options["type"]):
+                    if options["type"] == "all" or type_name==options["type"]:
+                        logging.info("begin sync [%s] data to es!" % type_name)
+                        self.__sync_data_by_type(type_name)
 
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/management/commands/trans2es_mapping2es.py

@@ -6,16 +6,53 @@ from django.core.management.base import BaseCommand, CommandError
 import traceback
 import logging
 from libs.es import ESPerform
+from trans2es.type_info import get_type_info_map,TypeInfo
+
 
 
 class Command(BaseCommand):
 
+    args = ''
+    help = 'dump mapping to elasticsearch'
+
+    from optparse import make_option
+
+    option_list = BaseCommand.option_list + (
+        make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',default=''),
+        make_option('-T', '--indices_template', dest='indices_template', help='index template name to dump data to elasticsearch', metavar='TYPE',
+                    default=''),
+        make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch', metavar='INDEX_PREFIX'),
+        make_option('-p', '--parallel', dest='parallel', help='parallel process count', metavar='PARALLEL'),
+        make_option('-s', '--pks', dest='pks', help='specify sync pks, comma separated', metavar='PKS', default=''),
+        make_option('--streaming-slicing', dest='streaming_slicing', action='store_true', default=True),
+        make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True),
+    )
+
+
     def handle(self, *args, **options):
         try:
-            es_obj = ESPerform()
-            es_cli = es_obj.get_cli()
+            es_cli = ESPerform.get_cli()
+
+            type_name_list = get_type_info_map().keys()
+            for type_name in type_name_list:
+
+                if len(options["type"]):
+                    if options["type"] == "all" or type_name==options["type"]:
+                        official_index_name = ESPerform.get_official_index_name(type_name)
+                        index_exists = es_cli.indices.exists(official_index_name)
+                        if not index_exists:
+                            logging.info("begin create [%s] index and mapping!" % type_name)
+                            ESPerform.create_index(es_cli,type_name)
+                            ESPerform.put_index_mapping(es_cli,type_name)
+                        else:
+                            logging.warning("index:[%s] has already existing!" % type_name)
+
 
-            es_obj.create_index(es_cli,"topic")
-            es_obj.put_index_mapping(es_cli=es_cli,sub_index_name="topic")
+            if len(options["indices_template"]):
+                template_file_name = options["indices_template"]
+                if ESPerform.put_indices_template(es_cli=es_cli,template_file_name=template_file_name,template_name=template_file_name):
+                    logging.info("put indices template suc!")
+                else:
+                    logging.error("put indices template err!")
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())

trans2es/mapping/all_index_template.json

+{
+  "index_patterns": ["*"],
+  "settings":{
+    "number_of_shards": 9,
+    "number_of_replicas": 3,
+    "index":{
+      "analysis":{
+        "filter":{
+          "gm_synonym_ik_smart":{
+            "type": "synonym",
+            "synonyms_path": "analysis/synonym.txt"
+          }
+        },
+        "analyzer":{
+          "gm_default_search":{
+            "tokenizer": "ik_smart"
+          },
+          "gm_default_index":{
+            "tokenizer": "ik_max_word",
+            "filter": ["gm_synonym_ik_smart"]
+          }
+        }
+      }
+    }
+  }
+}
\ No newline at end of file

trans2es/type_info.py

@@ -258,17 +258,17 @@ def get_type_info_map():
             round_insert_chunk_size=5,
             round_insert_period=2,
         ),
-        TypeInfo(
-            name="pick_celebrity",  # 打榜明星
-            type="pick_celebrity",
-            model=pick_celebrity.PickCelebrity,
-            # query_deferred=lambda:user.User.objects.all().query,
-            query_deferred=lambda: pick_celebrity.PickCelebrity.objects.all().query,
-            get_data_func=PickCelebrityTransfer.get_pick_celebrity_data,
-            bulk_insert_chunk_size=100,
-            round_insert_chunk_size=5,
-            round_insert_period=2,
-        ),
+        # TypeInfo(
+        #     name="pick_celebrity",  # 打榜明星
+        #     type="pick_celebrity",
+        #     model=pick_celebrity.PickCelebrity,
+        #     # query_deferred=lambda:user.User.objects.all().query,
+        #     query_deferred=lambda: pick_celebrity.PickCelebrity.objects.all().query,
+        #     get_data_func=PickCelebrityTransfer.get_pick_celebrity_data,
+        #     bulk_insert_chunk_size=100,
+        #     round_insert_chunk_size=5,
+        #     round_insert_period=2,
+        # ),
         TypeInfo(
             name="celebrity",  # 明星
             type="celebrity",