update contrast params

libs/es.py

@@ -17,7 +17,6 @@ class ESPerform(object):
     cli_info_list = settings.ES_INFO_LIST
     index_prefix = settings.ES_INDEX_PREFIX
 
-
     @classmethod
     def get_cli(cls):
         try:
@@ -28,7 +27,7 @@ class ESPerform(object):
             return None
 
     @classmethod
-    def get_official_index_name(cls,sub_index_name,index_flag=None):
+    def get_official_index_name(cls, sub_index_name, index_flag=None):
         """
         :remark:get official es index name
         :param sub_index_name:
@@ -36,7 +35,7 @@ class ESPerform(object):
         :return:
         """
         try:
-            assert (index_flag in [None,"read","write"])
+            assert (index_flag in [None, "read", "write"])
 
             official_index_name = cls.index_prefix + "-" + sub_index_name
             if index_flag:
@@ -48,11 +47,11 @@ class ESPerform(object):
             return None
 
     @classmethod
-    def __load_mapping(cls,doc_type):
+    def __load_mapping(cls, doc_type):
         try:
             mapping_file_path = os.path.join(
                 os.path.dirname(__file__),
-                '..', 'trans2es','mapping', '%s.json' % (doc_type,))
+                '..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
             mapping = ''
             with open(mapping_file_path, 'r') as f:
                 for line in f:
@@ -65,7 +64,7 @@ class ESPerform(object):
             return None
 
     @classmethod
-    def create_index(cls,es_cli,sub_index_name):
+    def create_index(cls, es_cli, sub_index_name):
         """
         :remark: create es index,alias index
         :param sub_index_name:
@@ -78,11 +77,11 @@ class ESPerform(object):
             index_exist = es_cli.indices.exists(official_index_name)
             if not index_exist:
                 es_cli.indices.create(official_index_name)
-                read_alias_name = cls.get_official_index_name(sub_index_name,"read")
-                es_cli.indices.put_alias(official_index_name,read_alias_name)
+                read_alias_name = cls.get_official_index_name(sub_index_name, "read")
+                es_cli.indices.put_alias(official_index_name, read_alias_name)
 
-                write_alias_name = cls.get_official_index_name(sub_index_name,"write")
-                es_cli.indices.put_alias(official_index_name,write_alias_name)
+                write_alias_name = cls.get_official_index_name(sub_index_name, "write")
+                es_cli.indices.put_alias(official_index_name, write_alias_name)
 
             return True
         except:
@@ -90,7 +89,7 @@ class ESPerform(object):
             return False
 
     @classmethod
-    def put_index_mapping(cls,es_cli,sub_index_name,mapping_type="_doc"):
+    def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc"):
         """
         :remark: put index mapping
         :param es_cli:
@@ -101,13 +100,13 @@ class ESPerform(object):
         try:
             assert (es_cli is not None)
 
-            write_alias_name = cls.get_official_index_name(sub_index_name,"write")
+            write_alias_name = cls.get_official_index_name(sub_index_name, "write")
             index_exist = es_cli.indices.exists(write_alias_name)
             if not index_exist:
                 return False
 
             mapping_dict = cls.__load_mapping(sub_index_name)
-            es_cli.indices.put_mapping(index=write_alias_name,body=mapping_dict,doc_type=mapping_type)
+            es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
 
             return True
         except:
@@ -115,7 +114,7 @@ class ESPerform(object):
             return False
 
     @classmethod
-    def put_indices_template(cls,es_cli,template_file_name, template_name):
+    def put_indices_template(cls, es_cli, template_file_name, template_name):
         """
         :remark put index template
         :param es_cli:
@@ -127,7 +126,7 @@ class ESPerform(object):
             assert (es_cli is not None)
 
             mapping_dict = cls.__load_mapping(template_file_name)
-            es_cli.indices.put_template(name=template_name,body=mapping_dict)
+            es_cli.indices.put_template(name=template_name, body=mapping_dict)
 
             return True
         except:
@@ -135,7 +134,7 @@ class ESPerform(object):
             return False
 
     @classmethod
-    def es_helpers_bulk(cls,es_cli,data_list,sub_index_name,auto_create_index=False,doc_type="_doc"):
+    def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
         try:
             assert (es_cli is not None)
 
@@ -146,8 +145,8 @@ class ESPerform(object):
                     logging.error("index:%s is not existing,bulk data error!" % official_index_name)
                     return False
                 else:
-                    cls.create_index(es_cli,sub_index_name)
-                    cls.put_index_mapping(es_cli,sub_index_name)
+                    cls.create_index(es_cli, sub_index_name)
+                    cls.put_index_mapping(es_cli, sub_index_name)
 
             bulk_actions = []
             for data in data_list:
@@ -158,7 +157,7 @@ class ESPerform(object):
                     '_id': data['id'],
                     '_source': data,
                 })
-            elasticsearch.helpers.bulk(es_cli,bulk_actions)
+            elasticsearch.helpers.bulk(es_cli, bulk_actions)
 
             return True
         except:
@@ -166,34 +165,35 @@ class ESPerform(object):
             return False
 
     @classmethod
-    def get_search_results(cls, es_cli,sub_index_name,query_body,offset=0,size=10,
-                           auto_create_index=False,doc_type="_doc",aggregations_query=False,is_suggest_request=False):
+    def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
+                           auto_create_index=False, doc_type="_doc", aggregations_query=False,
+                           is_suggest_request=False):
         try:
             assert (es_cli is not None)
 
-            official_index_name = cls.get_official_index_name(sub_index_name,"read")
+            official_index_name = cls.get_official_index_name(sub_index_name, "read")
             index_exists = es_cli.indices.exists(official_index_name)
             if not index_exists:
                 if not auto_create_index:
                     logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
                     return None
                 else:
-                    cls.create_index(es_cli,sub_index_name)
-                    cls.put_index_mapping(es_cli,sub_index_name)
+                    cls.create_index(es_cli, sub_index_name)
+                    cls.put_index_mapping(es_cli, sub_index_name)
 
             logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
-            res = es_cli.search(index=official_index_name,doc_type=doc_type,body=query_body,from_=offset,size=size)
+            res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset, size=size)
 
             if is_suggest_request:
                 return res
             else:
                 result_dict = {
-                    "total_count":res["hits"]["total"],
-                    "hits":res["hits"]["hits"]
+                    "total_count": res["hits"]["total"],
+                    "hits": res["hits"]["hits"]
                 }
                 if aggregations_query:
                     result_dict["aggregations"] = res["aggregations"]
                 return result_dict
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-            return {"total_count":0,"hits":[]}
+            return {"total_count": 0, "hits": []}
\ No newline at end of file

trans2es/management/commands/trans2es_mapping2es.py

@@ -6,29 +6,29 @@ from django.core.management.base import BaseCommand, CommandError
 import traceback
 import logging
 from libs.es import ESPerform
-from trans2es.type_info import get_type_info_map,TypeInfo
-
+from trans2es.type_info import get_type_info_map, TypeInfo
 
 
 class Command(BaseCommand):
-
     args = ''
     help = 'dump mapping to elasticsearch'
 
     from optparse import make_option
 
     option_list = BaseCommand.option_list + (
-        make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',default=''),
-        make_option('-T', '--indices_template', dest='indices_template', help='index template name to dump data to elasticsearch', metavar='TYPE',
+        make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',
+                    default=''),
+        make_option('-T', '--indices_template', dest='indices_template',
+                    help='index template name to dump data to elasticsearch', metavar='TYPE',
                     default=''),
-        make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch', metavar='INDEX_PREFIX'),
+        make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch',
+                    metavar='INDEX_PREFIX'),
         make_option('-p', '--parallel', dest='parallel', help='parallel process count', metavar='PARALLEL'),
         make_option('-s', '--pks', dest='pks', help='specify sync pks, comma separated', metavar='PKS', default=''),
         make_option('--streaming-slicing', dest='streaming_slicing', action='store_true', default=True),
         make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True),
     )
 
-
     def handle(self, *args, **options):
         try:
             es_cli = ESPerform.get_cli()
@@ -37,20 +37,20 @@ class Command(BaseCommand):
             for type_name in type_name_list:
 
                 if len(options["type"]):
-                    if options["type"] == "all" or type_name==options["type"]:
+                    if options["type"] == "all" or type_name == options["type"]:
                         official_index_name = ESPerform.get_official_index_name(type_name)
                         index_exists = es_cli.indices.exists(official_index_name)
                         if not index_exists:
                             logging.info("begin create [%s] index and mapping!" % type_name)
-                            ESPerform.create_index(es_cli,type_name)
-                            ESPerform.put_index_mapping(es_cli,type_name)
+                            ESPerform.create_index(es_cli, type_name)
+                            ESPerform.put_index_mapping(es_cli, type_name)
                         else:
                             logging.warning("index:[%s] has already existing!" % type_name)
 
-
             if len(options["indices_template"]):
                 template_file_name = options["indices_template"]
-                if ESPerform.put_indices_template(es_cli=es_cli,template_file_name=template_file_name,template_name=template_file_name):
+                if ESPerform.put_indices_template(es_cli=es_cli, template_file_name=template_file_name,
+                                                  template_name=template_file_name):
                     logging.info("put indices template suc!")
                 else:
                     logging.error("put indices template err!")

trans2es/type_info.py

@@ -304,7 +304,7 @@ def get_type_info_map():
             type='contrast_similar',
             model=contrast_similar.ContrastSimilar,
             query_deferred=lambda: contrast_similar.ContrastSimilar.objects.all().query,
-            get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data(),
+            get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data,
             bulk_insert_chunk_size=100,
             round_insert_chunk_size=5,
             round_insert_period=2