Skip to content

P
physical
Commit 45e3a766 authored by lixiaofang's avatar lixiaofang
Browse files
Options

update es

parent dc437ac3
Hide whitespace changes
Inline Side-by-side
Showing with 110 additions and 88 deletions
.idea/workspace.xml
View file @ 45e3a766
......@@ -24,7 +24,7 @@
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="json" value="11" />
<entry key="py" value="72" />
<entry key="py" value="74" />
<entry key="txt" value="1" />
<entry key="xml" value="1" />
</counts>
......@@ -33,7 +33,7 @@
<counts>
<entry key="JSON" value="11" />
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="72" />
<entry key="Python" value="74" />
<entry key="XML" value="1" />
</counts>
</usages-collector>
......@@ -64,19 +64,41 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/libs/es.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-135">
<caret line="174" column="13" lean-forward="true" selection-start-line="174" selection-end-line="175" />
<folding>
<marker date="1548062012000" expanded="true" signature="5756:5757" ph="..." />
<marker date="1548062012000" expanded="true" signature="5985:5986" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="136">
<state relative-caret-position="1065">
<caret line="78" column="43" selection-start-line="78" selection-start-column="43" selection-end-line="78" selection-end-column="43" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/models/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-120">
<caret line="17" column="46" selection-start-line="17" selection-start-column="20" selection-end-line="17" selection-end-column="46" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="99">
<state relative-caret-position="357">
<caret line="303" column="34" selection-start-line="303" selection-start-column="18" selection-end-line="303" selection-end-column="34" />
<folding>
<element signature="e#15#87#0" expanded="true" />
......@@ -138,20 +160,21 @@
<option value="$PROJECT_DIR$/trans2es/mapping/contrast_similar.json" />
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar_three.py" />
<option value="$PROJECT_DIR$/trans2es/management/commands/trans2es_mapping2es.py" />
<option value="$PROJECT_DIR$/libs/es.py" />
<option value="$PROJECT_DIR$/trans2es/utils/contrast_similar_transfer.py" />
<option value="$PROJECT_DIR$/trans2es/type_info.py" />
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py" />
<option value="$PROJECT_DIR$/trans2es/models/contrast_similar.py" />
<option value="$PROJECT_DIR$/physical/settings.py" />
<option value="$PROJECT_DIR$/search/views/contrast_similar.py" />
<option value="$PROJECT_DIR$/libs/es.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
<component name="ProjectFrameBounds">
<option name="x" value="-13" />
<option name="y" value="23" />
<option name="width" value="1280" />
<option name="height" value="735" />
<option name="height" value="736" />
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
......@@ -183,6 +206,27 @@
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="libs" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
<item name="models" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
......@@ -229,10 +273,10 @@
<servers />
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="736" extended-state="6" />
<frame x="-13" y="23" width="1280" height="736" extended-state="0" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.28911835" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.28911835" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info id="DB Browser" order="3" weight="0.32962668" />
......@@ -245,7 +289,7 @@
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" weight="0.017770598" />
<window_info anchor="bottom" id="DB Execution Console" order="8" />
<window_info active="true" anchor="bottom" id="Terminal" order="9" visible="true" weight="0.2753623" />
<window_info anchor="bottom" id="Terminal" order="9" visible="true" weight="0.273752" />
<window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
<window_info anchor="bottom" id="Python Console" order="11" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
......@@ -458,20 +502,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="101">
<caret line="17" column="46" selection-start-line="17" selection-start-column="20" selection-end-line="17" selection-end-column="46" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/libs/es.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="227">
<caret line="27" lean-forward="true" selection-start-line="27" selection-end-line="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/physical/settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="116">
......@@ -479,13 +509,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="136">
<caret line="78" column="43" selection-start-line="78" selection-start-column="43" selection-end-line="78" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/contrast_similar.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
......@@ -505,7 +528,7 @@
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/type_info.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="99">
<state relative-caret-position="357">
<caret line="303" column="34" selection-start-line="303" selection-start-column="18" selection-end-line="303" selection-end-column="34" />
<folding>
<element signature="e#15#87#0" expanded="true" />
......@@ -513,6 +536,31 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1065">
<caret line="78" column="43" selection-start-line="78" selection-start-column="43" selection-end-line="78" selection-end-column="43" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/models/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-120">
<caret line="17" column="46" selection-start-line="17" selection-start-column="20" selection-end-line="17" selection-end-column="46" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/libs/es.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-135">
<caret line="174" column="13" lean-forward="true" selection-start-line="174" selection-end-line="175" />
<folding>
<marker date="1548062012000" expanded="true" signature="5756:5757" ph="..." />
<marker date="1548062012000" expanded="true" signature="5985:5986" ph="..." />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
......
libs/es.py
View file @ 45e3a766
......@@ -17,6 +17,7 @@ class ESPerform(object):
cli_info_list = settings.ES_INFO_LIST
index_prefix = settings.ES_INDEX_PREFIX
@classmethod
def get_cli(cls):
try:
......@@ -27,7 +28,7 @@ class ESPerform(object):
return None
@classmethod
def get_official_index_name(cls, sub_index_name, index_flag=None):
def get_official_index_name(cls,sub_index_name,index_flag=None):
"""
:remark:get official es index name
:param sub_index_name:
......@@ -35,7 +36,7 @@ class ESPerform(object):
:return:
"""
try:
assert (index_flag in [None, "read", "write"])
assert (index_flag in [None,"read","write"])
official_index_name = cls.index_prefix + "-" + sub_index_name
if index_flag:
......@@ -47,11 +48,11 @@ class ESPerform(object):
return None
@classmethod
def __load_mapping(cls, doc_type):
def __load_mapping(cls,doc_type):
try:
mapping_file_path = os.path.join(
os.path.dirname(__file__),
'..', 'trans2es', 'mapping', '%s.json' % (doc_type,))
'..', 'trans2es','mapping', '%s.json' % (doc_type,))
mapping = ''
with open(mapping_file_path, 'r') as f:
for line in f:
......@@ -64,7 +65,7 @@ class ESPerform(object):
return None
@classmethod
def create_index(cls, es_cli, sub_index_name):
def create_index(cls,es_cli,sub_index_name):
"""
:remark: create es index,alias index
:param sub_index_name:
......@@ -77,11 +78,11 @@ class ESPerform(object):
index_exist = es_cli.indices.exists(official_index_name)
if not index_exist:
es_cli.indices.create(official_index_name)
read_alias_name = cls.get_official_index_name(sub_index_name, "read")
es_cli.indices.put_alias(official_index_name, read_alias_name)
read_alias_name = cls.get_official_index_name(sub_index_name,"read")
es_cli.indices.put_alias(official_index_name,read_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
es_cli.indices.put_alias(official_index_name, write_alias_name)
write_alias_name = cls.get_official_index_name(sub_index_name,"write")
es_cli.indices.put_alias(official_index_name,write_alias_name)
return True
except:
......@@ -89,7 +90,7 @@ class ESPerform(object):
return False
@classmethod
def put_index_mapping(cls, es_cli, sub_index_name, mapping_type="_doc"):
def put_index_mapping(cls,es_cli,sub_index_name,mapping_type="_doc"):
"""
:remark: put index mapping
:param es_cli:
......@@ -100,13 +101,13 @@ class ESPerform(object):
try:
assert (es_cli is not None)
write_alias_name = cls.get_official_index_name(sub_index_name, "write")
write_alias_name = cls.get_official_index_name(sub_index_name,"write")
index_exist = es_cli.indices.exists(write_alias_name)
if not index_exist:
return False
mapping_dict = cls.__load_mapping(sub_index_name)
es_cli.indices.put_mapping(index=write_alias_name, body=mapping_dict, doc_type=mapping_type)
es_cli.indices.put_mapping(index=write_alias_name,body=mapping_dict,doc_type=mapping_type)
return True
except:
......@@ -114,7 +115,7 @@ class ESPerform(object):
return False
@classmethod
def put_indices_template(cls, es_cli, template_file_name, template_name):
def put_indices_template(cls,es_cli,template_file_name, template_name):
"""
:remark put index template
:param es_cli:
......@@ -126,7 +127,7 @@ class ESPerform(object):
assert (es_cli is not None)
mapping_dict = cls.__load_mapping(template_file_name)
es_cli.indices.put_template(name=template_name, body=mapping_dict)
es_cli.indices.put_template(name=template_name,body=mapping_dict)
return True
except:
......@@ -134,7 +135,7 @@ class ESPerform(object):
return False
@classmethod
def es_helpers_bulk(cls, es_cli, data_list, sub_index_name, auto_create_index=False, doc_type="_doc"):
def es_helpers_bulk(cls,es_cli,data_list,sub_index_name,auto_create_index=False,doc_type="_doc"):
try:
assert (es_cli is not None)
......@@ -145,8 +146,8 @@ class ESPerform(object):
logging.error("index:%s is not existing,bulk data error!" % official_index_name)
return False
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
cls.create_index(es_cli,sub_index_name)
cls.put_index_mapping(es_cli,sub_index_name)
bulk_actions = []
for data in data_list:
......@@ -157,7 +158,7 @@ class ESPerform(object):
'_id': data['id'],
'_source': data,
})
elasticsearch.helpers.bulk(es_cli, bulk_actions)
elasticsearch.helpers.bulk(es_cli,bulk_actions)
return True
except:
......@@ -165,61 +166,34 @@ class ESPerform(object):
return False
@classmethod
<<<<<<< HEAD
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False,
is_suggest_request=False):
=======
def get_search_results(cls, es_cli,sub_index_name,query_body,offset=0,size=10,
auto_create_index=False,doc_type="_doc",aggregations_query=False,is_suggest_request=False,batch_search=False):
>>>>>>> 338c6f92836042068a01d3670f4b581b206fd71d
auto_create_index=False,doc_type="_doc",aggregations_query=False,is_suggest_request=False):
try:
assert (es_cli is not None)
official_index_name = cls.get_official_index_name(sub_index_name, "read")
official_index_name = cls.get_official_index_name(sub_index_name,"read")
index_exists = es_cli.indices.exists(official_index_name)
if not index_exists:
if not auto_create_index:
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
else:
cls.create_index(es_cli, sub_index_name)
cls.put_index_mapping(es_cli, sub_index_name)
cls.create_index(es_cli,sub_index_name)
cls.put_index_mapping(es_cli,sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
<<<<<<< HEAD
res = es_cli.search(index=official_index_name, doc_type=doc_type, body=query_body, from_=offset, size=size)
=======
>>>>>>> 338c6f92836042068a01d3670f4b581b206fd71d
if not batch_search:
res = es_cli.search(index=official_index_name,doc_type=doc_type,body=query_body,from_=offset,size=size)
res = es_cli.search(index=official_index_name,doc_type=doc_type,body=query_body,from_=offset,size=size)
if is_suggest_request:
return res
else:
result_dict = {
"total_count":res["hits"]["total"],
"hits":res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
if is_suggest_request:
return res
else:
<<<<<<< HEAD
result_dict = {
"total_count": res["hits"]["total"],
"hits": res["hits"]["hits"]
"total_count":res["hits"]["total"],
"hits":res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
=======
res = es_cli.msearch(body=query_body,index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
>>>>>>> 338c6f92836042068a01d3670f4b581b206fd71d
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count": 0, "hits": []}
return {"total_count":0,"hits":[]}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment