Skip to content

G
graces_es6
Commit 3e327384 authored by lixiaofang's avatar lixiaofang
Browse files
Options

删掉es_config参数

parent 36dd08e1
Hide whitespace changes
Inline Side-by-side
Showing with 47 additions and 57 deletions
.idea/workspace.xml
View file @ 3e327384
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="d124fa2a-32cc-4b9e-a06a-0fa9d00e5734" name="Default Changelist" comment=""> <list default="true" id="d124fa2a-32cc-4b9e-a06a-0fa9d00e5734" name="Default Changelist" comment="" />
<change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
...@@ -22,11 +20,13 @@ ...@@ -22,11 +20,13 @@
<counts> <counts>
<entry key="json" value="5" /> <entry key="json" value="5" />
<entry key="py" value="1" /> <entry key="py" value="1" />
<entry key="txt" value="1" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.open"> <usages-collector id="statistics.file.types.open">
<counts> <counts>
<entry key="JSON" value="5" /> <entry key="JSON" value="5" />
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="1" /> <entry key="Python" value="1" />
</counts> </counts>
</usages-collector> </usages-collector>
...@@ -44,11 +44,11 @@ ...@@ -44,11 +44,11 @@
</component> </component>
<component name="FileEditorManager"> <component name="FileEditorManager">
<leaf> <leaf>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json"> <entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="289"> <state relative-caret-position="1065">
<caret line="69" column="36" selection-start-line="69" selection-start-column="36" selection-end-line="69" selection-end-column="56" /> <caret line="71" column="102" selection-start-line="71" selection-start-column="85" selection-end-line="71" selection-end-column="102" />
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -89,7 +89,7 @@ ...@@ -89,7 +89,7 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/gaia/log_settings.py"> <entry file="file://$PROJECT_DIR$/gaia/log_settings.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180"> <state relative-caret-position="180">
...@@ -101,16 +101,26 @@ ...@@ -101,16 +101,26 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/requirements/common.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="208">
<caret line="14" column="20" selection-start-line="14" selection-start-column="20" selection-end-line="14" selection-end-column="20" />
</state>
</provider>
</entry>
</file>
</leaf> </leaf>
</component> </component>
<component name="FindInProjectRecents"> <component name="FindInProjectRecents">
<findStrings> <findStrings>
<find>&quot;type&quot;:&quot;string&quot;</find> <find>&quot;type&quot;:&quot;string&quot;</find>
<find>/data/lo</find>
<find>&quot;not_analyzed&quot;</find> <find>&quot;not_analyzed&quot;</find>
<find>index</find> <find>index</find>
<find>&quot;type&quot;: &quot;string&quot;</find> <find>&quot;type&quot;: &quot;string&quot;</find>
<find>/data/lo</find>
<find>&quot;index&quot;: &quot;not_analyzed&quot;</find> <find>&quot;index&quot;: &quot;not_analyzed&quot;</find>
<find>elasticsearch</find>
</findStrings> </findStrings>
<replaceStrings> <replaceStrings>
<replace>&quot;type&quot;:&quot;text&quot;</replace> <replace>&quot;type&quot;:&quot;text&quot;</replace>
...@@ -129,11 +139,12 @@ ...@@ -129,11 +139,12 @@
<option value="$PROJECT_DIR$/trans2es/mapping/sku.json" /> <option value="$PROJECT_DIR$/trans2es/mapping/sku.json" />
<option value="$PROJECT_DIR$/trans2es/mapping/sku_norms_disable.json" /> <option value="$PROJECT_DIR$/trans2es/mapping/sku_norms_disable.json" />
<option value="$PROJECT_DIR$/gaia/log_settings.py" /> <option value="$PROJECT_DIR$/gaia/log_settings.py" />
<option value="$PROJECT_DIR$/requirements/common.txt" />
</list> </list>
</option> </option>
</component> </component>
<component name="ProjectFrameBounds"> <component name="ProjectFrameBounds" extendedState="1">
<option name="x" value="-323" /> <option name="x" value="-317" />
<option name="y" value="-1080" /> <option name="y" value="-1080" />
<option name="width" value="1920" /> <option name="width" value="1920" />
<option name="height" value="1080" /> <option name="height" value="1080" />
...@@ -168,17 +179,6 @@ ...@@ -168,17 +179,6 @@
<item name="graces_es6" type="b2602c69:ProjectViewProjectNode" /> <item name="graces_es6" type="b2602c69:ProjectViewProjectNode" />
<item name="graces_es6" type="462c0819:PsiDirectoryNode" /> <item name="graces_es6" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path>
<item name="graces_es6" type="b2602c69:ProjectViewProjectNode" />
<item name="graces_es6" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="graces_es6" type="b2602c69:ProjectViewProjectNode" />
<item name="graces_es6" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
<item name="mapping" type="462c0819:PsiDirectoryNode" />
</path>
</expand> </expand>
<select /> <select />
</subPane> </subPane>
...@@ -215,17 +215,17 @@ ...@@ -215,17 +215,17 @@
<servers /> <servers />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="-323" y="-1080" width="1920" height="1080" extended-state="0" /> <frame x="-317" y="-1080" width="1920" height="1080" extended-state="1" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info id="DB Browser" /> <window_info id="DB Browser" />
<window_info id="Favorites" side_tool="true" /> <window_info id="Favorites" side_tool="true" />
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.19325961" /> <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.19325961" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" /> <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="DB Execution Console" /> <window_info anchor="bottom" id="DB Execution Console" />
<window_info anchor="bottom" id="Version Control" show_stripe_button="false" /> <window_info anchor="bottom" id="Version Control" show_stripe_button="false" />
<window_info anchor="bottom" id="Python Console" /> <window_info anchor="bottom" id="Python Console" />
<window_info active="true" anchor="bottom" id="Terminal" visible="true" weight="0.018652849" /> <window_info anchor="bottom" id="Terminal" visible="true" weight="0.22797927" />
<window_info anchor="bottom" id="Event Log" side_tool="true" /> <window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" /> <window_info anchor="bottom" id="Find" order="1" />
...@@ -243,13 +243,6 @@ ...@@ -243,13 +243,6 @@
<option name="myLimit" value="2678400000" /> <option name="myLimit" value="2678400000" />
</component> </component>
<component name="editorHistoryManager"> <component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="289">
<caret line="69" column="36" selection-start-line="69" selection-start-column="36" selection-end-line="69" selection-end-column="56" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service_ngram.json"> <entry file="file://$PROJECT_DIR$/trans2es/mapping/service_ngram.json">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-3072"> <state relative-caret-position="-3072">
...@@ -288,6 +281,20 @@ ...@@ -288,6 +281,20 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/requirements/common.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="208">
<caret line="14" column="20" selection-start-line="14" selection-start-column="20" selection-end-line="14" selection-end-column="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1065">
<caret line="71" column="102" selection-start-line="71" selection-start-column="85" selection-end-line="71" selection-end-column="102" />
</state>
</provider>
</entry>
</component> </component>
<component name="masterDetails"> <component name="masterDetails">
<states> <states>
......
injection/data_sync/tasks.py
View file @ 3e327384
...@@ -12,15 +12,14 @@ from rpc.all import get_rpc_remote_invoker ...@@ -12,15 +12,14 @@ from rpc.all import get_rpc_remote_invoker
@shared_task @shared_task
@thread_local(DB_FOR_READ_OVERRIDE=settings.DBMW_SLAVE_DB_NAME) @thread_local(DB_FOR_READ_OVERRIDE=settings.DBMW_SLAVE_DB_NAME)
def write_to_es(es_type, pk_list, configuration, use_batch_query_set=False, es_config=None): def write_to_es(es_type, pk_list, configuration, use_batch_query_set=False):
pk_list = list(frozenset(pk_list)) pk_list = list(frozenset(pk_list))
type_info_map = get_type_info_map() type_info_map = get_type_info_map()
type_info = type_info_map[es_type] type_info = type_info_map[es_type]
type_info.insert_table_by_pk_list( type_info.insert_table_by_pk_list(
index_prefix=settings.ES_INDEX_PREFIX, index_prefix=settings.ES_INDEX_PREFIX,
pk_list=pk_list, pk_list=pk_list,
use_batch_query_set=use_batch_query_set, use_batch_query_set=use_batch_query_set
es_config=es_config
) )
......
trans2es/type_info.py
View file @ 3e327384
...@@ -53,14 +53,9 @@ def get_talos_es6_instance(): ...@@ -53,14 +53,9 @@ def get_talos_es6_instance():
return __es6 return __es6
def get_es_list_by_type(es_type, es_config=None): def get_es_list_by_type(es_type):
if es_type in ['service', 'sku']: if es_type in ['service', 'sku']:
es_list = [get_talos_elasticsearch_instance()]
if es_config == True:
es_list = [get_talos_es6_instance()]
else:
es_list = [get_talos_elasticsearch_instance()]
logging.info("get ----------es_list:%s" % es_list)
else: else:
es_list = [get_elasticsearch_instance()] es_list = [get_elasticsearch_instance()]
return es_list return es_list
...@@ -170,9 +165,9 @@ class TypeInfo(object): ...@@ -170,9 +165,9 @@ class TypeInfo(object):
data_list.append(data) data_list.append(data)
return data_list return data_list
def elasticsearch_bulk_insert_data(self, index_prefix, data_list, es=None, es_config=None): def elasticsearch_bulk_insert_data(self, index_prefix, data_list, es=None):
if es is None: if es is None:
es = get_es_list_by_type(self.type, es_config=es_config) es = get_es_list_by_type(self.type)
if not isinstance(es, (list, tuple,)): if not isinstance(es, (list, tuple,)):
es = [es] es = [es]
...@@ -190,24 +185,14 @@ class TypeInfo(object): ...@@ -190,24 +185,14 @@ class TypeInfo(object):
bulk_actions = [] bulk_actions = []
if self.type == 'newitemwiki' or self.type == 'collectwiki' or self.type == 'brandwiki' or self.type == 'productwiki': if self.type == 'newitemwiki' or self.type == 'collectwiki' or self.type == 'brandwiki' or self.type == 'productwiki':
for data in data_list:
bulk_actions.append({
'_op_type': 'index',
'_index': index,
'_type': index_type,
'_id': data['id'] + data["wikitype"] * 10000,
'_source': data,
})
elif es_config == True:
for data in data_list: for data in data_list:
bulk_actions.append({ bulk_actions.append({
'_op_type': 'index', '_op_type': 'index',
'_index': index, '_index': index,
'_type': "_doc", '_type': "_doc",
'_id': data['id'], '_id': data['id'] + data["wikitype"] * 10000,
'_source': data, '_source': data,
}) })
else: else:
for data in data_list: for data in data_list:
bulk_actions.append({ bulk_actions.append({
...@@ -238,7 +223,7 @@ class TypeInfo(object): ...@@ -238,7 +223,7 @@ class TypeInfo(object):
es=es, es=es,
) )
def insert_table_by_pk_list(self, index_prefix, pk_list, es=None, use_batch_query_set=False, es_config=None): def insert_table_by_pk_list(self, index_prefix, pk_list, es=None, use_batch_query_set=False):
if use_batch_query_set: if use_batch_query_set:
qs = self.queryset qs = self.queryset
else: else:
...@@ -249,8 +234,7 @@ class TypeInfo(object): ...@@ -249,8 +234,7 @@ class TypeInfo(object):
self.elasticsearch_bulk_insert_data( self.elasticsearch_bulk_insert_data(
index_prefix=index_prefix, index_prefix=index_prefix,
data_list=data_list, data_list=data_list,
es=es, es=es
es_config=es_config
) )
def insert_table_chunk(self, index_prefix, table_chunk, es=None): def insert_table_chunk(self, index_prefix, table_chunk, es=None):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment