Skip to content

G
graces_es6
Commit 3e327384 authored by lixiaofang's avatar lixiaofang
Browse files
Options

删掉es_config参数

parent 36dd08e1
Hide whitespace changes
Inline Side-by-side
Showing with 47 additions and 57 deletions
.idea/workspace.xml
View file @ 3e327384
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="d124fa2a-32cc-4b9e-a06a-0fa9d00e5734" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
</list>
<list default="true" id="d124fa2a-32cc-4b9e-a06a-0fa9d00e5734" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -22,11 +20,13 @@
<counts>
<entry key="json" value="5" />
<entry key="py" value="1" />
<entry key="txt" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="JSON" value="5" />
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="1" />
</counts>
</usages-collector>
......@@ -44,11 +44,11 @@
</component>
<component name="FileEditorManager">
<leaf>
<file pinned="false" current-in-tab="false">
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="289">
<caret line="69" column="36" selection-start-line="69" selection-start-column="36" selection-end-line="69" selection-end-column="56" />
<state relative-caret-position="1065">
<caret line="71" column="102" selection-start-line="71" selection-start-column="85" selection-end-line="71" selection-end-column="102" />
</state>
</provider>
</entry>
......@@ -89,7 +89,7 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/gaia/log_settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
......@@ -101,16 +101,26 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/requirements/common.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="208">
<caret line="14" column="20" selection-start-line="14" selection-start-column="20" selection-end-line="14" selection-end-column="20" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>&quot;type&quot;:&quot;string&quot;</find>
<find>/data/lo</find>
<find>&quot;not_analyzed&quot;</find>
<find>index</find>
<find>&quot;type&quot;: &quot;string&quot;</find>
<find>/data/lo</find>
<find>&quot;index&quot;: &quot;not_analyzed&quot;</find>
<find>elasticsearch</find>
</findStrings>
<replaceStrings>
<replace>&quot;type&quot;:&quot;text&quot;</replace>
......@@ -129,11 +139,12 @@
<option value="$PROJECT_DIR$/trans2es/mapping/sku.json" />
<option value="$PROJECT_DIR$/trans2es/mapping/sku_norms_disable.json" />
<option value="$PROJECT_DIR$/gaia/log_settings.py" />
<option value="$PROJECT_DIR$/requirements/common.txt" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="-323" />
<component name="ProjectFrameBounds" extendedState="1">
<option name="x" value="-317" />
<option name="y" value="-1080" />
<option name="width" value="1920" />
<option name="height" value="1080" />
......@@ -168,17 +179,6 @@
<item name="graces_es6" type="b2602c69:ProjectViewProjectNode" />
<item name="graces_es6" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="graces_es6" type="b2602c69:ProjectViewProjectNode" />
<item name="graces_es6" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="graces_es6" type="b2602c69:ProjectViewProjectNode" />
<item name="graces_es6" type="462c0819:PsiDirectoryNode" />
<item name="trans2es" type="462c0819:PsiDirectoryNode" />
<item name="mapping" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
......@@ -215,17 +215,17 @@
<servers />
</component>
<component name="ToolWindowManager">
<frame x="-323" y="-1080" width="1920" height="1080" extended-state="0" />
<frame x="-317" y="-1080" width="1920" height="1080" extended-state="1" />
<editor active="true" />
<layout>
<window_info id="DB Browser" />
<window_info id="Favorites" side_tool="true" />
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.19325961" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.19325961" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="DB Execution Console" />
<window_info anchor="bottom" id="Version Control" show_stripe_button="false" />
<window_info anchor="bottom" id="Python Console" />
<window_info active="true" anchor="bottom" id="Terminal" visible="true" weight="0.018652849" />
<window_info anchor="bottom" id="Terminal" visible="true" weight="0.22797927" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
......@@ -243,13 +243,6 @@
<option name="myLimit" value="2678400000" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="289">
<caret line="69" column="36" selection-start-line="69" selection-start-column="36" selection-end-line="69" selection-end-column="56" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service_ngram.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-3072">
......@@ -288,6 +281,20 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/requirements/common.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="208">
<caret line="14" column="20" selection-start-line="14" selection-start-column="20" selection-end-line="14" selection-end-column="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/mapping/service.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1065">
<caret line="71" column="102" selection-start-line="71" selection-start-column="85" selection-end-line="71" selection-end-column="102" />
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
......
injection/data_sync/tasks.py
View file @ 3e327384
......@@ -12,15 +12,14 @@ from rpc.all import get_rpc_remote_invoker
@shared_task
@thread_local(DB_FOR_READ_OVERRIDE=settings.DBMW_SLAVE_DB_NAME)
def write_to_es(es_type, pk_list, configuration, use_batch_query_set=False, es_config=None):
def write_to_es(es_type, pk_list, configuration, use_batch_query_set=False):
pk_list = list(frozenset(pk_list))
type_info_map = get_type_info_map()
type_info = type_info_map[es_type]
type_info.insert_table_by_pk_list(
index_prefix=settings.ES_INDEX_PREFIX,
pk_list=pk_list,
use_batch_query_set=use_batch_query_set,
es_config=es_config
use_batch_query_set=use_batch_query_set
)
......
trans2es/type_info.py
View file @ 3e327384
......@@ -53,14 +53,9 @@ def get_talos_es6_instance():
return __es6
def get_es_list_by_type(es_type, es_config=None):
def get_es_list_by_type(es_type):
if es_type in ['service', 'sku']:
if es_config == True:
es_list = [get_talos_es6_instance()]
else:
es_list = [get_talos_elasticsearch_instance()]
logging.info("get ----------es_list:%s" % es_list)
es_list = [get_talos_elasticsearch_instance()]
else:
es_list = [get_elasticsearch_instance()]
return es_list
......@@ -170,9 +165,9 @@ class TypeInfo(object):
data_list.append(data)
return data_list
def elasticsearch_bulk_insert_data(self, index_prefix, data_list, es=None, es_config=None):
def elasticsearch_bulk_insert_data(self, index_prefix, data_list, es=None):
if es is None:
es = get_es_list_by_type(self.type, es_config=es_config)
es = get_es_list_by_type(self.type)
if not isinstance(es, (list, tuple,)):
es = [es]
......@@ -190,24 +185,14 @@ class TypeInfo(object):
bulk_actions = []
if self.type == 'newitemwiki' or self.type == 'collectwiki' or self.type == 'brandwiki' or self.type == 'productwiki':
for data in data_list:
bulk_actions.append({
'_op_type': 'index',
'_index': index,
'_type': index_type,
'_id': data['id'] + data["wikitype"] * 10000,
'_source': data,
})
elif es_config == True:
for data in data_list:
bulk_actions.append({
'_op_type': 'index',
'_index': index,
'_type': "_doc",
'_id': data['id'],
'_id': data['id'] + data["wikitype"] * 10000,
'_source': data,
})
else:
for data in data_list:
bulk_actions.append({
......@@ -238,7 +223,7 @@ class TypeInfo(object):
es=es,
)
def insert_table_by_pk_list(self, index_prefix, pk_list, es=None, use_batch_query_set=False, es_config=None):
def insert_table_by_pk_list(self, index_prefix, pk_list, es=None, use_batch_query_set=False):
if use_batch_query_set:
qs = self.queryset
else:
......@@ -249,8 +234,7 @@ class TypeInfo(object):
self.elasticsearch_bulk_insert_data(
index_prefix=index_prefix,
data_list=data_list,
es=es,
es_config=es_config
es=es
)
def insert_table_chunk(self, index_prefix, table_chunk, es=None):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment