Skip to content

S
search_tips
Commit cff79741 authored by lixiaofang's avatar lixiaofang
Browse files
Options

同步数据到es

parent 886736ef
Hide whitespace changes
Inline Side-by-side
Showing with 91 additions and 92 deletions
trans2es/management/commands/trans2es_data2es_parallel.py
View file @ cff79741
......@@ -11,8 +11,8 @@ import sys
from libs.es import ESPerform
import trans2es.models as md
from libs.table_scan import TableSlicer,TableSlicerChunk
from trans2es.type_info import get_type_info_map,TypeInfo
from libs.table_scan import TableSlicer, TableSlicerChunk
from trans2es.type_info import get_type_info_map, TypeInfo
from libs.cache import redis_client
import json
......@@ -46,22 +46,22 @@ class Job(object):
)
class Command(BaseCommand):
args = ''
help = 'dump data to elasticsearch, parallel'
from optparse import make_option
option_list = BaseCommand.option_list + (
make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',default=''),
make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch', metavar='INDEX_PREFIX'),
make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',
default=''),
make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch',
metavar='INDEX_PREFIX'),
make_option('-p', '--parallel', dest='parallel', help='parallel process count', metavar='PARALLEL'),
make_option('-s', '--pks', dest='pks', help='specify sync pks, comma separated', metavar='PKS', default=''),
make_option('--streaming-slicing', dest='streaming_slicing', action='store_true', default=True),
make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True),
make_option('-S', '--sync_type',dest='sync_type', help='sync data to es',metavar='TYPE',default='')
make_option('-S', '--sync_type', dest='sync_type', help='sync data to es', metavar='TYPE', default='')
)
def __sync_data_by_type(self, type_name):
......@@ -86,7 +86,7 @@ class Command(BaseCommand):
for type_name in type_name_list:
if len(options["type"]):
if options["type"] == "all" or type_name==options["type"]:
if options["type"] == "all" or type_name == options["type"]:
logging.info("begin sync [%s] data to es!" % type_name)
self.__sync_data_by_type(type_name)
......
trans2es/management/commands/trans2es_mapping2es.py
View file @ cff79741
......@@ -40,7 +40,6 @@ class Command(BaseCommand):
if len(options["type"]):
if options["type"] == "all" or type_name == options["type"] or type_name1 == options["type"]:
type_name = options["type"]
official_index_name = ESPerform.get_official_index_name(type_name)
index_exists = es_cli.indices.exists(official_index_name)
......
trans2es/type_info.py
View file @ cff79741
......@@ -354,89 +354,89 @@ def get_type_info_map():
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='doctor_tips', # doctor
model=doctor.Doctor,
query_deferred=lambda: doctor.Doctor.objects.all().filter(doctor_type=DOCTOR_TYPE.DOCTOR).query,
get_data_func=DoctorTransfer.get_doctor_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='hospital_tips', # hospital
model=doctor.Hospital,
# query_deferred=lambda: doctor.Doctor.objects.all().filter(doctor_type=DOCTOR_TYPE.OFFICER).query,
query_deferred=lambda: doctor.Hospital.objects.all().query,
get_data_func=HospitalTransfer.get_hospital_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='itemwiki_tips', # itemwiki
model=itemwiki.ItemWiki,
query_deferred=lambda: itemwiki.ItemWiki.objects.all().query,
get_data_func=ItemWikiTransfer.get_itemwiki_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='collectwiki_tips', # collectwiki
model=collectwiki.CollectWiki,
query_deferred=lambda: collectwiki.CollectWiki.objects.all().query,
get_data_func=CollectWikiTransfer.get_collectwiki_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='brandwiki_tips', # brandwiki
model=brandwiki.BrandWiki,
query_deferred=lambda: brandwiki.BrandWiki.objects.all().query,
get_data_func=BrandWikiTransfer.get_brandwiki_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='productwiki_tips', # productwiki
model=productwiki.ProductWiki,
query_deferred=lambda: productwiki.ProductWiki.objects.all().query,
get_data_func=ProduceWikiTransfer.get_productwiki_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='tag_tips', # tag
model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().filter(
tag_type__in=[TAG_TYPE.BODY_PART, TAG_TYPE.BODY_PART_SUB_ITEM, TAG_TYPE.ITEM_WIKI]).query,
get_data_func=TagTransfer.get_tag_suggest_data_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='suggest-v1',
type='wordrel_tips', # tag
model=wordresemble.WordRel,
query_deferred=lambda: wordresemble.WordRel.objects.filter(
category__in=[0, 1, 2, 9, 10, 11, 12, 13, 14, 15, 16, 17]).query,
get_data_func=WordResemble.get_resemble_list,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
)
# TypeInfo(
# name='suggest-v1',
# type='doctor_tips', # doctor
# model=doctor.Doctor,
# query_deferred=lambda: doctor.Doctor.objects.all().filter(doctor_type=DOCTOR_TYPE.DOCTOR).query,
# get_data_func=DoctorTransfer.get_doctor_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='hospital_tips', # hospital
# model=doctor.Hospital,
# # query_deferred=lambda: doctor.Doctor.objects.all().filter(doctor_type=DOCTOR_TYPE.OFFICER).query,
# query_deferred=lambda: doctor.Hospital.objects.all().query,
# get_data_func=HospitalTransfer.get_hospital_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='itemwiki_tips', # itemwiki
# model=itemwiki.ItemWiki,
# query_deferred=lambda: itemwiki.ItemWiki.objects.all().query,
# get_data_func=ItemWikiTransfer.get_itemwiki_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='collectwiki_tips', # collectwiki
# model=collectwiki.CollectWiki,
# query_deferred=lambda: collectwiki.CollectWiki.objects.all().query,
# get_data_func=CollectWikiTransfer.get_collectwiki_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='brandwiki_tips', # brandwiki
# model=brandwiki.BrandWiki,
# query_deferred=lambda: brandwiki.BrandWiki.objects.all().query,
# get_data_func=BrandWikiTransfer.get_brandwiki_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='productwiki_tips', # productwiki
# model=productwiki.ProductWiki,
# query_deferred=lambda: productwiki.ProductWiki.objects.all().query,
# get_data_func=ProduceWikiTransfer.get_productwiki_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='tag_tips', # tag
# model=tag.Tag,
# query_deferred=lambda: tag.Tag.objects.all().filter(
# tag_type__in=[TAG_TYPE.BODY_PART, TAG_TYPE.BODY_PART_SUB_ITEM, TAG_TYPE.ITEM_WIKI]).query,
# get_data_func=TagTransfer.get_tag_suggest_data_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# ),
# TypeInfo(
# name='suggest-v1',
# type='wordrel_tips', # tag
# model=wordresemble.WordRel,
# query_deferred=lambda: wordresemble.WordRel.objects.filter(
# category__in=[0, 1, 2, 9, 10, 11, 12, 13, 14, 15, 16, 17]).query,
# get_data_func=WordResemble.get_resemble_list,
# bulk_insert_chunk_size=100,
# round_insert_chunk_size=5,
# round_insert_period=2,
# )
]
type_info_map = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment