Skip to content

P
physical
Commit b6da0fd6 authored by lixiaofang's avatar lixiaofang
Browse files
Options

Merge branch 'master' into day_week_mouth

parents d7d10542 4dab11a7
Hide whitespace changes
Inline Side-by-side
Showing with 177 additions and 147 deletions
linucb/views/collect_data.py
View file @ b6da0fd6
...@@ -181,7 +181,7 @@ class CollectData(object): ...@@ -181,7 +181,7 @@ class CollectData(object):
raw_val_dict = json.loads(ori_msg.value) raw_val_dict = json.loads(ori_msg.value)
if "type" in raw_val_dict and \ if "type" in raw_val_dict and \
(raw_val_dict["type"] in ("on_click_feed_topic_card","tag_zone_click_focus")): (raw_val_dict["type"] in ("on_click_feed_topic_card","on_click_button")):
click_topic_tag_list = list() click_topic_tag_list = list()
if "on_click_feed_topic_card" == raw_val_dict["type"]: if "on_click_feed_topic_card" == raw_val_dict["type"]:
...@@ -210,16 +210,18 @@ class CollectData(object): ...@@ -210,16 +210,18 @@ class CollectData(object):
logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % ( logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % (
str(device_id), str(topic_id), str(click_topic_tag_list))) str(device_id), str(topic_id), str(click_topic_tag_list)))
else: elif raw_val_dict["type"] == "on_click_button" and "page_name" in \
tag_name = raw_val_dict["params"]["query"] raw_val_dict["params"] and "button_name" in raw_val_dict["params"] \
query_type = raw_val_dict["params"]["type"] and "extra_param" in raw_val_dict["params"]:
device_id = raw_val_dict["device"]["device_id"] if raw_val_dict["params"]["page_name"] == "search_detail" and \
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None raw_val_dict["params"]["button_name"] == "focus_tag":
if query_type=="do": tag_name = raw_val_dict["params"]["extra_param"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
tag_list = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(name=tag_name,is_online=True,is_deleted=False, is_category=False).values_list("id",flat=True)) tag_list = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(name=tag_name,is_online=True,is_deleted=False, is_category=False).values_list("id",flat=True))
click_topic_tag_list.extend(tag_list) click_topic_tag_list.extend(tag_list)
logging.info("query tag attention,positive tag_list,device_id:%s,query_name:%s,tag_list:%s" % ( logging.info("query tag attention,positive tag_list,device_id:%s,query_name:%s,tag_list:%s" % (
str(device_id), tag_name, str(click_topic_tag_list))) str(device_id), tag_name, str(click_topic_tag_list)))
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list))) logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
......
search/utils/topic.py
View file @ b6da0fd6
...@@ -163,17 +163,7 @@ class TopicUtils(object): ...@@ -163,17 +163,7 @@ class TopicUtils(object):
q["query"] = dict() q["query"] = dict()
functions_list = [ functions_list = list()
{
"gauss": {
"create_time": {
"scale": "1d",
"decay": 0.99
}
},
"weight": 60
}
]
query_function_score = { query_function_score = {
"query": { "query": {
...@@ -204,7 +194,7 @@ class TopicUtils(object): ...@@ -204,7 +194,7 @@ class TopicUtils(object):
] ]
} }
}, },
"score_mode": "max", "score_mode": "sum",
"boost_mode": "sum", "boost_mode": "sum",
"functions": functions_list "functions": functions_list
} }
...@@ -228,26 +218,11 @@ class TopicUtils(object): ...@@ -228,26 +218,11 @@ class TopicUtils(object):
query_function_score["query"]["bool"]["must_not"] = [ query_function_score["query"]["bool"]["must_not"] = [
{"terms": {"user_id": linucb_user_id_list}} {"terms": {"user_id": linucb_user_id_list}}
] ]
if query is not None: # 搜索帖子 q["_source"] = {
multi_match = { "includes": ["id", "highlight", "description"]
'query': query, }
'type': 'best_fields',
'operator': 'or',
'fields': ["content", "tag_name_list"],
}
query_function_score["boost_mode"] = "replace"
tag_ids = get_same_tagset_ids(tag_id) if query is None:
query_function_score["query"]["bool"]["should"] = [
{'multi_match': multi_match},
{"terms": {"tag_list": tag_ids}},
{"term": {"user_nick_name_pre": query.lower()}}
]
query_function_score["query"]["bool"]["minimum_should_match"] = 1
query_function_score["query"]["bool"]["filter"].append(
{"range": {"content_level": {"gte": 3, "lte": 6}}}
)
else:
if "must_not" in query_function_score["query"]["bool"]: if "must_not" in query_function_score["query"]["bool"]:
query_function_score["query"]["bool"]["must_not"] += [ query_function_score["query"]["bool"]["must_not"] += [
{"term": {"is_operation_home_recommend": True}} {"term": {"is_operation_home_recommend": True}}
...@@ -264,19 +239,26 @@ class TopicUtils(object): ...@@ -264,19 +239,26 @@ class TopicUtils(object):
query_function_score["query"]["bool"]["filter"].append( query_function_score["query"]["bool"]["filter"].append(
{"term": {"content_level": 6}} {"term": {"content_level": 6}}
) )
q["_source"] = {
"includes": ["id", "highlight", "description"]
}
if query is None: # if user_id and user_id > 0:
if user_id and user_id > 0: # redis_key_prefix = "physical:user_similar:participant_user_id:"
redis_key_prefix = "physical:user_similar:participant_user_id:" # similar_redis_key = redis_key_prefix + str(user_id)
similar_redis_key = redis_key_prefix + str(user_id) # redis_user_similar_data = redis_client.get(similar_redis_key)
redis_user_similar_data = redis_client.get(similar_redis_key) # user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else [] # if len(user_similar_list) > 0:
if len(user_similar_list) > 0: # functions_list.extend(user_similar_list)
functions_list.extend(user_similar_list)
functions_list.append(
{
"gauss": {
"create_time": {
"scale": "1d",
"decay": 0.99
}
},
"weight": 60
}
)
if len(attention_user_id_list) > 0: if len(attention_user_id_list) > 0:
functions_list.append( functions_list.append(
{ {
...@@ -334,87 +316,81 @@ class TopicUtils(object): ...@@ -334,87 +316,81 @@ class TopicUtils(object):
multi_match = { multi_match = {
'query': query, 'query': query,
'type': 'best_fields', 'type': 'best_fields',
'operator': 'and', 'operator': 'or',
'fields': ["content", "tag_name_list"], 'fields': ["content", "tag_name_list"],
"analyzer":"gm_default_index"
}
query_function_score["boost_mode"] = "replace"
same_tagset_ids = get_same_tagset_ids(tag_id)
query_function_score["query"]["bool"]["should"] = [
{'multi_match': multi_match},
{"terms": {"tag_list": same_tagset_ids}},
{"term": {"user_nick_name_pre": query.lower()}}
]
query_function_score["query"]["bool"]["minimum_should_match"] = 1
query_function_score["query"]["bool"]["filter"].append(
{"range": {"content_level": {"gte": 3, "lte": 6}}}
)
collection_redis_key_name="physical:official_tag_name_set"
collect_tag_name_set=set()
body = {
'text': query,
'analyzer': "gm_default_search"
}
analyze_res = ESPerform.get_analyze_results(es_cli=ESPerform.get_cli(), sub_index_name="topic",query_body=body)
for item in analyze_res["tokens"]:
token_word = item["token"]
# is_member = redis_client.sismember(collection_redis_key_name, token_word)
# if is_member:
collect_tag_name_set.add(token_word)
query_fields = ["content", "tag_name_list"]
multi_match = {
'query': query,
'type': 'best_fields',
'operator': 'and',
'fields': query_fields,
} }
functions_list += [ functions_list += [
{ {
"weight": 400, "weight":10,
"filter": { "filter":{
"constant_score": { "term":{
"filter": { "language_type":1
"term": {"user_nick_name_pre": query.lower()}
}
}
}
},
{
"weight": 400,
"filter": {
"constant_score": {
"filter": {
"bool": {
"must": {
"term": {"content_level": 6},
},
"minimum_should_match": 1,
"should": [
{'match_phrase': {"content": query}},
{'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
}
}
},
{
"weight": 400,
"filter": {
"constant_score": {
"filter": {
"bool": {
"must": {
"term": {"content_level": 5},
},
"minimum_should_match": 1,
"should": [
{'match_phrase': {"content": query}},
{'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
} }
} }
}, },
{ {
"weight": 400, "weight": 1000,
"filter": { "filter": {
"constant_score": { "bool": {
"filter": { "minimum_should_match": 1,
"bool": { "should": [
"must": { {'match_phrase': {"content": query}},
"term": {"content_level": 4}, {'match_phrase': {"tag_name_list": query}},
}, # {'multi_match': multi_match},
"minimum_should_match": 1, {"terms": {"tag_list": same_tagset_ids}},
"should": [ {"term": {"user_nick_name_pre": query.lower()}}
{'match_phrase': {"content": query}}, ]
{'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
} }
} }
} }
] ]
for query_item in collect_tag_name_set:
for field_item in query_fields:
term_dict = {
"filter": {
"term": {
field_item: query_item
}
},
"weight": 2
}
functions_list.append(term_dict)
query_function_score["functions"] = functions_list query_function_score["functions"] = functions_list
q["query"]["function_score"] = query_function_score q["query"]["function_score"] = query_function_score
q["sort"] = [ q["sort"] = [
......
search/views/group.py
View file @ b6da0fd6
...@@ -139,7 +139,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -139,7 +139,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
} }
} }
q["_source"] = { q["_source"] = {
"includes": ["id", "pictorial_id", "tag_list"] "includes": ["id", "pictorial_id", "edit_tag_list"]
} }
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size) result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size)
pict_pictorial_ids_list = [] pict_pictorial_ids_list = []
...@@ -148,7 +148,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -148,7 +148,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
if len(result_dict["hits"]) > 0: if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]: for item in result_dict["hits"]:
pict_pictorial_ids_list = item["_source"]["pictorial_id"] pict_pictorial_ids_list = item["_source"]["pictorial_id"]
topic_tag_list = item["_source"]["tag_list"] topic_tag_list = item["_source"]["edit_tag_list"]
##拿到这些标签的同义词 ##拿到这些标签的同义词
...@@ -178,6 +178,11 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -178,6 +178,11 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"is_deleted": False "is_deleted": False
} }
}, },
{
"term": {
"is_cover": True
}
}
] ]
} }
...@@ -185,12 +190,18 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -185,12 +190,18 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
q["_source"] = { q["_source"] = {
"includes": ["id", "update_time"] "includes": ["id", "update_time"]
} }
q["sort"] = { q["sort"] = [
'update_time': { {
"real_user_activate_time": {
"order": "desc"
}
},
{'update_time': {
'order': 'desc' 'order': 'desc'
} }
}
} ]
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size) result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
if len(result_dict["hits"]) > 0: if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]: for item in result_dict["hits"]:
...@@ -208,7 +219,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -208,7 +219,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"bool": { "bool": {
"must": [{ "must": [{
"terms": { "terms": {
"tag_id": all_tag "edit_tag_id": all_tag
}}, { }}, {
"term": { "term": {
"is_online": True "is_online": True
...@@ -224,6 +235,10 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -224,6 +235,10 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
} }
} }
}, {
"term": {
"is_cover": True
}
}] }]
} }
} }
...@@ -233,7 +248,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -233,7 +248,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"constant_score": { "constant_score": {
"filter": { "filter": {
"terms": { "terms": {
"tag_id": topic_tag_list "edit_tag_id": topic_tag_list
} }
} }
} }
...@@ -251,13 +266,18 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -251,13 +266,18 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
} }
} }
q["_source"] = { q["_source"] = {
"includes": ["id", "tag_id"]} "includes": ["id", "edit_tag_id"]}
q["sort"] = [ q["sort"] = [
{ {
"_score": { "_score": {
"order": "desc" "order": "desc"
} }
}, },
{
"real_user_activate_time": {
"order": "desc"
}
},
{ {
'update_time': { 'update_time': {
'order': 'desc' 'order': 'desc'
......
search/views/tag.py
View file @ b6da0fd6
...@@ -238,7 +238,7 @@ def get_same_tagset_ids(tag_list): ...@@ -238,7 +238,7 @@ def get_same_tagset_ids(tag_list):
try: try:
all_tag = list() all_tag = list()
if isinstance(tag_list, int): if isinstance(tag_list, int):
all_tag.append(list) all_tag.append(tag_list)
else: else:
all_tag = tag_list all_tag = tag_list
...@@ -246,14 +246,16 @@ def get_same_tagset_ids(tag_list): ...@@ -246,14 +246,16 @@ def get_same_tagset_ids(tag_list):
tag_set_list_id = list( tag_set_list_id = list(
CommunityTagSetRelation.objects.filter(tag_id__in=all_tag, is_deleted=False).values_list("tag_set_id", CommunityTagSetRelation.objects.filter(tag_id__in=all_tag, is_deleted=False).values_list("tag_set_id",
flat=True)) flat=True))
logging.info("get tag_set_list_id:%s" % tag_set_list_id)
tag_ids = list( tag_ids = list(
CommunityTagSetRelation.objects.filter(tag_set_id__in=tag_set_list_id, is_deleted=False).values_list( CommunityTagSetRelation.objects.filter(tag_set_id__in=tag_set_list_id, is_deleted=False).values_list(
"tag_id", "tag_id",
flat=True)) flat=True))
return tag_ids all_tag.extend(tag_ids)
logging.info("get_same_tagset_ids:%s" % str(all_tag))
return list(set(all_tag))
except: except:
logging_exception() logging_exception()
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []} return []
trans2es/management/commands/trans2es_data2es_parallel.py
View file @ b6da0fd6
...@@ -29,6 +29,12 @@ from search.utils.common import * ...@@ -29,6 +29,12 @@ from search.utils.common import *
from linucb.views.collect_data import CollectData from linucb.views.collect_data import CollectData
from injection.data_sync.tasks import sync_user_similar_score from injection.data_sync.tasks import sync_user_similar_score
from trans2es.models.tag import Tag
from libs.cache import redis_client
from trans2es.models.tag import TopicTag
class Job(object): class Job(object):
__es = None __es = None
...@@ -189,6 +195,26 @@ class Command(BaseCommand): ...@@ -189,6 +195,26 @@ class Command(BaseCommand):
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def sync_tag_collecction_name_set(self):
try:
collection_redis_key_name = "physical:official_tag_name_set"
tag_list = list(TopicTag.objects.filter(is_online=True, is_collection=True).values_list("tag_id", flat=True))
tag_name_list = list(Tag.objects.filter(id__in=tag_list, is_online=True, is_deleted=False,
is_category=False).values_list("name", flat=True))
logging.info("duan add,tag_list_len:%d,tag_name_list_len:%d" % (len(tag_list),len(tag_name_list)))
for tag_name in tag_name_list:
try:
if tag_name and len(tag_name)>0 and tag_name[0]:
redis_client.sadd(collection_redis_key_name,tag_name[0])
except:
pass
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def handle(self, *args, **options): def handle(self, *args, **options):
try: try:
type_name_list = get_type_info_map().keys() type_name_list = get_type_info_map().keys()
...@@ -211,5 +237,9 @@ class Command(BaseCommand): ...@@ -211,5 +237,9 @@ class Command(BaseCommand):
if len(options["sync_type"]) and options["sync_type"]=="similar": if len(options["sync_type"]) and options["sync_type"]=="similar":
sync_user_similar_score() sync_user_similar_score()
if len(options["sync_type"]) and options["sync_type"]=="tagname":
self.sync_tag_collecction_name_set()
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
trans2es/type_info.py
View file @ b6da0fd6
...@@ -13,7 +13,7 @@ import elasticsearch.helpers ...@@ -13,7 +13,7 @@ import elasticsearch.helpers
import sys import sys
from libs.cache import redis_client from libs.cache import redis_client
import copy import copy
from django.conf import settings
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar, pictorial, product from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar, pictorial, product
from trans2es.utils.user_transfer import UserTransfer from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
...@@ -315,7 +315,7 @@ def get_type_info_map(): ...@@ -315,7 +315,7 @@ def get_type_info_map():
name='topic-star', name='topic-star',
type='topic-star', type='topic-star',
model=topic.Topic, model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query, # 假的 query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query, # 假的
get_data_func=TopicTransfer.get_topic_data, # 假的 get_data_func=TopicTransfer.get_topic_data, # 假的
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -325,7 +325,7 @@ def get_type_info_map(): ...@@ -325,7 +325,7 @@ def get_type_info_map():
name='topic-star-routing', name='topic-star-routing',
type='topic-star-routing', type='topic-star-routing',
model=topic.Topic, model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query, query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data, get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -335,7 +335,7 @@ def get_type_info_map(): ...@@ -335,7 +335,7 @@ def get_type_info_map():
name='topic-high-star', # >=4星日记 name='topic-high-star', # >=4星日记
type='topic-high-star', type='topic-high-star',
model=topic.Topic, model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query, query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data, get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -345,7 +345,7 @@ def get_type_info_map(): ...@@ -345,7 +345,7 @@ def get_type_info_map():
name='excellect-topic', # 优质帖子 name='excellect-topic', # 优质帖子
type='excellect-topic', type='excellect-topic',
model=topic.ExcellentTopic, model=topic.ExcellentTopic,
query_deferred=lambda: topic.ExcellentTopic.objects.all().query, query_deferred=lambda: topic.ExcellentTopic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=ExcellectTopicTransfer.get_excellect_topic_data, get_data_func=ExcellectTopicTransfer.get_excellect_topic_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -355,7 +355,7 @@ def get_type_info_map(): ...@@ -355,7 +355,7 @@ def get_type_info_map():
name='topic', # 日记 name='topic', # 日记
type='topic', type='topic',
model=topic.Topic, model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query, query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data, get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -365,7 +365,7 @@ def get_type_info_map(): ...@@ -365,7 +365,7 @@ def get_type_info_map():
name='topic-v1', # 日记 name='topic-v1', # 日记
type='topic-v1', type='topic-v1',
model=topic.Topic, model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query, query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data, get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -377,7 +377,7 @@ def get_type_info_map(): ...@@ -377,7 +377,7 @@ def get_type_info_map():
model=user.User, model=user.User,
# query_deferred=lambda:user.User.objects.all().query, # query_deferred=lambda:user.User.objects.all().query,
# query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query, # query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query,
query_deferred=lambda: user.User.objects.all().query, query_deferred=lambda: user.User.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=UserTransfer.get_user_data, get_data_func=UserTransfer.get_user_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -399,7 +399,7 @@ def get_type_info_map(): ...@@ -399,7 +399,7 @@ def get_type_info_map():
type="celebrity", type="celebrity",
model=celebrity.Celebrity, model=celebrity.Celebrity,
# query_deferred=lambda:user.User.objects.all().query, # query_deferred=lambda:user.User.objects.all().query,
query_deferred=lambda: celebrity.Celebrity.objects.all().query, query_deferred=lambda: celebrity.Celebrity.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=CelebrityTransfer.get_celebrity_data, get_data_func=CelebrityTransfer.get_celebrity_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -410,7 +410,7 @@ def get_type_info_map(): ...@@ -410,7 +410,7 @@ def get_type_info_map():
type="group", type="group",
model=group.Group, model=group.Group,
# query_deferred=lambda:user.User.objects.all().query, # query_deferred=lambda:user.User.objects.all().query,
query_deferred=lambda: group.Group.objects.all().query, query_deferred=lambda: group.Group.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=GroupTransfer.get_group_data, get_data_func=GroupTransfer.get_group_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -420,7 +420,7 @@ def get_type_info_map(): ...@@ -420,7 +420,7 @@ def get_type_info_map():
name="tag", # 标签 name="tag", # 标签
type="tag", type="tag",
model=tag.Tag, model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().query, query_deferred=lambda: tag.Tag.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TagTransfer.get_tag_data, get_data_func=TagTransfer.get_tag_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -430,7 +430,7 @@ def get_type_info_map(): ...@@ -430,7 +430,7 @@ def get_type_info_map():
name="tag_v1", # 标签 name="tag_v1", # 标签
type="tag_v1", type="tag_v1",
model=tag.Tag, model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().query, query_deferred=lambda: tag.Tag.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TagTransfer.get_tag_data, get_data_func=TagTransfer.get_tag_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -440,7 +440,7 @@ def get_type_info_map(): ...@@ -440,7 +440,7 @@ def get_type_info_map():
name="tag-name", # 标签名字 name="tag-name", # 标签名字
type="tag-name", type="tag-name",
model=tag.Tag, model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().query, query_deferred=lambda: tag.Tag.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TagTransfer.get_tag_name_data, get_data_func=TagTransfer.get_tag_name_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -450,7 +450,7 @@ def get_type_info_map(): ...@@ -450,7 +450,7 @@ def get_type_info_map():
name='contrast_similar', # facesimilar name='contrast_similar', # facesimilar
type='contrast_similar', type='contrast_similar',
model=contrast_similar.ContrastSimilar, model=contrast_similar.ContrastSimilar,
query_deferred=lambda: contrast_similar.ContrastSimilar.objects.all().query, query_deferred=lambda: contrast_similar.ContrastSimilar.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data, get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -460,7 +460,7 @@ def get_type_info_map(): ...@@ -460,7 +460,7 @@ def get_type_info_map():
name="pictorial", # 画报 name="pictorial", # 画报
type="pictorial", type="pictorial",
model=pictorial.Pictorial, model=pictorial.Pictorial,
query_deferred=lambda: pictorial.Pictorial.objects.all().query, query_deferred=lambda: pictorial.Pictorial.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=PictorialTransfer.get_poctorial_data, get_data_func=PictorialTransfer.get_poctorial_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment