Skip to content

S
search_tips
Commit 6c4ef049 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parent a6d68011
Hide whitespace changes
Inline Side-by-side
Showing with 205 additions and 177 deletions
trans2es/commons/words_utils.py
View file @ 6c4ef049
......@@ -100,8 +100,11 @@ class QueryWordAttr(object):
@classmethod
def get_hot_search_query_word_weight(cls, name):
try:
print(name)
search_num = StrategyHistoryQueryWords.objects.filter(keyword=name).values_list(
"search_num", flat=True).first()
if not search_num:
search_num = 0
return search_num * 0.01
except:
......@@ -191,51 +194,63 @@ class QueryWordAttr(object):
return 0
@classmethod
def save_query_to_redis(cls, query, from_type=None, get_ret=False):
def save_query_to_redis(cls, query, type_v1=False):
try:
lower_query = str(query).lower()
key = "save_query_to_redis"
redis_data = redis_client.hget(key, lower_query)
if from_type == "search_query":
order_weight = cls.get_hot_search_query_word_weight(name=query)
search_num = cls.get_search_query_results_num(name=query)
else:
order_weight = cls.get_project_query_word_weight(name=query)
search_num = cls.get_query_results_num(name=query)
key = "save_sub_query_to_redis"
if type_v1:
key = "save_sub_query_to_redis_v1"
redis_data = redis_client.get(key)
redis_client.expire(key, 10 * 60)
if redis_data:
json_data = json.loads(redis_data)
have_save_order_weight = json_data.get('order_weight', 0)
have_save_search_num = json_data.get('order_weight', 0)
logging.info(
"query:%s,have_save_order_weight:%s,have_save_search_num:%s,order_weight:%s,search_num:%s" % (
query, have_save_order_weight, have_save_search_num, order_weight, search_num))
if get_ret:
if have_save_order_weight >= order_weight and have_save_search_num == search_num:
return True
else:
return False
redis_data = json.loads(redis_data)
if lower_query in redis_data:
return False
else:
if have_save_order_weight < order_weight or (
have_save_order_weight == order_weight and have_save_search_num < search_num):
logging.info(
"query:%s,have_save_order_weight:%s,have_save_search_num:%s,order_weight:%s,search_num:%s" % (
query, have_save_order_weight, have_save_search_num, order_weight, search_num))
redis_client.hset(key, lower_query,
json.dumps({"order_weight": order_weight, "search_num": search_num}))
else:
pass
elif get_ret == False:
redis_client.hset(key, lower_query,
json.dumps({"order_weight": order_weight, "search_num": search_num}))
redis_data.append(lower_query)
redis_client.set(key, json.dumps(redis_data))
return True
else:
pass
redis_client.set(key, json.dumps([lower_query]))
return True
# if from_type == "search_query":
# order_weight = cls.get_hot_search_query_word_weight(name=query)
# search_num = cls.get_search_query_results_num(name=query)
# else:
# order_weight = cls.get_project_query_word_weight(name=query)
# search_num = cls.get_query_results_num(name=query)
# if redis_data:
# json_data = json.loads(redis_data)
# have_save_order_weight = json_data.get('order_weight', 0)
# have_save_search_num = json_data.get('order_weight', 0)
#
# logging.info(
# "query:%s,have_save_order_weight:%s,have_save_search_num:%s,order_weight:%s,search_num:%s" % (
# query, have_save_order_weight, have_save_search_num, order_weight, search_num))
#
# if get_ret:
# if have_save_order_weight >= order_weight and have_save_search_num == search_num:
# return True
# else:
# return False
# else:
# if have_save_order_weight < order_weight or (
# have_save_order_weight == order_weight and have_save_search_num < search_num):
# logging.info(
# "query:%s,have_save_order_weight:%s,have_save_search_num:%s,order_weight:%s,search_num:%s" % (
# query, have_save_order_weight, have_save_search_num, order_weight, search_num))
#
# redis_client.hset(key, lower_query,
# json.dumps({"order_weight": order_weight, "search_num": search_num}))
# else:
# pass
# elif get_ret == False:
# redis_client.hset(key, lower_query,
# json.dumps({"order_weight": order_weight, "search_num": search_num}))
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
from gm_rpcd.all import bind
......
trans2es/type_info.py
View file @ 6c4ef049
......@@ -385,7 +385,7 @@ def get_type_info_map():
model=strategy_search_word.StrategyHistoryQueryWords,
query_deferred=lambda: strategy_search_word.StrategyHistoryQueryWords.objects.using(
settings.DORIS_DB_NAME).all().query,
get_data_func=SearchWord.get_search_query,
get_data_func=SearchWord.get_search_query_v1,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2
......
trans2es/utils/brandwiki_transfer.py
View file @ 6c4ef049
......@@ -19,22 +19,20 @@ class BrandWikiTransfer(object):
def get_brandwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name))
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -47,22 +45,20 @@ class BrandWikiTransferV1(object):
def get_brandwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
suggest_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), type_v1=True)
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/collectwiki_transfer.py
View file @ 6c4ef049
......@@ -19,22 +19,20 @@ class CollectWikiTransfer(object):
def get_collectwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name))
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -47,22 +45,20 @@ class CollectWikiTransferV1(object):
def get_collectwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), type_v1=True)
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/doctor_transfer.py
View file @ 6c4ef049
......@@ -183,24 +183,19 @@ class DoctorTransferV1(object):
for i in cut_word:
if keyword.find(i) >= 0:
cut_bool = True
item_dict = dict()
suggest_list = list()
if cut_bool == False:
if not cut_bool :
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_doctor_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 2
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.name)
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/itemwiki_transfer.py
View file @ 6c4ef049
......@@ -86,7 +86,7 @@ class ItemWikiTransfer(object):
ret_list = list()
item_dict = dict()
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), from_type="wiki", get_ret=True)
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name))
logging.info("get_wiki_query:%s,ret_bol:%s" % (instance.name, ret_bol))
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
......@@ -175,22 +175,20 @@ class ItemWikiTransferV1(object):
def get_itemwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), type_v1=True)
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/productwiki_transfer.py
View file @ 6c4ef049
......@@ -19,22 +19,20 @@ class ProduceWikiTransfer(object):
def get_productwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name))
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -47,21 +45,20 @@ class ProduceWikiTransferV1(object):
def get_productwiki_suggest_data_list(cls, instance):
try:
ret_list = list()
item_dict = dict()
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
suggest_list = list()
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), type_v1=True)
if ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 3
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list)
except:
......
trans2es/utils/search_query.py
View file @ 6c4ef049
......@@ -16,23 +16,56 @@ class SearchWord(object):
try:
ret_list = list()
item_dict = dict()
suggest_list = list()
sub_query = re.sub('\W+', '', instance.keyword)
keyword = str(sub_query).lower()
ret_bol = QueryWordAttr.save_query_to_redis(query=sub_query)
if ret_bol:
item_dict["id"] = getMd5Digest(keyword)
item_dict["ori_name"] = keyword
if QueryWordAttr.get_search_query_results_num(keyword):
is_online = False if instance.is_delete else True
else:
is_online = False
item_dict["is_online"] = is_online
item_dict["order_weight"] = QueryWordAttr.get_hot_search_query_word_weight(keyword)
item_dict["results_num"] = QueryWordAttr.get_search_query_results_num(keyword)
item_dict["type_flag"] = get_tips_word_type(keyword)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 5
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(keyword)
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([], [])
@classmethod
def get_search_query_v1(self, instance):
item_dict["id"] = getMd5Digest(keyword)
item_dict["ori_name"] = keyword
if QueryWordAttr.get_search_query_results_num(keyword):
is_online = False if instance.is_delete else True
else:
is_online = False
item_dict["is_online"] = is_online
item_dict["order_weight"] = QueryWordAttr.get_hot_search_query_word_weight(keyword)
item_dict["results_num"] = QueryWordAttr.get_search_query_results_num(keyword)
item_dict["type_flag"] = get_tips_word_type(keyword)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 5
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(keyword)
try:
ret_list = list()
item_dict = dict()
suggest_list = list()
sub_query = re.sub('\W+', '', instance.keyword)
keyword = str(sub_query).lower()
ret_bol = QueryWordAttr.save_query_to_redis(query=sub_query,type_v1=True)
if ret_bol:
item_dict["id"] = getMd5Digest(keyword)
item_dict["ori_name"] = keyword
if QueryWordAttr.get_search_query_results_num(keyword):
is_online = False if instance.is_delete else True
else:
is_online = False
item_dict["is_online"] = is_online
item_dict["order_weight"] = QueryWordAttr.get_hot_search_query_word_weight(keyword)
item_dict["results_num"] = QueryWordAttr.get_search_query_results_num(keyword)
item_dict["type_flag"] = get_tips_word_type(keyword)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 5
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(keyword)
return (item_dict, suggest_list)
except:
......
trans2es/utils/tag_transfer.py
View file @ 6c4ef049
......@@ -91,6 +91,7 @@ class TagTransfer(object):
def get_tag_suggest_data_list(cls, instance):
try:
ret_list = list()
suggest_list = list()
keyword = instance.name
cut_bool = False
cut_word = ["下线", "停用", "已经下线", "账号停用"]
......@@ -98,21 +99,18 @@ class TagTransfer(object):
if keyword.find(i) >= 0:
cut_bool = True
item_dict = dict()
if cut_bool == False:
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name))
if not cut_bool and ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 0
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -196,6 +194,7 @@ class TagTransferV1(object):
def get_tag_suggest_data_list(cls, instance):
try:
ret_list = list()
suggest_list = list()
keyword = instance.name
cut_bool = False
cut_word = ["下线", "停用", "已经下线", "账号停用"]
......@@ -203,21 +202,18 @@ class TagTransferV1(object):
if keyword.find(i) >= 0:
cut_bool = True
item_dict = dict()
if cut_bool == False:
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), type_v1=True)
if not cut_bool and ret_bol:
item_dict["id"] = getMd5Digest(str(instance.name))
item_dict["ori_name"] = instance.name
item_dict["is_online"] = instance.is_online
item_dict["order_weight"] = QueryWordAttr.get_project_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name)
item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 0
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/wordresemble.py
View file @ 6c4ef049
......@@ -50,7 +50,8 @@ class WordResemble(object):
for i in cut_word:
if keyword.find(i) >= 0:
cut_bool = True
if cut_bool == False:
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name))
if not cut_bool and ret_bol:
item_dict["id"] = getMd5Digest(str(instance.keyword))
item_dict["ori_name"] = instance.keyword
item_dict["is_online"] = True
......@@ -229,13 +230,15 @@ class WordResembleV1(object):
try:
ret_list = list()
item_dict = dict()
suggest_list = list()
keyword = instance.keyword
cut_bool = False
cut_word = ["下线", "停用", "已经下线", "账号停用"]
for i in cut_word:
if keyword.find(i) >= 0:
cut_bool = True
if cut_bool == False:
ret_bol = QueryWordAttr.save_query_to_redis(query=str(instance.name), type_v1=True)
if not cut_bool and ret_bol:
item_dict["id"] = getMd5Digest(str(instance.keyword))
item_dict["ori_name"] = instance.keyword
item_dict["is_online"] = True
......@@ -246,7 +249,6 @@ class WordResembleV1(object):
item_dict["tips_name_type"] = 4
ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(str(instance.keyword).lower())
return (item_dict, suggest_list)
except:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment