Skip to content

S
search_tips
Commit 9822f668 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parents 53e88b81 2d063776
Hide whitespace changes
Inline Side-by-side
Showing with 151 additions and 115 deletions
search/utils/auto_tips.py
View file @ 9822f668
...@@ -5,14 +5,14 @@ from __future__ import unicode_literals, absolute_import, print_function ...@@ -5,14 +5,14 @@ from __future__ import unicode_literals, absolute_import, print_function
import logging import logging
import traceback import traceback
import json import json
import base64 import base64, hashlib
import hashlib
from libs.es import ESPerform from libs.es import ESPerform
from libs.cache import redis_client from libs.cache import redis_client
from libs.tools import g_hospital_pos_dict from libs.tools import g_hospital_pos_dict
from libs.tools import point_distance from libs.tools import point_distance
from gm_rpcd.all import bind from gm_rpcd.all import bind
from trans2es.commons.words_utils import QueryWordAttr, get_tips_word_type from trans2es.commons.words_utils import QueryWordAttr, get_tips_word_type
from pypinyin import pinyin, lazy_pinyin
def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
...@@ -21,8 +21,6 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -21,8 +21,6 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
50%前段强加权后根据tab转化率和词频进行排序 索引 suggest gray_number=1 尾号["0", "1", "2", "3", "c", "d", "e", "f"] 50%前段强加权后根据tab转化率和词频进行排序 索引 suggest gray_number=1 尾号["0", "1", "2", "3", "c", "d", "e", "f"]
25%去掉前段强加权后 直接根据tag转化率和词频进行排序 suggest-v1 gray_number=2 尾号["4", "5", "6", "a"] 25%去掉前段强加权后 直接根据tag转化率和词频进行排序 suggest-v1 gray_number=2 尾号["4", "5", "6", "a"]
25%去掉前段强加权后 根据tag转化率和词频进行排序 再根据前段加权展示排序 suggest-v1 gray_number=3 尾号["9", "8", "7", "b"] 25%去掉前段强加权后 根据tag转化率和词频进行排序 再根据前段加权展示排序 suggest-v1 gray_number=3 尾号["9", "8", "7", "b"]
:param query: :param query:
:param lat: :param lat:
:param lng: :param lng:
...@@ -31,8 +29,8 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -31,8 +29,8 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
:param device_id: :param device_id:
:return: :return:
""" """
try:
try:
###在原来的逻辑上加两层灰度 ###在原来的逻辑上加两层灰度
gray_number = recommed_service_category_device_id(device_id) gray_number = recommed_service_category_device_id(device_id)
if gray_number == 1: if gray_number == 1:
...@@ -40,93 +38,11 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -40,93 +38,11 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
else: else:
sub_index_name = "suggest-v1" sub_index_name = "suggest-v1"
###获取联想到的数据 have_read_tips_set, ret_list, result_dict = get_query_by_es(query=str(query).lower(), lat=lat, lng=lng,
# ios输入法在某些情况下会携带\\u2006 offset=offset,
query = query.replace("\u2006", '') size=size, highlight_query=query,
q = { have_read_tips_set=set(),
"suggest": { sub_index_name=sub_index_name)
"tips-suggest": {
"prefix": query,
"completion": {
"field": "suggest",
"size": size,
"contexts": {
"is_online": [True]
},
"fuzzy": {
"fuzziness": 0
}
}
}
},
"_source": {
"includes": ["id", "ori_name", "offline_score", "is_online", "type_flag", "results_num"]
}
}
get_doctor_hospital_data = list()
get_tag_wiki_data = list()
have_read_tips_set = set()
ret_list = list()
doctor_hospital_equal_query = list()
tag_equal_query = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=sub_index_name, query_body=q,
offset=offset, size=size, is_suggest_request=True)
for tips_item in result_dict["suggest"]["tips-suggest"]:
for hit_item in tips_item["options"]:
if hit_item["_source"]["ori_name"] not in have_read_tips_set:
have_read_tips_set.add(hit_item["_source"]["ori_name"])
highlight_marks = u'<ems>%s</ems>' % query
hit_item["_source"]["highlight_name"] = set_highlihgt(query, hit_item["_source"]["ori_name"])
if hit_item["_source"]["type_flag"] == "hospital":
if lat is not None and lng is not None and lat != 0.0 and lng != 0.0:
if hit_item["_source"]["ori_name"] in g_hospital_pos_dict:
distance = point_distance(lng, lat,
g_hospital_pos_dict[hit_item["_source"]["ori_name"]][0],
g_hospital_pos_dict[hit_item["_source"]["ori_name"]][1])
if distance < 1000 * 50:
if distance < 1000:
if distance < 100:
hit_item["_source"]["describe"] = "<100" + "米"
else:
hit_item["_source"]["describe"] = "约" + str(int(distance)) + "米"
else:
hit_item["_source"]["describe"] = "约" + str(
round(1.0 * distance / 1000, 1)) + "km"
else:
hit_item["_source"]["describe"] = ">50km"
else:
hit_item["_source"]["describe"] = ""
else:
hit_item["_source"]["describe"] = ""
if hit_item["_source"]["ori_name"] == query:
doctor_hospital_equal_query.append(hit_item["_source"])
else:
get_doctor_hospital_data.append(hit_item["_source"])
else:
if hit_item["_source"]["type_flag"] == "doctor":
hit_item["_source"]["describe"] = ""
if hit_item["_source"]["ori_name"] == query:
doctor_hospital_equal_query.append(hit_item["_source"])
else:
get_doctor_hospital_data.append(hit_item["_source"])
else:
hit_item["_source"]["describe"] = "约" + str(hit_item["_source"]["results_num"]) + "个结果" if \
hit_item["_source"]["results_num"] else ""
if hit_item["_source"]["ori_name"] == query:
tag_equal_query.append(hit_item["_source"])
else:
get_tag_wiki_data.append(hit_item["_source"])
ret_list.extend(tag_equal_query)
ret_list.extend(doctor_hospital_equal_query)
ret_list.extend(get_tag_wiki_data)
ret_list.extend(get_doctor_hospital_data)
if gray_number == 3: if gray_number == 3:
# 在去掉强加权的逻辑上根据词频和tag转化率排序后再前段强加权 # 在去掉强加权的逻辑上根据词频和tag转化率排序后再前段强加权
...@@ -148,7 +64,8 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -148,7 +64,8 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
ret_list = equal_data ret_list = equal_data
if len(ret_list) >= 50: if len(ret_list) >= 50:
return ret_list[:50] return ret_list
else: else:
query_ret_list = [] query_ret_list = []
wordresemble_ret_list = [] wordresemble_ret_list = []
...@@ -168,6 +85,7 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -168,6 +85,7 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
describe = "约" + str(result_num) + "个结果" if result_num else "" describe = "约" + str(result_num) + "个结果" if result_num else ""
logging.info("get result_num:%s" % result_num) logging.info("get result_num:%s" % result_num)
highlight_marks = u'<ems>%s</ems>' % query highlight_marks = u'<ems>%s</ems>' % query
# highlight_name = ori_name.replace(query, highlight_marks)
highlight_name = set_highlihgt(query, ori_name) highlight_name = set_highlihgt(query, ori_name)
if ori_name == query: if ori_name == query:
query_ret_list.append( query_ret_list.append(
...@@ -187,6 +105,7 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -187,6 +105,7 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
ret_list.extend(wordresemble_ret_list) ret_list.extend(wordresemble_ret_list)
###繁体字删掉，把搜索结果加到简体字上边 ###繁体字删掉，把搜索结果加到简体字上边
fanti_query = [{'痩脸针': "瘦脸针"}] fanti_query = [{'痩脸针': "瘦脸针"}]
for item in ret_list: for item in ret_list:
result_num = [[item['results_num'], list(ret.values())[0], list(ret.keys())[0]] for ret in fanti_query result_num = [[item['results_num'], list(ret.values())[0], list(ret.keys())[0]] for ret in fanti_query
if list(ret.keys())[0] == item['ori_name']] if list(ret.keys())[0] == item['ori_name']]
...@@ -197,10 +116,32 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None): ...@@ -197,10 +116,32 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50, device_id=None):
if item['ori_name'] == result_num[0][1]: if item['ori_name'] == result_num[0][1]:
item['results_num'] += result_num[0][0] item['results_num'] += result_num[0][0]
####
if len(ret_list) >= 50: if len(ret_list) >= 50:
return ret_list[0:50] return ret_list[0:50]
elif len(ret_list) < 50 and len(ret_list) > 3:
return ret_list
else: else:
##无结果的时候把汉字转成拼音再搜一次
ss = lazy_pinyin(query)
str_query = ''
for item in ss:
str_query += str(item)
have_read_tips_set, pinyin_ret_list, result_dict = get_query_by_es(query=str_query, lat=lat, lng=lng,
offset=offset,
size=size - len(ret_list),
highlight_query=query,
have_read_tips_set=have_read_tips_set,
sub_index_name=sub_index_name)
ret_list.extend(pinyin_ret_list)
logging.info("get ret_list:%s" % ret_list)
return ret_list return ret_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list() return list()
...@@ -212,23 +153,116 @@ def set_highlihgt(query=None, ori_name=None): ...@@ -212,23 +153,116 @@ def set_highlihgt(query=None, ori_name=None):
query2 = ori_name query2 = ori_name
for item in range(0, len(query)): for item in range(0, len(query)):
all_word.add(query[item]) all_word.add(query[item])
for item in all_word: for item in all_word:
is_find = query2.find(item) is_find = query2.find(item)
high_query = None
if is_find >= 0: if is_find >= 0:
highlight_marks = u'<ems>%s</ems>' % item highlight_marks = u'<>%s</>' % item
high_query = query2.replace(item, highlight_marks) high_query = query2.replace(item, highlight_marks)
query2 = high_query query2 = high_query
if high_query == None:
highlight_name = query2
else: highlight_name = query2.replace('<>', '<ems>').replace('</>', '</ems>')
highlight_name = high_query
return highlight_name return highlight_name
def get_query_by_es(query='', lat=0, lng=0, size=0, offset=0, highlight_query=None,
have_read_tips_set=None, sub_index_name="suggest"):
try:
query = query.replace("\u2006", '')
q = {
"suggest": {
"tips-suggest": {
"prefix": query,
"completion": {
"field": "suggest",
"size": size,
"contexts": {
"is_online": [True]
},
"fuzzy": {
"fuzziness": 0
}
}
}
},
"_source": {
"includes": ["id", "ori_name", "offline_score", "is_online", "type_flag", "results_num"]
}
}
get_doctor_hospital_data = list()
get_tag_wiki_data = list()
ret_list = list()
doctor_hospital_equal_query = list()
tag_equal_query = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=sub_index_name, query_body=q,
offset=offset, size=size, is_suggest_request=True)
for tips_item in result_dict["suggest"]["tips-suggest"]:
for hit_item in tips_item["options"]:
logging.info("get hit_item:%s" % hit_item)
if hit_item["_source"]["ori_name"] not in have_read_tips_set:
have_read_tips_set.add(hit_item["_source"]["ori_name"])
highlight_marks = u'<ems>%s</ems>' % query
# hit_item["_source"]["highlight_name"] = hit_item["_source"]["ori_name"].replace(query,
# highlight_marks)
hit_item["_source"]["highlight_name"] = set_highlihgt(highlight_query,
hit_item["_source"]["ori_name"])
if hit_item["_source"]["type_flag"] == "hospital":
if lat is not None and lng is not None and lat != 0.0 and lng != 0.0:
if hit_item["_source"]["ori_name"] in g_hospital_pos_dict:
distance = point_distance(lng, lat,
g_hospital_pos_dict[hit_item["_source"]["ori_name"]][0],
g_hospital_pos_dict[hit_item["_source"]["ori_name"]][1])
if distance < 1000 * 50:
if distance < 1000:
if distance < 100:
hit_item["_source"]["describe"] = "<100" + "米"
else:
hit_item["_source"]["describe"] = "约" + str(int(distance)) + "米"
else:
hit_item["_source"]["describe"] = "约" + str(
round(1.0 * distance / 1000, 1)) + "km"
else:
hit_item["_source"]["describe"] = ">50km"
else:
hit_item["_source"]["describe"] = ""
else:
hit_item["_source"]["describe"] = ""
if hit_item["_source"]["ori_name"] == query:
doctor_hospital_equal_query.append(hit_item["_source"])
else:
get_doctor_hospital_data.append(hit_item["_source"])
else:
if hit_item["_source"]["type_flag"] == "doctor":
hit_item["_source"]["describe"] = ""
if hit_item["_source"]["ori_name"] == query:
doctor_hospital_equal_query.append(hit_item["_source"])
else:
get_doctor_hospital_data.append(hit_item["_source"])
else:
hit_item["_source"]["describe"] = "约" + str(hit_item["_source"]["results_num"]) + "个结果" if \
hit_item["_source"]["results_num"] else ""
if hit_item["_source"]["ori_name"] == query:
tag_equal_query.append(hit_item["_source"])
else:
get_tag_wiki_data.append(hit_item["_source"])
ret_list.extend(tag_equal_query)
ret_list.extend(doctor_hospital_equal_query)
ret_list.extend(get_tag_wiki_data)
ret_list.extend(get_doctor_hospital_data)
return have_read_tips_set, ret_list, result_dict
except:
return set(), list(), list()
def recommed_service_category_device_id(device_id): def recommed_service_category_device_id(device_id):
try: try:
''' '''
......
trans2es/commons/commons.py
View file @ 9822f668
...@@ -20,6 +20,7 @@ def uuid4(): ...@@ -20,6 +20,7 @@ def uuid4():
def get_tips_suggest_list(instance_cn_name): def get_tips_suggest_list(instance_cn_name):
try: try:
print(instance_cn_name)
# ch_full_weight = 6.0 * 1000 # ch_full_weight = 6.0 * 1000
# py_full_weight = 3.0 * 1000 # py_full_weight = 3.0 * 1000
full_weight = 3.0 * 1000 full_weight = 3.0 * 1000
......
trans2es/utils/brandwiki_transfer.py
View file @ 9822f668
...@@ -33,7 +33,7 @@ class BrandWikiTransfer(object): ...@@ -33,7 +33,7 @@ class BrandWikiTransfer(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(instance.name) suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
...@@ -61,7 +61,7 @@ class BrandWikiTransferV1(object): ...@@ -61,7 +61,7 @@ class BrandWikiTransferV1(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.name) suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
......
trans2es/utils/collectwiki_transfer.py
View file @ 9822f668
...@@ -33,7 +33,7 @@ class CollectWikiTransfer(object): ...@@ -33,7 +33,7 @@ class CollectWikiTransfer(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(instance.name) suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
...@@ -61,7 +61,7 @@ class CollectWikiTransferV1(object): ...@@ -61,7 +61,7 @@ class CollectWikiTransferV1(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.name) suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
......
trans2es/utils/doctor_transfer.py
View file @ 9822f668
...@@ -97,7 +97,7 @@ class DoctorTransfer(object): ...@@ -97,7 +97,7 @@ class DoctorTransfer(object):
item_dict["order_weight"] = QueryWordAttr.get_doctor_query_word_weight(instance.name) item_dict["order_weight"] = QueryWordAttr.get_doctor_query_word_weight(instance.name)
item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name) item_dict["results_num"] = QueryWordAttr.get_query_results_num(instance.name)
item_dict["type_flag"] = get_tips_word_type(instance.name) item_dict["type_flag"] = get_tips_word_type(str(instance.name).lower())
item_dict["offline_score"] = 0.0 item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 2 item_dict["tips_name_type"] = 2
......
trans2es/utils/hospital_transfer.py
View file @ 9822f668
...@@ -43,7 +43,7 @@ class HospitalTransfer(object): ...@@ -43,7 +43,7 @@ class HospitalTransfer(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(name) suggest_list = get_tips_suggest_list(str(name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
...@@ -94,7 +94,7 @@ class HospitalTransferV1(object): ...@@ -94,7 +94,7 @@ class HospitalTransferV1(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(name) suggest_list = get_tips_suggest_list_v1(str(name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
......
trans2es/utils/itemwiki_transfer.py
View file @ 9822f668
...@@ -97,8 +97,9 @@ class ItemWikiTransfer(object): ...@@ -97,8 +97,9 @@ class ItemWikiTransfer(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(instance.name) suggest_list = get_tips_suggest_list(str(instance.name).lower())
logging.info("get suggest_list:%s" % suggest_list)
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -188,7 +189,7 @@ class ItemWikiTransferV1(object): ...@@ -188,7 +189,7 @@ class ItemWikiTransferV1(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.name) suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
......
trans2es/utils/productwiki_transfer.py
View file @ 9822f668
...@@ -33,7 +33,7 @@ class ProduceWikiTransfer(object): ...@@ -33,7 +33,7 @@ class ProduceWikiTransfer(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(instance.name) suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
...@@ -61,7 +61,7 @@ class ProduceWikiTransferV1(object): ...@@ -61,7 +61,7 @@ class ProduceWikiTransferV1(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.name) suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
......
trans2es/utils/tag_transfer.py
View file @ 9822f668
...@@ -111,7 +111,7 @@ class TagTransfer(object): ...@@ -111,7 +111,7 @@ class TagTransfer(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(instance.name) suggest_list = get_tips_suggest_list(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
...@@ -216,7 +216,7 @@ class TagTransferV1(object): ...@@ -216,7 +216,7 @@ class TagTransferV1(object):
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.name) suggest_list = get_tips_suggest_list_v1(str(instance.name).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
except: except:
......
trans2es/utils/wordresemble.py
View file @ 9822f668
...@@ -60,7 +60,7 @@ class WordResemble(object): ...@@ -60,7 +60,7 @@ class WordResemble(object):
item_dict["offline_score"] = 0.0 item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 4 item_dict["tips_name_type"] = 4
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list(instance.keyword) suggest_list = get_tips_suggest_list(str(instance.keyword).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
...@@ -245,7 +245,7 @@ class WordResembleV1(object): ...@@ -245,7 +245,7 @@ class WordResembleV1(object):
item_dict["offline_score"] = 0.0 item_dict["offline_score"] = 0.0
item_dict["tips_name_type"] = 4 item_dict["tips_name_type"] = 4
ret_list.append(item_dict) ret_list.append(item_dict)
suggest_list = get_tips_suggest_list_v1(instance.keyword) suggest_list = get_tips_suggest_list_v1(str(instance.keyword).lower())
return (item_dict, suggest_list) return (item_dict, suggest_list)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment