Skip to content

S
search_tips
Commit d688562c authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parent 30985760
Show whitespace changes
Inline Side-by-side
Showing with 87 additions and 84 deletions
search/utils/auto_tips.py
View file @ d688562c
...@@ -17,80 +17,77 @@ from trans2es.commons.words_utils import QueryWordAttr, get_tips_word_type ...@@ -17,80 +17,77 @@ from trans2es.commons.words_utils import QueryWordAttr, get_tips_word_type
def get_suggest_tips(query, lat, lng, offset=0, size=50): def get_suggest_tips(query, lat, lng, offset=0, size=50):
try: try:
###先获取完全命中的query ###先获取完全命中的query
qq = { # qq = {
"query": { # "query": {
"bool": { # "bool": {
"must": [{ # "must": [{
"term": { # "term": {
"ori_name": query # "ori_name": query
} # }
}, { # }, {
"term": { # "term": {
"is_online": True # "is_online": True
} # }
}] # }]
} # }
} # }
} # }
logging.info("get qqqqqqqqq:%s" % qq) # logging.info("get qqqqqqqqq:%s" % qq)
whole_query = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="suggest", query_body=qq, # whole_query = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="suggest", query_body=qq,
offset=0, size=1, is_suggest_request=True) # offset=0, size=1, is_suggest_request=True)
#
logging.info("get whole_query:%s" % whole_query) # logging.info("get whole_query:%s" % whole_query)
have_read_tips_set = set()
ret_list = list()
get_doctor_hospital_data = list() # hits = whole_query["hits"]["hits"]
get_tag_wiki_data = list() #
# for item in hits:
hits = whole_query["hits"]["hits"] # have_read_tips_set.add(item["_source"]["ori_name"])
# id = item["_source"]["id"]
for item in hits: # ori_name = item["_source"]["ori_name"]
have_read_tips_set.add(item["_source"]["ori_name"]) # results_num = item["_source"]["results_num"]
id = item["_source"]["id"] # is_online = item["_source"]["is_online"]
ori_name = item["_source"]["ori_name"] # offline_score = item["_source"]["offline_score"]
results_num = item["_source"]["results_num"] # type_flag = item["_source"]["type_flag"]
is_online = item["_source"]["is_online"] #
offline_score = item["_source"]["offline_score"] # highlight_marks = u'<ems>%s</ems>' % query
type_flag = item["_source"]["type_flag"] # item["_source"]["highlight_name"] = item["_source"]["ori_name"].replace(query, highlight_marks)
# highlight_name = item["_source"]["highlight_name"]
highlight_marks = u'<ems>%s</ems>' % query # if item["_source"]["type_flag"] == "hospital":
item["_source"]["highlight_name"] = item["_source"]["ori_name"].replace(query, highlight_marks) # if lat is not None and lng is not None and lat != 0.0 and lng != 0.0:
highlight_name = item["_source"]["highlight_name"] #
if item["_source"]["type_flag"] == "hospital": # logging.info("get g_hospital_pos_dict:%s" % g_hospital_pos_dict)
if lat is not None and lng is not None and lat != 0.0 and lng != 0.0: # if item["_source"]["ori_name"] in g_hospital_pos_dict:
# distance = point_distance(lng, lat,
logging.info("get g_hospital_pos_dict:%s" % g_hospital_pos_dict) # g_hospital_pos_dict[item["_source"]["ori_name"]][0],
if item["_source"]["ori_name"] in g_hospital_pos_dict: # g_hospital_pos_dict[item["_source"]["ori_name"]][1])
distance = point_distance(lng, lat, # if distance < 1000 * 50:
g_hospital_pos_dict[item["_source"]["ori_name"]][0], # if distance < 1000:
g_hospital_pos_dict[item["_source"]["ori_name"]][1]) # if distance < 100:
if distance < 1000 * 50: # item["_source"]["describe"] = "<100" + "米"
if distance < 1000: # else:
if distance < 100: # item["_source"]["describe"] = "约" + str(int(distance)) + "米"
item["_source"]["describe"] = "<100" + "米" # else:
else: # item["_source"]["describe"] = "约" + str(
item["_source"]["describe"] = "约" + str(int(distance)) + "米" # round(1.0 * distance / 1000, 1)) + "km"
else: # else:
item["_source"]["describe"] = "约" + str( # item["_source"]["describe"] = ">50km"
round(1.0 * distance / 1000, 1)) + "km" # else:
else: # item["_source"]["describe"] = ""
item["_source"]["describe"] = ">50km" #
else: # else:
item["_source"]["describe"] = "" # item["_source"]["describe"] = ""
# else:
else: # if item["_source"]["type_flag"] == "doctor":
item["_source"]["describe"] = "" # item["_source"]["describe"] = ""
else: # else:
if item["_source"]["type_flag"] == "doctor": # item["_source"]["describe"] = "约" + str(item["_source"]["results_num"]) + "个结果" if \
item["_source"]["describe"] = "" # item["_source"]["results_num"] else ""
else: #
item["_source"]["describe"] = "约" + str(item["_source"]["results_num"]) + "个结果" if \ # ret_list.append({"offline_score": offline_score, "ori_name": ori_name, "results_num": results_num, "id": id,
item["_source"]["results_num"] else "" # "highlight_name": highlight_name, "type_flag": type_flag,
# "is_online": is_online})
ret_list.append({"offline_score": offline_score, "ori_name": ori_name, "results_num": results_num, "id": id,
"highlight_name": highlight_name, "type_flag": type_flag,
"is_online": is_online})
###获取联想到的数据 ###获取联想到的数据
# ios输入法在某些情况下会携带\\u2006 # ios输入法在某些情况下会携带\\u2006
...@@ -115,7 +112,8 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50): ...@@ -115,7 +112,8 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50):
"includes": ["id", "ori_name", "offline_score", "is_online", "type_flag", "results_num"] "includes": ["id", "ori_name", "offline_score", "is_online", "type_flag", "results_num"]
} }
} }
logging.info("get qqqqqqqq:%s" % q) get_doctor_hospital_data = list()
get_tag_wiki_data = list()
have_read_tips_set = set() have_read_tips_set = set()
ret_list = list() ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="suggest", query_body=q, result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="suggest", query_body=q,
...@@ -164,8 +162,6 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50): ...@@ -164,8 +162,6 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50):
hit_item["_source"]["results_num"] else "" hit_item["_source"]["results_num"] else ""
get_tag_wiki_data.append(hit_item["_source"]) get_tag_wiki_data.append(hit_item["_source"])
logging.info("get get_doctor_hospital_data:%s" % get_doctor_hospital_data)
logging.info("get get_tag_wiki_data:%s" % get_tag_wiki_data)
ret_list.extend(get_tag_wiki_data) ret_list.extend(get_tag_wiki_data)
ret_list.extend(get_doctor_hospital_data) ret_list.extend(get_doctor_hospital_data)
...@@ -181,6 +177,7 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50): ...@@ -181,6 +177,7 @@ def get_suggest_tips(query, lat, lng, offset=0, size=50):
ori = i.split(":")[0] ori = i.split(":")[0]
ori_name = base64.b64decode(ori.encode('utf8')).decode('utf8') ori_name = base64.b64decode(ori.encode('utf8')).decode('utf8')
if ori_name not in have_read_tips_set: if ori_name not in have_read_tips_set:
have_read_tips_set.add(ori_name)
result_num = i.split(":")[1] result_num = i.split(":")[1]
ret_list.append( ret_list.append(
{"results_num": result_num, "ori_name": ori_name, "id": None, "is_online": True, {"results_num": result_num, "ori_name": ori_name, "id": None, "is_online": True,
......
trans2es/type_info.py
View file @ d688562c
...@@ -210,18 +210,19 @@ class TypeInfo(object): ...@@ -210,18 +210,19 @@ class TypeInfo(object):
es=es, es=es,
) )
def insert_table_by_pk_list(self, sub_index_name, pk_list, es=None, use_batch_query_set=False,es_type=None): def insert_table_by_pk_list(self, sub_index_name, pk_list, es=None, use_batch_query_set=False, es_type=None):
if use_batch_query_set: if use_batch_query_set:
qs = self.queryset qs = self.queryset
else: else:
qs = self.model.objects.all() qs = self.model.objects.all()
if es_type == "doctor_tips": if es_type == "doctor_tips":
instance_list = qs.filter(pk__in=pk_list,doctor_type=DOCTOR_TYPE.DOCTOR) instance_list = qs.filter(pk__in=pk_list, doctor_type=DOCTOR_TYPE.DOCTOR)
elif es_type == "wordrel_tips": elif es_type == "wordrel_tips":
instance_list = qs.filter(pk__in=pk_list, category__in=[13, 12, 11, 9, 1]) instance_list = qs.filter(pk__in=pk_list, category__in=[13, 12, 11, 9, 1])
elif es_type == "tag_tips": elif es_type == "tag_tips":
instance_list = qs.filter(pk__in=pk_list, tag_type__in=[TAG_TYPE.BODY_PART, TAG_TYPE.BODY_PART_SUB_ITEM, TAG_TYPE.ITEM_WIKI]) instance_list = qs.filter(pk__in=pk_list, tag_type__in=[TAG_TYPE.BODY_PART, TAG_TYPE.BODY_PART_SUB_ITEM,
TAG_TYPE.ITEM_WIKI])
else: else:
instance_list = qs.filter(pk__in=pk_list) instance_list = qs.filter(pk__in=pk_list)
...@@ -540,7 +541,7 @@ def tips_attr_sync_to_redis_type_info_map(): ...@@ -540,7 +541,7 @@ def tips_attr_sync_to_redis_type_info_map():
type='wordrel_results_num', # api_wordrelresemble type='wordrel_results_num', # api_wordrelresemble
model=doctor.Hospital, model=doctor.Hospital,
query_deferred=lambda: wordresemble.WordRel.objects.filter( query_deferred=lambda: wordresemble.WordRel.objects.filter(
category__in=[0, 1, 2, 9, 10, 11, 12, 13, 14, 15, 16, 17]).query, category__in=[13, 12, 11, 9, 1]).query,
get_data_func=WordResemble.get_all_data_name_mapping_results_to_redis, get_data_func=WordResemble.get_all_data_name_mapping_results_to_redis,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
...@@ -551,7 +552,7 @@ def tips_attr_sync_to_redis_type_info_map(): ...@@ -551,7 +552,7 @@ def tips_attr_sync_to_redis_type_info_map():
type='wordresemble_results_num', # api_wordrelresemble type='wordresemble_results_num', # api_wordrelresemble
model=doctor.Hospital, model=doctor.Hospital,
query_deferred=lambda: wordresemble.WordRel.objects.filter( query_deferred=lambda: wordresemble.WordRel.objects.filter(
category__in=[0, 1, 2, 9, 10, 11, 12, 13, 14, 15, 16, 17]).query, category__in=[13, 12, 11, 9, 1]).query,
get_data_func=WordResemble.set_data_to_redis, get_data_func=WordResemble.set_data_to_redis,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
......
trans2es/utils/wordresemble.py
View file @ d688562c
...@@ -156,9 +156,8 @@ class WordResemble(object): ...@@ -156,9 +156,8 @@ class WordResemble(object):
QUERY_KEY = "query:{}:search_tip" QUERY_KEY = "query:{}:search_tip"
tips_num_redis_key_prefix = "search_tips:tips_mapping_num" tips_num_redis_key_prefix = "search_tips:tips_mapping_num"
# result = wordresemble.WordRel.objects.all().values("keyword", "id")
# for wordrm in result:
query_sql_item = wordresemble.WordRel.objects.filter(keyword=instance.keyword) query_sql_item = wordresemble.WordRel.objects.filter(keyword=instance.keyword)
for sql_obj in query_sql_item: for sql_obj in query_sql_item:
words = list(sql_obj.all_resembles.all().values_list('word', flat=True)) words = list(sql_obj.all_resembles.all().values_list('word', flat=True))
for items in words: for items in words:
...@@ -169,6 +168,7 @@ class WordResemble(object): ...@@ -169,6 +168,7 @@ class WordResemble(object):
# 获取本体词的结果 # 获取本体词的结果
redis_data = redis_client.hget(tips_num_redis_key_prefix, items) redis_data = redis_client.hget(tips_num_redis_key_prefix, items)
logging.info("get redis_data:%s" % redis_data)
redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {} redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {}
total_count = 0 total_count = 0
if 't' in redis_val_dict: if 't' in redis_val_dict:
...@@ -177,6 +177,8 @@ class WordResemble(object): ...@@ -177,6 +177,8 @@ class WordResemble(object):
# 存储到Redis # 存储到Redis
value = value_name + str(":" + str(total_count)) value = value_name + str(":" + str(total_count))
redis_client.sadd(key, value) redis_client.sadd(key, value)
logging.info("get value:%s" % value)
# 存储近义词 # 存储近义词
# query_base64 = base64.b64encode(items.encode('utf8')).decode('utf8') # query_base64 = base64.b64encode(items.encode('utf8')).decode('utf8')
...@@ -191,16 +193,19 @@ class WordResemble(object): ...@@ -191,16 +193,19 @@ class WordResemble(object):
total_count = int(redis_val_dict['t']) total_count = int(redis_val_dict['t'])
# 存储结果 # 存储结果
value = value_names + str(":" + str(total_count)) value = value_names + str(":" + str(total_count))
redis_client.sadd(key, value) redis_client.sadd(key, value)
if words: if words:
for w in words: for w in words:
value_name_w = base64.b64encode(w.encode('utf8')).decode('utf8') value_name_w = base64.b64encode(w.encode('utf8')).decode('utf8')
redis_data = redis_client.hget(tips_num_redis_key_prefix, w) redis_data = redis_client.hget(tips_num_redis_key_prefix, w)
redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {} redis_val_dict = json.loads(str(redis_data, encoding="utf-8")) if redis_data else {}
total_count = 0 total_count = 0
if 't' in redis_val_dict: if 't' in redis_val_dict:
total_count = int(redis_val_dict['t']) total_count = int(redis_val_dict['t'])
value = value_name_w + str(":" + str(total_count)) value = value_name_w + str(":" + str(total_count))
redis_client.sadd(key, value) redis_client.sadd(key, value)
except: except:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment