Skip to content

P
physical
Commit 191864a8 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'highlight' into 'master' 

Highlight

See merge request !419
parents c170184a 7f2be2f5
Hide whitespace changes
Inline Side-by-side
Showing with 220 additions and 154 deletions
libs/es.py
View file @ 191864a8
...@@ -155,9 +155,9 @@ class ESPerform(object): ...@@ -155,9 +155,9 @@ class ESPerform(object):
bulk_actions = [] bulk_actions = []
if sub_index_name=="topic" or \ if sub_index_name == "topic" or \
sub_index_name=="topic-star-routing" or \ sub_index_name == "topic-star-routing" or \
sub_index_name=="topic-high-star": sub_index_name == "topic-high-star":
for data in data_list: for data in data_list:
if data: if data:
bulk_actions.append({ bulk_actions.append({
...@@ -232,7 +232,7 @@ class ESPerform(object): ...@@ -232,7 +232,7 @@ class ESPerform(object):
return {"total_count": 0, "hits": []} return {"total_count": 0, "hits": []}
@classmethod @classmethod
def get_analyze_results(cls,es_cli, sub_index_name, query_body): def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
...@@ -242,7 +242,7 @@ class ESPerform(object): ...@@ -242,7 +242,7 @@ class ESPerform(object):
logging.error("index:%s is not existing,get_search_results error!" % official_index_name) logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None return None
res = es_cli.indices.analyze(index=official_index_name,body=query_body) res = es_cli.indices.analyze(index=official_index_name, body=query_body)
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -276,7 +276,7 @@ class ESPerform(object): ...@@ -276,7 +276,7 @@ class ESPerform(object):
if high_num > 3: if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % ( logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list))) str(es_nodes_list), str(es_nodes_info_list)))
return True return True
else: else:
return False return False
...@@ -298,8 +298,8 @@ class ESPerform(object): ...@@ -298,8 +298,8 @@ class ESPerform(object):
functions_list += [ functions_list += [
{ {
"filter": { "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"term": {"content_level": 6}} "term": {"content_level": 6}}
} }
}, },
...@@ -307,8 +307,8 @@ class ESPerform(object): ...@@ -307,8 +307,8 @@ class ESPerform(object):
}, },
{ {
"filter": { "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"term": {"content_level": 5}} "term": {"content_level": 5}}
} }
}, },
...@@ -316,8 +316,8 @@ class ESPerform(object): ...@@ -316,8 +316,8 @@ class ESPerform(object):
}, },
{ {
"filter": { "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"term": {"content_level": 4}} "term": {"content_level": 4}}
} }
}, },
...@@ -411,7 +411,7 @@ class ESPerform(object): ...@@ -411,7 +411,7 @@ class ESPerform(object):
} }
}, },
"_source": { "_source": {
"include": ["id","user_id"] "include": ["id", "user_id"]
}, },
"sort": [ "sort": [
{"latest_reply_time": {"order": "desc"}}, {"latest_reply_time": {"order": "desc"}},
...@@ -420,7 +420,7 @@ class ESPerform(object): ...@@ -420,7 +420,7 @@ class ESPerform(object):
], ],
"collapse": { "collapse": {
"field": "user_id" "field": "user_id"
} }
} }
if len(have_read_topic_id_list) > 0: if len(have_read_topic_id_list) > 0:
...@@ -429,7 +429,8 @@ class ESPerform(object): ...@@ -429,7 +429,8 @@ class ESPerform(object):
"id": have_read_topic_id_list "id": have_read_topic_id_list
} }
} }
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q, result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6") offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]] topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
...@@ -441,7 +442,17 @@ class ESPerform(object): ...@@ -441,7 +442,17 @@ class ESPerform(object):
logging.info("topic_id_list:%s" % str(topic_id_dict)) logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list,topic_id_dict return topic_id_list, topic_id_dict
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list() return list()
@classmethod
def get_highlight(cls, fields=[]):
field_highlight = {
'fields': {k: {} for k in fields},
'pre_tags': ['<%s>' % 'ems'],
'post_tags': ['</%s>' % 'ems']
}
return field_highlight
search/utils/topic.py
View file @ 191864a8
...@@ -159,6 +159,9 @@ class TopicUtils(object): ...@@ -159,6 +159,9 @@ class TopicUtils(object):
user_tag_list = result_dict["hits"][0]["_source"]["tag_list"] user_tag_list = result_dict["hits"][0]["_source"]["tag_list"]
q = dict() q = dict()
topic_id_list = list()
q["query"] = dict() q["query"] = dict()
functions_list = [ functions_list = [
...@@ -233,6 +236,7 @@ class TopicUtils(object): ...@@ -233,6 +236,7 @@ class TopicUtils(object):
'operator': 'or', 'operator': 'or',
'fields': ["content", "tag_name_list"], 'fields': ["content", "tag_name_list"],
} }
query_function_score["boost_mode"] = "replace"
query_function_score["query"]["bool"]["should"] = [ query_function_score["query"]["bool"]["should"] = [
{'multi_match': multi_match}, {'multi_match': multi_match},
...@@ -261,7 +265,7 @@ class TopicUtils(object): ...@@ -261,7 +265,7 @@ class TopicUtils(object):
{"term": {"content_level": 6}} {"term": {"content_level": 6}}
) )
q["_source"] = { q["_source"] = {
"includes": ["id"] "includes": ["id", "highlight", "description"]
} }
if query is None: if query is None:
...@@ -319,6 +323,13 @@ class TopicUtils(object): ...@@ -319,6 +323,13 @@ class TopicUtils(object):
} }
} }
] ]
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size, routing=routing)
for item in result_dict["hits"]:
topic_id_list.append(item["_source"]["id"])
else: else:
multi_match = { multi_match = {
'query': query, 'query': query,
...@@ -329,9 +340,9 @@ class TopicUtils(object): ...@@ -329,9 +340,9 @@ class TopicUtils(object):
functions_list += [ functions_list += [
{ {
"weight": 400, "weight": 400,
"filter":{ "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"term": {"user_nick_name_pre": query.lower()} "term": {"user_nick_name_pre": query.lower()}
} }
} }
...@@ -340,15 +351,15 @@ class TopicUtils(object): ...@@ -340,15 +351,15 @@ class TopicUtils(object):
{ {
"weight": 400, "weight": 400,
"filter": { "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"bool": { "bool": {
"must": { "must": {
"term": {"content_level": 6}, "term": {"content_level": 6},
}, },
"minimum_should_match": 1, "minimum_should_match": 1,
"should": [ "should": [
{'match_phrase': {"content":query}}, {'match_phrase': {"content": query}},
{'match_phrase': {"tag_name_list": query}}, {'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match}, # {'multi_match': multi_match},
{"term": {"tag_list": tag_id}}, {"term": {"tag_list": tag_id}},
...@@ -362,8 +373,8 @@ class TopicUtils(object): ...@@ -362,8 +373,8 @@ class TopicUtils(object):
{ {
"weight": 400, "weight": 400,
"filter": { "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"bool": { "bool": {
"must": { "must": {
"term": {"content_level": 5}, "term": {"content_level": 5},
...@@ -384,8 +395,8 @@ class TopicUtils(object): ...@@ -384,8 +395,8 @@ class TopicUtils(object):
{ {
"weight": 400, "weight": 400,
"filter": { "filter": {
"constant_score":{ "constant_score": {
"filter":{ "filter": {
"bool": { "bool": {
"must": { "must": {
"term": {"content_level": 4}, "term": {"content_level": 4},
...@@ -425,7 +436,7 @@ class TopicUtils(object): ...@@ -425,7 +436,7 @@ class TopicUtils(object):
} }
}, },
{ {
"latest_reply_time":{ "latest_reply_time": {
"order": "desc" "order": "desc"
} }
}, },
...@@ -435,14 +446,14 @@ class TopicUtils(object): ...@@ -435,14 +446,14 @@ class TopicUtils(object):
} }
} }
] ]
q["highlight"] = ESPerform.get_highlight(["content"])
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q, result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size, routing=routing) offset=offset, size=size, routing=routing)
topic_id_list = list() for item in result_dict["hits"]:
topic_id_list.append({"id": item["_source"]["id"], "highlight": item.get("highlight", {})})
for item in result_dict["hits"]:
topic_id_list.append(item["_source"]["id"])
return topic_id_list return topic_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
search/views/topic.py
View file @ 191864a8
...@@ -14,6 +14,7 @@ from search.utils.common import * ...@@ -14,6 +14,7 @@ from search.utils.common import *
from libs.es import ESPerform from libs.es import ESPerform
from django.conf import settings from django.conf import settings
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE): def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try: try:
if user_id == -1: if user_id == -1:
...@@ -28,10 +29,11 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -28,10 +29,11 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size, recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,
query_type=query_type, single_size=size,
filter_topic_id_list=have_read_topic_id_list,index_type="topic",routing="4,5,6") query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids) have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = { redis_dict = {
...@@ -46,27 +48,31 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -46,27 +48,31 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None, def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1,useful_tag_list =[]): query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1, useful_tag_list=[]):
try: try:
topic_star_routing = "6" topic_star_routing = "6"
index_type = "topic-high-star" index_type = "topic-high-star"
device_redis_key = "" device_redis_key = ""
if query is None: if query is None:
if user_id>0: if user_id > 0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type) device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(
query_type)
else: else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else: else:
topic_star_routing = "3,4,5,6" topic_star_routing = "3,4,5,6"
index_type = "topic" index_type = "topic"
if user_id>0: if user_id > 0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type) redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type) query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
else: else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type) redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
...@@ -78,18 +84,17 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -78,18 +84,17 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if query is None: if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else: else:
if offset>0: # 首次搜索时不需要过滤已读 if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
elif len(device_redis_key)>0: elif len(device_redis_key) > 0:
redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list)
if redis_field_val_list[0]: if redis_field_val_list[0]:
if query is None: if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else: else:
if offset>0: # 首次搜索时不需要过滤已读 if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list) have_read_topic_id_list.extend(promote_topic_list)
...@@ -98,66 +103,72 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -98,66 +103,72 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
attention_tag_list = list() attention_tag_list = list()
recommend_topic_list = list() recommend_topic_list = list()
if query is None: if query is None:
if user_id != -1: if user_id != -1:
# 有用标签召回 # 有用标签召回
if usefulrecall != -1: if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4, useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6",useful_tag_list=useful_tag_list) "topic-high-star", "6",
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id) useful_tag_list=useful_tag_list)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key) # user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# user_similar_score_redis_list = json.loads( # redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else [] # user_similar_score_redis_list = json.loads(
size = size - len(useful_topic_id_list) # redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
have_read_topic_id_list.extend(useful_topic_id_list) size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id) # linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list() recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
recommend_topic_list = list() linucb_recommend_topic_id_list = list()
if b"data" in recommend_topic_dict: recommend_topic_list = list()
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"]) if b"data" in recommend_topic_dict:
if linucb_recommend_topic_id_list == None: linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
linucb_recommend_topic_id_list = list() if linucb_recommend_topic_id_list == None:
# 推荐帖子是强插的，要保证推荐帖子不在已读里 linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list)) logging.warning(
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index) "type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list)))
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8")) recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
# newcursor = cursor + 6 recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
if len(recommend_topic_id_list) > 0: # cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
recommend_topic_list = recommend_topic_id_list[0:size] # newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor) # redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict: if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"]) linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0: if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0:
for i in recommend_topic_list: for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)]) recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None: # if have_read_topic_id_list == None:
# have_read_topic_id_list = list() # have_read_topic_id_list = list()
# 用户关注标签 # 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id) redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else [] attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list)>0: if len(recommend_topic_list) > 0:
size = size-len(recommend_topic_list) size = size - len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list) have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list() # have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list) # have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter( # promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True) # is_online=1).values_list("topic_id",flat=True)
# #
# for topic_id in promote_recommend_topic_id_list: # for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id) # have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list() rank_topic_id_list = list()
if size>0: if size > 0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size, rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type, single_size=size, query=query,
filter_topic_id_list=have_read_topic_id_list, query_type=query_type,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase) filter_topic_id_list=have_read_topic_id_list,
index_type=index_type, routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None: # if len(recommend_topic_list) == 6 and query is None:
# if (size < 11): # if (size < 11):
...@@ -172,9 +183,14 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -172,9 +183,14 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(recommend_topic_list[3:6]) # topic_id_list.extend(recommend_topic_list[3:6])
# else: # else:
# topic_id_list.extend(rank_topic_id_list) # topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list) if query is None:
have_read_topic_id_list.extend(item["id"] for item in rank_topic_id_list)
else:
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000: if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list)-30000 cut_len = len(have_read_topic_id_list) - 30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:] have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = { redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list), "have_read_topic_list": json.dumps(have_read_topic_id_list),
...@@ -183,13 +199,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -183,13 +199,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 每个session key保存60天 # 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60) redis_client.expire(redis_key, 60 * 60 * 24 * 60)
if usefulrecall != -1: if usefulrecall != -1:
return recommend_topic_list,rank_topic_id_list,useful_topic_id_list return recommend_topic_list, rank_topic_id_list, useful_topic_id_list
else: else:
return recommend_topic_list, rank_topic_id_list return recommend_topic_list, rank_topic_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return [],[],[] return [], [], []
else: else:
return [], [] return [], []
...@@ -204,7 +220,8 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1): ...@@ -204,7 +220,8 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend") @bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1,useful_tag_list=[]): def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[]):
""" """
:remark:首页推荐，目前只推荐日记 :remark:首页推荐，目前只推荐日记
:param session_id: :param session_id:
...@@ -237,26 +254,34 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic ...@@ -237,26 +254,34 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset: if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset+size] recommend_topic_ids = have_read_topic_id_list[offset:offset + size]
else: else:
recommend_topic_ids = have_read_topic_id_list[0:size] recommend_topic_ids = have_read_topic_id_list[0:size]
else: else:
if usefulrecall != -1 and len(useful_tag_list)>0: if usefulrecall != -1 and len(useful_tag_list) > 0:
recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size, recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id,
query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall,useful_tag_list=useful_tag_list) tag_id=0, offset=0,
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids} size=size,
else: query_type=query_type,
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size, promote_topic_list=promote_topic_list,
query_type=query_type,promote_topic_list=promote_topic_list) usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids} return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]} return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []}
else: else:
return {"linucb_topic_ids": [], "rank_topic_ids": []} return {"linucb_topic_ids": [], "rank_topic_ids": []}
@bind("physical/search/discover_page") @bind("physical/search/discover_page")
...@@ -284,7 +309,7 @@ def discover_page(device_id="", user_id=-1, size=10): ...@@ -284,7 +309,7 @@ def discover_page(device_id="", user_id=-1, size=10):
@bind("physical/search/home_query") @bind("physical/search/home_query")
def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10): def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10, query_type=-1):
""" """
:remark:首页搜索，目前只推荐日记 :remark:首页搜索，目前只推荐日记
:param session_id: :param session_id:
...@@ -295,16 +320,28 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10) ...@@ -295,16 +320,28 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
:return: :return:
""" """
try: try:
result_topic_data = list()
if not user_id: if not user_id:
user_id = -1 user_id = -1
if not isinstance(device_id, str): if not isinstance(device_id, str):
device_id = "" device_id = ""
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query) recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size: offset=offset, size=size, query=query)
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, offset=offset, size=size,
query=query,disable_collpase=True) query=query, disable_collpase=True)
if query_type != 3 and rank_topic_id_list:
for item in rank_topic_id_list:
result_topic_data.append(item["id"])
logging.info("get result_topic_data:%s" % result_topic_data)
return {"recommend_topic_ids": result_topic_data}
return {"recommend_topic_ids": rank_topic_id_list} return {"recommend_topic_ids": rank_topic_id_list}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -326,9 +363,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -326,9 +363,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if not isinstance(user_id, int): if not isinstance(user_id, int):
user_id = -1 user_id = -1
if user_id > 0: if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id) redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id)
else: else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id) redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id)
if int(offset) == 0: if int(offset) == 0:
have_read_topic_list = list() have_read_topic_list = list()
# redis_dict = { # redis_dict = {
...@@ -341,7 +378,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -341,7 +378,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
have_read_topic_list = list() have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list) have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else [] have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id) have_read_topic_list.append(topic_id)
...@@ -349,28 +386,33 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -349,28 +386,33 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
topic_tag_result = list() topic_tag_result = list()
result = list() result = list()
if len(topic_tag_list) != 0: if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj, topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
topic_tag_size = 0
if topic_tag_size <size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj, size, es_cli_obj,
index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id index_type="topic", routing="3,4,5,6",
) topic_tag_list=topic_tag_list)
topic_user_size = len(topic_user_result) topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_user_result) have_read_topic_list.extend(topic_tag_result)
if topic_user_size < size: else:
size = size - topic_user_size topic_tag_size = 0
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, if topic_tag_size < size:
have_read_topic_list, size = size - topic_tag_size
size, es_cli_obj, if topic_user_id != -1:
index_type="topic", routing="4,5,6") topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list.extend(result) have_read_topic_list,
size, es_cli_obj,
index_type="topic",
routing="3,4,5,6",
topic_user_id=topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key) # have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else [] # have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
...@@ -446,8 +488,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10): ...@@ -446,8 +488,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。""" """帖子搜索。"""
try: try:
(topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters, (topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size) sorts_by=sorts_by, offset=offset, size=size)
return { return {
"topic_ids": topic_id_list, "topic_ids": topic_id_list,
...@@ -470,8 +512,10 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -470,8 +512,10 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try: try:
must_topic_id_list = list(topic_similarity_score_dict.keys()) must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size, topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,
must_topic_id_list=must_topic_id_list,index_type="topic",routing="4,5,6") single_size=size,
must_topic_id_list=must_topic_id_list, index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list} return {"recommend_topic_ids": topic_id_list}
except: except:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment