Commit c125262b authored by 段英荣's avatar 段英荣

Merge branch 'master' into 'similar_sort'

# Conflicts:
#   search/views/topic.py
parents 4654b320 3e502a65
...@@ -121,7 +121,7 @@ class TopicUtils(object): ...@@ -121,7 +121,7 @@ class TopicUtils(object):
return {} return {}
@classmethod @classmethod
def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],user_similar_score_list=[]): def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],user_similar_score_list=[]):
""" """
:需增加打散逻辑 :需增加打散逻辑
:remark:获取首页推荐帖子列表 :remark:获取首页推荐帖子列表
...@@ -291,8 +291,10 @@ class TopicUtils(object): ...@@ -291,8 +291,10 @@ class TopicUtils(object):
'operator': 'and', 'operator': 'and',
'fields': query_fields, 'fields': query_fields,
} }
query_function_score["query"]["bool"]["should"] = [ query_function_score["query"]["bool"]["should"] = [
{'multi_match': multi_match} {'multi_match': multi_match},
{"term":{"tag_list":tag_id}}
] ]
query_function_score["query"]["bool"]["minimum_should_match"] = 1 query_function_score["query"]["bool"]["minimum_should_match"] = 1
......
...@@ -13,7 +13,7 @@ from search.utils.common import * ...@@ -13,7 +13,7 @@ from search.utils.common import *
from libs.es import ESPerform from libs.es import ESPerform
def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE): def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try: try:
if user_id == -1: if user_id == -1:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id) + ":device_id:" + device_id redis_key = "physical:discover_page" + ":user_id:" + str(user_id) + ":device_id:" + device_id
...@@ -21,12 +21,14 @@ def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType. ...@@ -21,12 +21,14 @@ def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.
redis_key = "physical:discover_page" + ":user_id:" + str(user_id) redis_key = "physical:discover_page" + ":user_id:" + str(user_id)
redis_field_list = [b'have_read_topic_id'] redis_field_list = [b'have_read_topic_id']
redis_field_val_list = redis_client.hmget(redis_key,redis_field_list) redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else [] have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else []
recommend_topic_ids = [] recommend_topic_ids = []
topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id=user_id, offset=0, size=size,query_type=query_type,filter_topic_id_list=have_read_topic_id_list) topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list)
for topic_id in topic_id_dict: for topic_id in topic_id_dict:
recommend_topic_ids.append(topic_id) recommend_topic_ids.append(topic_id)
...@@ -35,35 +37,40 @@ def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType. ...@@ -35,35 +37,40 @@ def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.
redis_dict = { redis_dict = {
"have_read_topic_id": json.dumps(have_read_topic_id_list) "have_read_topic_id": json.dumps(have_read_topic_id_list)
} }
redis_client.hmset(redis_key,redis_dict) redis_client.hmset(redis_key, redis_dict)
return recommend_topic_ids return recommend_topic_ids
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return []
def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_type=TopicPageType.HOME_RECOMMEND):
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND):
try: try:
if query is None: if query is None:
if user_id == -1: if user_id == -1:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":user_id:" + str(
user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
else: else:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
else: else:
if user_id == -1: if user_id == -1:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type) redis_key = "physical:home_query" + ":user_id:" + str(
user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
else: else:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type) redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key,redis_field_list) redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
tag_recommend_redis_key = "physical:linucb:tag_recommend:device_id:" + str(device_id) tag_recommend_redis_key = "physical:linucb:tag_recommend:device_id:" + str(device_id)
recommend_tag_list = [] recommend_tag_list = []
tag_recommend_val = redis_client.get(tag_recommend_redis_key) tag_recommend_val = redis_client.get(tag_recommend_redis_key)
if tag_recommend_val: if tag_recommend_val:
recommend_tag_list = json.loads(str(tag_recommend_val,encoding="utf-8")) recommend_tag_list = json.loads(str(tag_recommend_val, encoding="utf-8"))
recommend_topic_ids = [] recommend_topic_ids = []
have_read_topic_id_list = list() have_read_topic_id_list = list()
...@@ -73,9 +80,13 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_ ...@@ -73,9 +80,13 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id) user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key) redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
user_similar_score_redis_list = json.loads(redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else [] user_similar_score_redis_list = json.loads(
redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query,query_type,filter_topic_id_list=have_read_topic_id_list,recommend_tag_list=recommend_tag_list,user_similar_score_list=user_similar_score_redis_list) topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id, tag_id, offset, size * size, query, query_type,
filter_topic_id_list=have_read_topic_id_list,
recommend_tag_list=recommend_tag_list,
user_similar_score_list=user_similar_score_redis_list)
have_read_group_id_set = set() have_read_group_id_set = set()
have_read_user_id_set = set() have_read_user_id_set = set()
unread_topic_id_dict = dict() unread_topic_id_dict = dict()
...@@ -86,16 +97,18 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_ ...@@ -86,16 +97,18 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
cur_page_user_num = 0 cur_page_user_num = 0
for topic_id in topic_id_dict: for topic_id in topic_id_dict:
if topic_id_dict[topic_id][0] in have_read_group_id_set or topic_id_dict[topic_id][1] in have_read_user_id_set: if topic_id_dict[topic_id][0] in have_read_group_id_set or topic_id_dict[topic_id][
1] in have_read_user_id_set:
unread_topic_id_dict[topic_id] = topic_id_dict[topic_id] unread_topic_id_dict[topic_id] = topic_id_dict[topic_id]
else: else:
if isinstance(topic_id_dict[topic_id][0],int) and topic_id_dict[topic_id][0]>0 and cur_page_group_num<(size*0.9): if isinstance(topic_id_dict[topic_id][0], int) and topic_id_dict[topic_id][
0] > 0 and cur_page_group_num < (size * 0.9):
have_read_group_id_set.add(topic_id_dict[topic_id][0]) have_read_group_id_set.add(topic_id_dict[topic_id][0])
have_read_user_id_set.add(topic_id_dict[topic_id][1]) have_read_user_id_set.add(topic_id_dict[topic_id][1])
have_read_topic_id_list.append(topic_id) have_read_topic_id_list.append(topic_id)
cur_page_group_num += 1 cur_page_group_num += 1
recommend_topic_ids.append(topic_id) recommend_topic_ids.append(topic_id)
elif topic_id_dict[topic_id] and cur_page_user_num<(size*0.1): elif topic_id_dict[topic_id] and cur_page_user_num < (size * 0.1):
have_read_user_id_set.add(topic_id_dict[topic_id][1]) have_read_user_id_set.add(topic_id_dict[topic_id][1])
cur_page_user_num += 1 cur_page_user_num += 1
recommend_topic_ids.append(topic_id) recommend_topic_ids.append(topic_id)
...@@ -106,10 +119,9 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_ ...@@ -106,10 +119,9 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
if len(recommend_topic_ids) >= size: if len(recommend_topic_ids) >= size:
break break
if len(recommend_topic_ids) < size and len(unread_topic_id_dict) > 0:
if len(recommend_topic_ids) < size and len(unread_topic_id_dict)>0:
for unread_topic_id in unread_topic_id_dict: for unread_topic_id in unread_topic_id_dict:
if len(recommend_topic_ids)<size: if len(recommend_topic_ids) < size:
recommend_topic_ids.append(unread_topic_id) recommend_topic_ids.append(unread_topic_id)
have_read_topic_id_list.append(unread_topic_id) have_read_topic_id_list.append(unread_topic_id)
else: else:
...@@ -119,27 +131,29 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_ ...@@ -119,27 +131,29 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
cut_len = len(have_read_topic_id_list)-5000 cut_len = len(have_read_topic_id_list)-5000
have_read_topic_id_list = have_read_topic_id_list[cut_len:] have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = { redis_dict = {
"have_read_topic_list":json.dumps(have_read_topic_id_list), "have_read_topic_list": json.dumps(have_read_topic_id_list),
} }
redis_client.hmset(redis_key,redis_dict) redis_client.hmset(redis_key, redis_dict)
# 每个session key保存15分钟 # 每个session key保存15分钟
redis_client.expire(redis_key,60*60*24*3) redis_client.expire(redis_key, 60 * 60 * 24 * 3)
return recommend_topic_ids return recommend_topic_ids
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return []
@bind("physical/search/query_tag_id_by_topic") @bind("physical/search/query_tag_id_by_topic")
def query_tag_id_by_topic(offset=0,size=10,topic_id_list=[],user_id=-1): def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
try: try:
return TopicUtils.get_topic_tag_info(offset,size,topic_id_list,user_id) return TopicUtils.get_topic_tag_info(offset, size, topic_id_list, user_id)
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {} return {}
@bind("physical/search/home_recommend") @bind("physical/search/home_recommend")
def home_recommend(device_id="",user_id=-1,offset=0,size=10,query_type=TopicPageType.HOME_RECOMMEND): def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND):
""" """
:remark:首页推荐,目前只推荐日记 :remark:首页推荐,目前只推荐日记
:param session_id: :param session_id:
...@@ -150,20 +164,21 @@ def home_recommend(device_id="",user_id=-1,offset=0,size=10,query_type=TopicPage ...@@ -150,20 +164,21 @@ def home_recommend(device_id="",user_id=-1,offset=0,size=10,query_type=TopicPage
""" """
try: try:
if not user_id: if not user_id:
user_id=-1 user_id = -1
if not isinstance(device_id,str): if not isinstance(device_id, str):
device_id = "" device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset=0,size=size,query_type=query_type) recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type)
return {"recommend_topic_ids":recommend_topic_ids} return {"recommend_topic_ids": recommend_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []} return {"recommend_topic_ids": []}
@bind("physical/search/discover_page") @bind("physical/search/discover_page")
def discover_page(device_id="",user_id=-1,size=10): def discover_page(device_id="", user_id=-1, size=10):
""" """
:remark:首页推荐,目前只推荐日记 :remark:首页推荐,目前只推荐日记
:param session_id: :param session_id:
...@@ -174,20 +189,20 @@ def discover_page(device_id="",user_id=-1,size=10): ...@@ -174,20 +189,20 @@ def discover_page(device_id="",user_id=-1,size=10):
""" """
try: try:
if not user_id: if not user_id:
user_id=-1 user_id = -1
if not isinstance(device_id,str): if not isinstance(device_id, str):
device_id = "" device_id = ""
recommend_topic_ids = get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE) recommend_topic_ids = get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE)
return {"recommend_topic_ids":recommend_topic_ids} return {"recommend_topic_ids": recommend_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []} return {"recommend_topic_ids": []}
@bind("physical/search/home_query") @bind("physical/search/home_query")
def home_query(device_id="",user_id=-1,query="",offset=0,size=10): def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10):
""" """
:remark:首页搜索,目前只推荐日记 :remark:首页搜索,目前只推荐日记
:param session_id: :param session_id:
...@@ -199,19 +214,20 @@ def home_query(device_id="",user_id=-1,query="",offset=0,size=10): ...@@ -199,19 +214,20 @@ def home_query(device_id="",user_id=-1,query="",offset=0,size=10):
""" """
try: try:
if not user_id: if not user_id:
user_id=-1 user_id = -1
if not isinstance(device_id,str): if not isinstance(device_id, str):
device_id = "" device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query) recommend_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query)
return {"recommend_topic_ids":recommend_topic_ids} return {"recommend_topic_ids": recommend_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []} return {"recommend_topic_ids": []}
@bind("physical/search/topic_detail_page_recommend") @bind("physical/search/topic_detail_page_recommend")
def topic_detail_page_recommend(device_id="",user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,filter_topic_user_id=False,offset=0,size=10): def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_group_id=-1, topic_user_id=-1,
filter_topic_user_id=False, offset=0, size=10):
""" """
:remark:帖子详情页推荐策略,缺少第一个卡片策略 :remark:帖子详情页推荐策略,缺少第一个卡片策略
:param user_id: :param user_id:
...@@ -221,29 +237,31 @@ def topic_detail_page_recommend(device_id="",user_id=-1,topic_id=-1,topic_group_ ...@@ -221,29 +237,31 @@ def topic_detail_page_recommend(device_id="",user_id=-1,topic_id=-1,topic_group_
:return: :return:
""" """
try: try:
if not isinstance(user_id,int): if not isinstance(user_id, int):
user_id = -1 user_id = -1
redis_key = "physical:topic_detail_page_recommend" + ":user_id:" + str(user_id) + ":device_id:" + str(device_id) redis_key = "physical:topic_detail_page_recommend" + ":user_id:" + str(user_id) + ":device_id:" + str(device_id)
have_read_topic_redis_data = redis_client.get(redis_key) have_read_topic_redis_data = redis_client.get(redis_key)
have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else [] have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
#获取es链接对象 # 获取es链接对象
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
# 获取帖子标签列表 # 获取帖子标签列表
topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id,es_cli_obj) topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id, es_cli_obj)
result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,have_read_topic_list,offset,size,es_cli_obj) result_list = TopicUtils.get_topic_detail_recommend_list(user_id, topic_id, topic_tag_list, topic_group_id,
topic_user_id, filter_topic_user_id,
have_read_topic_list, offset, size, es_cli_obj)
recommend_topic_ids_list = list() recommend_topic_ids_list = list()
if len(result_list)>0: if len(result_list) > 0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list] recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
have_read_topic_list.extend(recommend_topic_ids_list) have_read_topic_list.extend(recommend_topic_ids_list)
have_read_topic_len = len(have_read_topic_list) have_read_topic_len = len(have_read_topic_list)
if have_read_topic_len>5000: if have_read_topic_len > 5000:
have_read_topic_list = have_read_topic_list[(have_read_topic_len-5000):] have_read_topic_list = have_read_topic_list[(have_read_topic_len - 5000):]
redis_client.set(redis_key, json.dumps(have_read_topic_list)) redis_client.set(redis_key, json.dumps(have_read_topic_list))
return {"recommend_topic_ids": recommend_topic_ids_list} return {"recommend_topic_ids": recommend_topic_ids_list}
...@@ -253,7 +271,7 @@ def topic_detail_page_recommend(device_id="",user_id=-1,topic_id=-1,topic_group_ ...@@ -253,7 +271,7 @@ def topic_detail_page_recommend(device_id="",user_id=-1,topic_id=-1,topic_group_
@bind("physical/search/query_topic_by_tag_aggregation") @bind("physical/search/query_topic_by_tag_aggregation")
def query_topic_by_tag_aggregation(user_id,tag_id, offset, size): def query_topic_by_tag_aggregation(user_id, tag_id, offset, size):
""" """
:remark 按标签聚合召回帖子 :remark 按标签聚合召回帖子
:param tag_id: :param tag_id:
...@@ -267,9 +285,9 @@ def query_topic_by_tag_aggregation(user_id,tag_id, offset, size): ...@@ -267,9 +285,9 @@ def query_topic_by_tag_aggregation(user_id,tag_id, offset, size):
if not tag_id: if not tag_id:
tag_id = -1 tag_id = -1
result_list = TopicUtils.get_tag_aggregation_topic_id_list(user_id,tag_id,offset,size) result_list = TopicUtils.get_tag_aggregation_topic_id_list(user_id, tag_id, offset, size)
recommend_topic_ids_list = list() recommend_topic_ids_list = list()
if len(result_list)>0: if len(result_list) > 0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list] recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
return {"recommend_topic_id": recommend_topic_ids_list} return {"recommend_topic_id": recommend_topic_ids_list}
...@@ -303,11 +321,11 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -303,11 +321,11 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
""" """
try: try:
must_topic_id_list = list(topic_similarity_score_dict.keys()) must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id=-1, offset=offset, size=size,must_topic_id_list=must_topic_id_list) topic_id_dict = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,
must_topic_id_list=must_topic_id_list)
recommend_topic_ids = list(topic_id_dict.keys()) recommend_topic_ids = list(topic_id_dict.keys())
return {"recommend_topic_ids":recommend_topic_ids} return {"recommend_topic_ids": recommend_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_id": []} return {"recommend_topic_id": []}
...@@ -129,7 +129,7 @@ class Command(BaseCommand): ...@@ -129,7 +129,7 @@ class Command(BaseCommand):
def generate_topic_score_detail(self): def generate_topic_score_detail(self):
try: try:
topic_id_dict = TopicUtils.get_recommend_topic_ids(241432787, 0, 500,query_type=TopicPageType.HOME_RECOMMEND,test_score=True) topic_id_dict = TopicUtils.get_recommend_topic_ids(241432787,0, 0, 500,query_type=TopicPageType.HOME_RECOMMEND,test_score=True)
for topic_id in topic_id_dict: for topic_id in topic_id_dict:
offline_score = 0.0 offline_score = 0.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment