Skip to content

P
physical
Commit f90e0b9e authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parent e10b33c3
Hide whitespace changes
Inline Side-by-side
Showing with 168 additions and 129 deletions
app_conf.xml
View file @ f90e0b9e
......@@ -17,5 +17,6 @@
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
<element value="search.views.search_hotword"/>
</config>
</gm_rpcd_config>
search/views/search_hotword.py
View file @ f90e0b9e
......@@ -33,36 +33,37 @@ def search_hotword(device_id=-1):
try:
all_tag_name_list = set()
# 获取搜索推荐热词
results_registr_tag = RegisterShowTag.objects.filter(is_deleted=False, is_online=1).count()
results_tag = Tag.objects.filter(is_online=True, collection=1).count()
for i in range(0, 6):
tag_id = random.randint(0, results_registr_tag)
results_tag_chose = Tag.objects.filter(id=tag_id).values_list("name", flat=True)
all_tag_name_list.add(results_tag_chose)
tag_id = random.sample(range(1, results_registr_tag), 1)
results_tag_chose = Tag.objects.filter(id__in=tag_id).values_list("name", flat=True)
for i in results_tag_chose:
all_tag_name_list.add(i)
logging.info("get all_tag_name_list01:%s" % all_tag_name_list)
# 获取个性化标签
linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
linucb_recommend_tag_data = redis_client.get(tag_recommend_redis_key)
linucb_recommend_tag_list = json.loads(linucb_recommend_tag_data) if linucb_recommend_tag_data else []
for item in linucb_recommend_tag_list:
all_tag_name_list.add(item)
results_tag_recommend = Tag.objects.filter(id=item, is_online=True).values_list("name", flat=True)
all_tag_name_list.add(results_tag_recommend[0])
if len(all_tag_name_list) == 12:
return {"recommend_tag_name": list(all_tag_name_list)}
logging.info("get all_tag_name_list02:%s" % all_tag_name_list)
logging.info("get all_tag_name_list02:%s" % all_tag_name_list)
# 取不够数则取核心标签
if len(all_tag_name_list) < 12:
for i in range(0, 12):
tag_id = random.randint(0, results_tag)
results_tag_hexin = Tag.objects.filter(id=tag_id).values_list("name", flat=True)
tag_id = random.sample(range(1, results_tag), 12 - len(all_tag_name_list))
results_tag_hexin = Tag.objects.filter(id__in=tag_id).values_list("name", flat=True)
if results_tag_hexin not in all_tag_name_list:
all_tag_name_list.add(results_tag_hexin)
all_tag_name_list.add(results_tag_hexin[0])
if len(all_tag_name_list) >= 12:
logging.info("get all_tag_name_list03:%s" % all_tag_name_list)
return {"recommend_name": all_tag_name_list}
return {"recommend_tag_name": list(all_tag_name_list)}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_name": []}
return {"recommend_tag_name": []}
search/views/topic.py
View file @ f90e0b9e
......@@ -15,7 +15,6 @@ from libs.es import ESPerform
from django.conf import settings
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try:
if user_id == -1:
......@@ -30,10 +29,11 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,index_type="topic",routing="4,5,6")
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,
single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = {
......@@ -48,23 +48,25 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1):
query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1):
try:
topic_star_routing = "6"
index_type = "topic-high-star"
if query is None:
if user_id>0:
if user_id > 0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
topic_star_routing = "3,4,5,6"
index_type = "topic"
if user_id>0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
if user_id > 0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
......@@ -76,7 +78,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset>0: # 首次搜索时不需要过滤已读
if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
......@@ -86,66 +88,71 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
attention_tag_list = list()
recommend_topic_list = list()
if query is None:
if user_id != -1:
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6")
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list()
recommend_topic_list = list()
if b"data" in recommend_topic_dict:
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
if linucb_recommend_topic_id_list == None:
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
if user_id != -1:
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6")
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list()
recommend_topic_list = list()
if b"data" in recommend_topic_dict:
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
if linucb_recommend_topic_id_list == None:
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning(
"type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
# have_read_topic_id_list = list()
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list)>0:
size = size-len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True)
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
# have_read_topic_id_list = list()
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list) > 0:
size = size - len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True)
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list()
if size>0:
if size > 0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
single_size=size, query=query,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type, routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None:
# if (size < 11):
......@@ -162,7 +169,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list)-30000
cut_len = len(have_read_topic_id_list) - 30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list),
......@@ -171,13 +178,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60)
if usefulrecall != -1:
return recommend_topic_list,rank_topic_id_list,useful_topic_id_list
return recommend_topic_list, rank_topic_id_list, useful_topic_id_list
else:
return recommend_topic_list, rank_topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return [],[],[]
return [], [], []
else:
return [], []
......@@ -192,7 +199,8 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1):
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
......@@ -225,26 +233,33 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset+size]
recommend_topic_ids = have_read_topic_id_list[offset:offset + size]
else:
recommend_topic_ids = have_read_topic_id_list[0:size]
else:
if usefulrecall != -1:
recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids}
else:
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
if usefulrecall != -1:
recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id,
tag_id=0, offset=0,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]}
return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []}
else:
return {"linucb_topic_ids": [], "rank_topic_ids": []}
return {"linucb_topic_ids": [], "rank_topic_ids": []}
@bind("physical/search/discover_page")
......@@ -288,11 +303,12 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
if not isinstance(device_id, str):
device_id = ""
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, query=query)
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size,
query=query,disable_collpase=True)
query=query, disable_collpase=True)
return {"recommend_topic_ids": rank_topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -314,9 +330,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if not isinstance(user_id, int):
user_id = -1
if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id)
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id)
else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id)
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id)
if int(offset) == 0:
have_read_topic_list = list()
# redis_dict = {
......@@ -329,7 +345,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list)
have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id)
......@@ -337,28 +353,33 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
topic_tag_result = list()
result = list()
if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
topic_tag_size = 0
if topic_tag_size <size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
index_type="topic", routing="3,4,5,6",
topic_tag_list=topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
topic_tag_size = 0
if topic_tag_size < size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic",
routing="3,4,5,6",
topic_user_id=topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
......@@ -434,8 +455,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
(topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
(topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
return {
"topic_ids": topic_id_list,
......@@ -458,10 +479,26 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try:
must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
must_topic_id_list=must_topic_id_list,index_type="topic",routing="4,5,6")
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,
single_size=size,
must_topic_id_list=must_topic_id_list, index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_id": []}
@bind("physical/search/query_topic_search")
def query_topic_search(query=""):
"""
召回搜索帖子
当搜索的内容 完全匹配 用户昵称，且该用户为推荐用户时，帖子tab中也会展示用户栏
:param query:
:return:
"""
#召回完全匹配的用户
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment