#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function
from gm_rpcd.all import bind
import logging
import traceback
import json
from search.utils.topic import TopicUtils
from trans2es.models.topic import TopicHomeRecommend
from libs.cache import redis_client
from search.utils.common import *
from libs.es import ESPerform
from django.conf import settings
from libs.tools import get_have_read_lin_pictorial_id_list
import datetime
from trans2es.models.tag import Tag
from search.views.tag import get_same_tagset_ids
from linucb.views.linucb import LinUCB
from alpha_types.physical.enum import STRATEGY_TYPE
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try:
if user_id == -1:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id) + ":device_id:" + device_id
else:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id)
redis_field_list = [b'have_read_topic_id']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else []
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
recommend_topic_ids, ret_data_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0,
size=size,
single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = {
"have_read_topic_id": json.dumps(have_read_topic_id_list)
}
redis_client.hmset(redis_key, redis_dict)
return recommend_topic_ids
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, underexposure_lin_topic_count=0, size=0, query=None,
query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1, useful_tag_list=[], has_score=False, gray_list=[1]):
try:
topic_star_routing = "6"
index_type = "topic-high-star"
device_redis_key = ""
if query is None:
if user_id > 0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(
query_type)
else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
topic_star_routing = "3,4,5,6"
index_type = "topic"
if user_id > 0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
# 获取已读帖子
have_read_topic_id_list = list()
if redis_field_val_list[0]:
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
elif len(device_redis_key) > 0:
redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list)
if redis_field_val_list[0]:
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list)
useful_topic_id_list = list()
recommend_topic_user_list = list()
attention_tag_list = list()
recommend_topic_list = list()
underexposure_lin_topic_ids = list()
if query is None:
if user_id != -1:
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6",
useful_tag_list=useful_tag_list)
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
linucb_recommend_tags = list()
if STRATEGY_TYPE.CTR_GRAY in gray_list:
topic_recommend_redis_key = "ctr_physical:linucb:topic_recommend:device_id:" + str(device_id)
linucb_recommend_tags_key = "ctr_physical:linucb:tag_recommend:device_id:" + str(device_id)
linucb_recommend_tags = redis_client.get(linucb_recommend_tags_key)
if linucb_recommend_tags:
linucb_recommend_tags = json.loads(linucb_recommend_tags)
else:
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
linucb_recommend_tags_key = "physical:linucb:tag_recommend:device_id:" + str(device_id)
linucb_recommend_tags = redis_client.get(linucb_recommend_tags_key)
if linucb_recommend_tags:
linucb_recommend_tags = json.loads(linucb_recommend_tags)
# linucb 推荐新帖子
if linucb_recommend_tags:
linucb_recommend_tags_set_tags = get_same_tagset_ids(linucb_recommend_tags)
if underexposure_lin_topic_count:
underexposure_lin_topic_ids = ESPerform.get_tag_new_topic_list(linucb_recommend_tags_set_tags, have_read_topic_id_list, underexposure_lin_topic_count)
size = size - len(underexposure_lin_topic_ids)
have_read_topic_id_list.extend(underexposure_lin_topic_ids)
redis_client.publish("new_topic_impression", json.dumps(underexposure_lin_topic_ids))
# linucb 推荐老帖子
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list()
recommend_topic_list = list()
if b"data" in recommend_topic_dict:
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
if linucb_recommend_topic_id_list == None:
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning(
"type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
# have_read_topic_id_list = list()
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list) > 0:
size = size - len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True)
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list()
ret_data_list = list()
rank_topic_score = list()
if size > 0:
if has_score:
rank_topic_id_list, ret_data_list, rank_topic_score = TopicUtils.get_recommend_topic_ids(
user_id=user_id, tag_id=tag_id,
offset=0, size=size,
single_size=size, query=query,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,
routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase, has_score=has_score)
else:
rank_topic_id_list, ret_data_list = TopicUtils.get_recommend_topic_ids(
user_id=user_id, tag_id=tag_id,
offset=0, size=size,
single_size=size, query=query,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,
routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None:
# if (size < 11):
# topic_id_list.extend(rank_topic_id_list[0:3])
# topic_id_list.extend(recommend_topic_list[0:3])
# topic_id_list.extend(rank_topic_id_list[3:size])
# topic_id_list.extend(recommend_topic_list[3:6])
# else:
# topic_id_list.extend(rank_topic_id_list[0:size - 7])
# topic_id_list.extend(recommend_topic_list[0:3])
# topic_id_list.extend(rank_topic_id_list[size - 7:size])
# topic_id_list.extend(recommend_topic_list[3:6])
# else:
# topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list) - 30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list),
}
redis_client.hmset(redis_key, redis_dict)
# 失效时间为第二天凌晨3点
# if redis_client.ttl(redis_key)<0:
# today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d")
# if STRATEGY_TYPE.READ_GRAY in gray_list:
# end_day = today + datetime.timedelta(days=10)
# else:
# end_day = today + datetime.timedelta(days=1)
# nowTime = datetime.datetime.now()
# expire_time = (end_day - nowTime).seconds + 3 * 60 * 60
# redis_client.expire(redis_key, expire_time)
ret_list = rank_topic_id_list if query is None else ret_data_list
if usefulrecall != -1:
if has_score:
return underexposure_lin_topic_ids, recommend_topic_list, ret_list, useful_topic_id_list, rank_topic_score
else:
return underexposure_lin_topic_ids, recommend_topic_list, ret_list, useful_topic_id_list
else:
if has_score:
return underexposure_lin_topic_ids, recommend_topic_list, ret_list, rank_topic_score
else:
return underexposure_lin_topic_ids, recommend_topic_list, ret_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
if has_score:
return [], [], [], [], []
else:
return [], [], [], []
else:
if has_score:
return [], [], [], []
else:
return [], [], []
def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list=[1]):
try:
if STRATEGY_TYPE.CTR_GRAY in gray_list:
pictorial_recommend_redis_key = "ctr_physical:linucb:pictorial_recommend:device_id:" + str(device_id)
else:
pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id)
have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id, user_id,
TopicPageType.HOME_RECOMMEND)
pictorial_recommend_dict = redis_client.hgetall(pictorial_recommend_redis_key)
pictorial_recommend_list = list()
if b"data" in pictorial_recommend_dict:
pictorial_recommend_list = json.loads(pictorial_recommend_dict[b"data"])
if pictorial_recommend_list == None:
pictorial_recommend_list = list()
recommend_pictorial_id_list = list(set(pictorial_recommend_list) - set(have_read_lin_pictorial_id_list))
recommend_pictorial_id_list.sort(key=pictorial_recommend_list.index)
recommend_pictorial_ids = recommend_pictorial_id_list[0:size]
have_read_lin_pictorial_id_list.extend(recommend_pictorial_ids)
if user_id and int(user_id) > 0:
redis_key = "physical:home_pictorial_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(
TopicPageType.HOME_RECOMMEND)
else:
redis_key = "physical:home_pictorial_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(
TopicPageType.HOME_RECOMMEND)
redis_field_list = 'have_read_pictorial_list'
redis_client.hset(redis_key, redis_field_list, have_read_lin_pictorial_id_list)
today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d")
# 问题改成10天已读
end_day = today + datetime.timedelta(days=10)
nowTime = datetime.datetime.now()
expire_time = (end_day - nowTime).seconds + 3 * 60 * 60
redis_client.expire(redis_key, expire_time)
return recommend_pictorial_ids
else:
return []
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_topic_and_pictorial_detail_for_debug(device_id="", underexposure_lin_topic_ids=[], linucb_topic_id_list=[],
rank_topic_id_list=[], linucb_pictorial_id_list=[],
rank_topic_score=[], gray_list=[1]):
try:
linucb_tag_dict = get_device_linucb_tag(device_id, size=20, has_score=True, gray_list=gray_list)
linucb_topic_results = dict()
linucb_pictorial_results = dict()
rank_topic_results = dict()
linucb_topic_id_list_all = linucb_topic_id_list + underexposure_lin_topic_ids
underexposure_lin_topic_results = dict()
exposure_lin_topic_results = dict()
if len(linucb_topic_id_list_all) > 0:
linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list_all),
linucb_topic_list=linucb_topic_id_list_all)
if len(linucb_topic_dict) > 0:
for topic_id in linucb_topic_id_list_all:
topic_id = topic_id
edit_tag_List = linucb_topic_dict[topic_id]['edit_tag_list']
content_level = linucb_topic_dict[topic_id]['content_level']
edit_tag_name_score_list = list()
for edit_tag in edit_tag_List:
edit_tag_List_dict = get_edit_tag_name(edit_tag_List)
if edit_tag in linucb_tag_dict:
edit_tag_name_score_list.append(
{"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"],
"name": linucb_tag_dict[edit_tag]["name"]})
else:
edit_tag_name_score_list.append(
{"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
linucb_topic_results.update({str(topic_id): {"id": topic_id, "content_level": content_level,
"edit_tags": edit_tag_name_score_list}})
# logging.info("linucb_topic:{}".format(linucb_topic_results))
underexposure_lin_topic_results = {k: v for k, v in linucb_topic_results.items() if
int(k) in underexposure_lin_topic_ids}
exposure_lin_topic_results = {k: v for k, v in linucb_topic_results.items() if
int(k) in linucb_topic_id_list}
if len(rank_topic_id_list) > 0:
rank_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(rank_topic_id_list),
linucb_topic_list=rank_topic_id_list)
if len(rank_topic_dict) > 0:
for i in range(len(rank_topic_id_list)):
topic_id = rank_topic_id_list[i]
score = rank_topic_score[i]
rank_topic_results.update({str(topic_id): {"id": topic_id, "rank": score}})
# logging.info("rank_topic:{}".format(rank_topic_results))
if len(linucb_pictorial_id_list) > 0:
if len(linucb_pictorial_id_list) > 0:
linucb_pictorial_dict = TopicUtils.get_linucb_pictorial_info_for_debug(len(linucb_pictorial_id_list),
linucb_pictorial_list=linucb_pictorial_id_list)
if len(linucb_pictorial_dict) > 0:
for pictorial_id in linucb_pictorial_id_list:
pictorial_id = pictorial_id
edit_tag_List = linucb_pictorial_dict[pictorial_id]['edit_tag_list']
edit_tag_name_score_list = list()
for edit_tag in edit_tag_List:
edit_tag_List_dict = get_edit_tag_name(edit_tag_List)
if edit_tag in linucb_tag_dict:
edit_tag_name_score_list.append(
{"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"],
"name": linucb_tag_dict[edit_tag]["name"]})
else:
edit_tag_name_score_list.append(
{"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
linucb_pictorial_results.update({str(pictorial_id): {"id": pictorial_id,
"edit_tags": edit_tag_name_score_list}})
# logging.info("linucb_pictorial:{}".format(linucb_pictorial_results))
result = {"underexposure_lin_topic": underexposure_lin_topic_results,
"exposure_lin_topic": exposure_lin_topic_results, "rank_topic": rank_topic_results,
"linucb_pictorial": linucb_pictorial_results}
logging.info("get_topic_and_pictorial_detail_for_debug:result:{}:device_id:{}".format(result, device_id))
return result
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"underexposure_lin_topic": {}, "exposure_lin_topic": {}, "rank_topic": {}, "linucb_pictorial": {}}
@bind("physical/search/query_tag_id_by_topic")
def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
try:
return TopicUtils.get_topic_tag_info(offset, size, topic_id_list, user_id)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {}
@bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_count =4, lin_pictorial_count=4, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[], is_debug=False,gray_list=[1]):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
:param user_id:
:param offset:
:param size:
:return:
"""
try:
if not user_id:
user_id = -1
if not isinstance(device_id, str):
device_id = ""
recommend_topic_ids = list()
recommend_pictorial_ids = list()
rank_topic_ids = list()
useful_topic_ids = list()
rank_topic_score = list()
es_node_load_high_flag = False
# try:
# es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
# except:
# logging.error("catch exception,err_msg:%s" % traceback.format_exc())
# es_node_load_high_flag = True
if es_node_load_high_flag:
temp_downgrading_key = "physical:home_recommend:user_id:241407656:query_type:1"
redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(temp_downgrading_key, redis_field_list)
if redis_field_val_list[0]:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset + size]
else:
recommend_topic_ids = have_read_topic_id_list[0:size]
else:
if usefulrecall != -1 and len(useful_tag_list) > 0:
recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count,
gray_list=gray_list)
size = size - len(recommend_pictorial_ids)
if is_debug:
has_score = True
underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids, useful_topic_ids, rank_topic_score = get_home_recommend_topic_ids(
user_id, device_id,
tag_id=0, offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list, has_score=has_score, gray_list=gray_list)
else:
has_score = False
underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id,
device_id,
tag_id=0,
offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list,
has_score=has_score,
gray_list=gray_list)
if not is_debug:
return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids}
else:
results = get_topic_and_pictorial_detail_for_debug(device_id, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids,
recommend_pictorial_ids, rank_topic_score, gray_list)
return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids,
"debug_model_data": results}
else:
recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count,
gray_list=gray_list)
size = size - len(recommend_pictorial_ids)
if is_debug:
has_score = True
underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids, rank_topic_score = get_home_recommend_topic_ids(user_id,
device_id,
tag_id=0,
offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
has_score=has_score,
gray_list=gray_list)
else:
has_score = False
underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
has_score=has_score,
gray_list=gray_list)
if not is_debug:
return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"linucb_pictorial_ids": recommend_pictorial_ids}
else:
results = get_topic_and_pictorial_detail_for_debug(device_id, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids,
recommend_pictorial_ids, rank_topic_score, gray_list)
return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"linucb_pictorial_ids": recommend_pictorial_ids, "debug_model_data": results}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return {"underexposure_lin_topic_ids": [], "linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": [],"linucb_pictorial_ids": [], "debug_model_data": {}}
else:
return {"underexposure_lin_topic_ids": [], "linucb_topic_ids": [], "rank_topic_ids": [], "linucb_pictorial_ids": [],"debug_model_data": {}}
@bind("physical/search/discover_page")
def discover_page(device_id="", user_id=-1, size=10):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
:param user_id:
:param offset:
:param size:
:return:
"""
try:
if not user_id:
user_id = -1
if not isinstance(device_id, str):
device_id = ""
recommend_topic_ids = get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE)
return {"recommend_topic_ids": recommend_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
@bind("physical/search/home_query")
def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10, query_type=-1):
"""
:remark:首页搜索，目前只推荐日记
:param session_id:
:param user_id:
:param query:
:param offset:
:param size:
:return:
"""
try:
result_topic_data = list()
if not user_id:
user_id = -1
if not isinstance(device_id, str):
device_id = ""
unexposure_lin_topic, recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, query=query)
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
unexposure_lin_topic, recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size,
query=query, disable_collpase=True)
if query_type != 3 and rank_topic_id_list:
for item in rank_topic_id_list:
result_topic_data.append(item["id"])
logging.info("get result_topic_data:%s" % result_topic_data)
return {"recommend_topic_ids": result_topic_data}
return {"recommend_topic_ids": rank_topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
@bind("physical/search/topic_detail_page_recommend")
def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pictorial_id=-1, topic_user_id=-1,
filter_topic_user_id=False, offset=0, size=10, topic_tag_list=[]):
"""
:remark:帖子详情页推荐策略，缺少第一个卡片策略
:param user_id:
:param topic_id:
:param topic_group_id:
:param topic_user_id:
:return:
"""
try:
if not isinstance(user_id, int):
user_id = -1
if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id)
else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id)
if int(offset) == 0:
have_read_topic_list = list()
# redis_dict = {
# "have_read_topic_id": json.dumps(have_read_topic_list)
# }
redis_client.delete(redis_key)
# redis_client.expire(redis_key, 60 * 60 * 24)
else:
have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id)
topic_user_result = list()
topic_tag_result = list()
normal_tag_result = list()
category_tag_reuslt = list()
result = list()
topic_tag_list = get_same_tagset_ids(topic_tag_list)
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id__in=topic_tag_list, is_online=True, is_deleted=False).values_list("id", "is_category")
category_tag_list = list()
normal_tag_list = list()
for id, is_category in tag_query_results:
if is_category:
category_tag_list.append(id)
else:
normal_tag_list.append(id)
if len(normal_tag_list) != 0:
normal_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="3,4,5,6",
topic_tag_list=normal_tag_list)
normal_tag_size = len(normal_tag_result)
have_read_topic_list.extend(normal_tag_result)
else:
normal_tag_size = 0
if normal_tag_size < size:
size = size - normal_tag_size
if len(category_tag_list) != 0:
category_tag_reuslt = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic",
routing="3,4,5,6",
topic_tag_list=category_tag_list)
category_tag_size = len(category_tag_reuslt)
have_read_topic_list.extend(category_tag_reuslt)
else:
category_tag_size = 0
if category_tag_size < size:
size = size - category_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic",
routing="3,4,5,6",
topic_user_id=topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
#
# redis_key = "physical:topic_detail_page_recommend" + ":user_id:" + str(user_id) + ":device_id:" + str(device_id)
# have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
#
# # 获取es链接对象
# es_cli_obj = ESPerform.get_cli()
#
# # 获取帖子标签列表
# topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id, es_cli_obj)
# result_list = TopicUtils.get_topic_detail_recommend_list(user_id, topic_id, topic_tag_list, topic_pictorial_id,
# topic_user_id, filter_topic_user_id,
# have_read_topic_list, offset, size, es_cli_obj,index_type="topic",routing="4,5,6")
# result_list = TopicUtils.get_recommend_topic_ids(user_id, tag_id=-1, single_size=size, offset=offset, size=size,
# topic_user_id=topic_user_id, current_topic_id=topic_id,
# topic_tag_list=topic_tag_list)
#
# user_recommend_list = TopicUtils.get_recommend_topic_ids(user_id, tag_id=-1, single_size=size, offset=offset, size=size,
# topic_user_id=topic_user_id, current_topic_id=topic_id,
# topic_tag_list=topic_tag_list)
redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_list),
}
redis_client.hmset(redis_key, redis_dict)
redis_client.expire(redis_key, 60 * 60 * 24)
result_list = []
logging.warning("topic_tag_result:%s" % str(topic_tag_result))
logging.warning("topic_user_result:%s" % str(topic_user_result))
logging.warning("result:%s" % str(result))
recommend_topic_ids_list = list()
recommend_topic_ids_list.extend(normal_tag_result)
recommend_topic_ids_list.extend(category_tag_reuslt)
recommend_topic_ids_list.extend(topic_user_result)
recommend_topic_ids_list.extend(result)
return {"recommend_topic_ids": recommend_topic_ids_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
@bind("physical/search/query_topic_by_tag_aggregation")
def query_topic_by_tag_aggregation(user_id, tag_id, offset, size):
"""
:remark 按标签聚合召回帖子
:param tag_id:
:param offset:
:param size:
:return:
"""
try:
if not user_id:
user_id = -1
if not tag_id:
tag_id = -1
result_list = TopicUtils.get_tag_aggregation_topic_id_list(user_id, tag_id, offset, size)
recommend_topic_ids_list = list()
if len(result_list) > 0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
return {"recommend_topic_id": recommend_topic_ids_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_id": []}
@bind("physical/search/topic")
def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10, user_id=-1):
"""帖子搜索。"""
try:
(topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
return {
"topic_ids": topic_id_list,
"total_count": total_count
}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": [], "total_count": 0}
@bind("physical/search/query_topic_by_user_similarity")
def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=10):
"""
:remark 按帖子所属用户相似度召回帖子
:param tag_id:
:param offset:
:param size:
:return:
"""
try:
must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list, ret_data_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset,
size=size,
single_size=size,
must_topic_id_list=must_topic_id_list,
index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_id": []}
@bind("physical/search/clear_history_for_debug")
def clear_history(device_id=None, user_id=-1):
try:
msg = list()
redis_key_list = list()
if device_id:
redis_key_list.extend(('physical:linucb:device_id:%s', 'ctr_physical:linucb:device_id:%s',
'physical:linucb:tag_recommend:device_id:%s', 'ctr_physical:linucb:tag_recommend:device_id:%s',
'physical:linucb:topic_recommend:device_id:%s', 'ctr_physical:linucb:topic_recommend:device_id:%s',
'physical:linucb:pictorial_recommend:device_id:%s', 'ctr_physical:linucb:pictorial_recommend:device_id:%s',
'physical:home_recommend:device_id:%s:query_type:1', 'physical:home_pictorial_recommend:device_id:%s:query_type:1'))
for redis_key in redis_key_list:
key = redis_key % device_id
if redis_client.type(key) != "b'none'":
redis_client.delete(key)
msg.append('device_id_clear')
if user_id != -1:
redis_user_key_list = ["physical:home_recommend:user_id:%s:query_type:1",
'physical:home_pictorial_recommend:user_id:%s:query_type:1']
for redis_key in redis_user_key_list:
key = redis_key % user_id
if redis_client.type(key) != "b'none'":
redis_client.delete(key)
msg.append('user_id_clear')
logging.info("physical/search/clear_history_for_debug:done:device_id:{}".format(device_id))
return msg
except:
return ['fail']
def get_device_linucb_tag(device_id=None, size=20, has_score=False, gray_list=[1]):
try:
user_feature = [1, 1]
if device_id:
if STRATEGY_TYPE.CTR_GRAY in gray_list:
linucb_matrix_redis_prefix = "ctr_physical:linucb:device_id:"
else:
linucb_matrix_redis_prefix = "physical:linucb:device_id:"
redis_key = linucb_matrix_redis_prefix + str(device_id)
redis_linucb_tag_data_dict = redis_client.hgetall(redis_key)
if len(redis_linucb_tag_data_dict) == 0:
return {"linucb": []}
else:
(recommend_tag_dict, recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,
redis_linucb_tag_data_dict,
user_feature, list(
redis_linucb_tag_data_dict.keys()))
recommend_tag_list = list(recommend_tag_dict.keys())
linucb_tag_dict_list = list()
linucb_tag_dict = dict()
part_recommend_tag_list = recommend_tag_list[0:size]
tag_query_results = Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
id__in=part_recommend_tag_list, is_online=True, is_deleted=False).values_list("id", "name")
if has_score:
for id, name in tag_query_results:
linucb_tag_dict.update({id: {"name": name, "score": recommend_tag_dict[str(id)]}})
return linucb_tag_dict
else:
for id, name in tag_query_results:
linucb_tag_dict_list.append({"id": id, "name": name})
return linucb_tag_dict_list
else:
return {"linucb": []}
except:
return {"linucb": []}
def get_edit_tag_name(tag_lst):
try:
tag_query_results = Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
id__in=tag_lst, is_online=True, is_deleted=False).values_list("id", "name")
tag_dict = dict()
for id, name in tag_query_results:
tag_dict.update({id: name})
return tag_dict
except:
return dict()
@bind("physical/search/device_linucb_tag_debug")
def get_device_linucb_tag2(device_id=None, size=20, has_score=False, gray_list=[1]):
try:
user_feature = [1, 1]
if device_id:
if STRATEGY_TYPE.CTR_GRAY in gray_list:
linucb_matrix_redis_prefix = "ctr_physical:linucb:device_id:"
else:
linucb_matrix_redis_prefix = "physical:linucb:device_id:"
redis_key = linucb_matrix_redis_prefix + str(device_id)
redis_linucb_tag_data_dict = redis_client.hgetall(redis_key)
if len(redis_linucb_tag_data_dict) == 0:
return {"linucb": []}
else:
(recommend_tag_dict, recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,
redis_linucb_tag_data_dict,
user_feature, list(
redis_linucb_tag_data_dict.keys()))
recommend_tag_list = list(recommend_tag_dict.keys())
linucb_tag_dict_list = list()
linucb_tag_dict = dict()
part_recommend_tag_list = recommend_tag_list[0:size]
tag_query_results = Tag.objects.using(settings.SLAVE1_DB_NAME).filter(
id__in=part_recommend_tag_list, is_online=True, is_deleted=False).values_list("id", "name")
if has_score:
result_lst = []
for id, name in tag_query_results:
result_lst.append({"name": name, "score": recommend_tag_dict[str(id)]})
linucb_tag_dict = {"linucb": result_lst}
logging.info("physical/search/device_linucb_tag_debug:%s" % str(linucb_tag_dict))
return linucb_tag_dict
else:
for id, name in tag_query_results:
linucb_tag_dict_list.append({"id": id, "name": name})
return linucb_tag_dict_list
else:
return {"linucb": []}
except:
return {"linucb": []}