Skip to content

P
physical
Commit 4ae7e49f authored by lixiaofang's avatar lixiaofang
Browse files
Options

修改mapping

parent c4378545
Hide whitespace changes
Inline Side-by-side
Showing with 213 additions and 159 deletions
search/utils/topic.py
View file @ 4ae7e49f
...@@ -251,9 +251,9 @@ class TopicUtils(object): ...@@ -251,9 +251,9 @@ class TopicUtils(object):
] ]
if query is not None: # 搜索帖子 if query is not None: # 搜索帖子
multi_fields = { multi_fields = {
#'description': 200, # 'description': 200,
'content': 300, 'content': 300,
#'name': 400, # 'name': 400,
'tag_name_list': 300, 'tag_name_list': 300,
} }
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()] query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
...@@ -329,18 +329,18 @@ class TopicUtils(object): ...@@ -329,18 +329,18 @@ class TopicUtils(object):
'query': query, 'query': query,
'type': 'best_fields', 'type': 'best_fields',
'operator': 'and', 'operator': 'and',
'fields': ["content","tag_name_list"], 'fields': ["content", "tag_name_list"],
} }
functions_list += [ functions_list += [
{ {
"weight": 300, "weight": 300,
"filter":{ "filter": {
"bool":{ "bool": {
"must":{ "must": {
"term": {"content_level": 6}, "term": {"content_level": 6},
}, },
"minimum_should_match": 1, "minimum_should_match": 1,
"should":[ "should": [
{'multi_match': multi_match}, {'multi_match': multi_match},
{"term": {"tag_list": tag_id}}, {"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}} {"term": {"user_nick_name_pre": query.lower()}}
...@@ -1067,6 +1067,20 @@ class TopicUtils(object): ...@@ -1067,6 +1067,20 @@ class TopicUtils(object):
} }
}, },
}) })
elif sort_by == TOPIC_SEARCH_SORT.TOPIC_ADD_TIME:
sort_rule.append({
"related_billboard.topic_add_createtime": {
"order": "desc",
"nested_path": "related_billboard",
"nested_filter": {
"term": {
"related_billboard.pictorial_id": pictorial_id
}
}
},
})
logging.info("get picotirial:%s" % sort_rule) logging.info("get picotirial:%s" % sort_rule)
return sort_rule return sort_rule
......
search/views/topic.py
View file @ 4ae7e49f
...@@ -14,6 +14,7 @@ from search.utils.common import * ...@@ -14,6 +14,7 @@ from search.utils.common import *
from libs.es import ESPerform from libs.es import ESPerform
from django.conf import settings from django.conf import settings
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE): def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try: try:
if user_id == -1: if user_id == -1:
...@@ -28,10 +29,11 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -28,10 +29,11 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size, recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,
query_type=query_type, single_size=size,
filter_topic_id_list=have_read_topic_id_list,index_type="topic",routing="4,5,6") query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids) have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = { redis_dict = {
...@@ -46,27 +48,31 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -46,27 +48,31 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None, def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1,useful_tag_list =[]): query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1, useful_tag_list=[]):
try: try:
topic_star_routing = "6" topic_star_routing = "6"
index_type = "topic-high-star" index_type = "topic-high-star"
device_redis_key = "" device_redis_key = ""
if query is None: if query is None:
if user_id>0: if user_id > 0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type) device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(
query_type)
else: else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else: else:
topic_star_routing = "3,4,5,6" topic_star_routing = "3,4,5,6"
index_type = "topic" index_type = "topic"
if user_id>0: if user_id > 0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type) redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type) query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
else: else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type) redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
...@@ -78,18 +84,17 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -78,18 +84,17 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if query is None: if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else: else:
if offset>0: # 首次搜索时不需要过滤已读 if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
elif len(device_redis_key)>0: elif len(device_redis_key) > 0:
redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list)
if redis_field_val_list[0]: if redis_field_val_list[0]:
if query is None: if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else: else:
if offset>0: # 首次搜索时不需要过滤已读 if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list) have_read_topic_id_list.extend(promote_topic_list)
...@@ -98,66 +103,72 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -98,66 +103,72 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
attention_tag_list = list() attention_tag_list = list()
recommend_topic_list = list() recommend_topic_list = list()
if query is None: if query is None:
if user_id != -1: if user_id != -1:
# 有用标签召回 # 有用标签召回
if usefulrecall != -1: if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4, useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6",useful_tag_list=useful_tag_list) "topic-high-star", "6",
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id) useful_tag_list=useful_tag_list)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key) # user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# user_similar_score_redis_list = json.loads( # redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else [] # user_similar_score_redis_list = json.loads(
size = size - len(useful_topic_id_list) # redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
have_read_topic_id_list.extend(useful_topic_id_list) size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id) # linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list() recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
recommend_topic_list = list() linucb_recommend_topic_id_list = list()
if b"data" in recommend_topic_dict: recommend_topic_list = list()
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"]) if b"data" in recommend_topic_dict:
if linucb_recommend_topic_id_list == None: linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
linucb_recommend_topic_id_list = list() if linucb_recommend_topic_id_list == None:
# 推荐帖子是强插的，要保证推荐帖子不在已读里 linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list)) logging.warning(
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index) "type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list)))
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8")) recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
# newcursor = cursor + 6 recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
if len(recommend_topic_id_list) > 0: # cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
recommend_topic_list = recommend_topic_id_list[0:size] # newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor) # redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict: if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"]) linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0: if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0:
for i in recommend_topic_list: for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)]) recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None: # if have_read_topic_id_list == None:
# have_read_topic_id_list = list() # have_read_topic_id_list = list()
# 用户关注标签 # 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id) redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else [] attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list)>0: if len(recommend_topic_list) > 0:
size = size-len(recommend_topic_list) size = size - len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list) have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list() # have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list) # have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter( # promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True) # is_online=1).values_list("topic_id",flat=True)
# #
# for topic_id in promote_recommend_topic_id_list: # for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id) # have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list() rank_topic_id_list = list()
if size>0: if size > 0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size, rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type, single_size=size, query=query,
filter_topic_id_list=have_read_topic_id_list, query_type=query_type,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase) filter_topic_id_list=have_read_topic_id_list,
index_type=index_type, routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None: # if len(recommend_topic_list) == 6 and query is None:
# if (size < 11): # if (size < 11):
...@@ -174,7 +185,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -174,7 +185,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(rank_topic_id_list) # topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list) have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000: if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list)-30000 cut_len = len(have_read_topic_id_list) - 30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:] have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = { redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list), "have_read_topic_list": json.dumps(have_read_topic_id_list),
...@@ -183,13 +194,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -183,13 +194,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 每个session key保存60天 # 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60) redis_client.expire(redis_key, 60 * 60 * 24 * 60)
if usefulrecall != -1: if usefulrecall != -1:
return recommend_topic_list,rank_topic_id_list,useful_topic_id_list return recommend_topic_list, rank_topic_id_list, useful_topic_id_list
else: else:
return recommend_topic_list, rank_topic_id_list return recommend_topic_list, rank_topic_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return [],[],[] return [], [], []
else: else:
return [], [] return [], []
...@@ -204,7 +215,8 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1): ...@@ -204,7 +215,8 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend") @bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1,useful_tag_list=[]): def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[]):
""" """
:remark:首页推荐，目前只推荐日记 :remark:首页推荐，目前只推荐日记
:param session_id: :param session_id:
...@@ -237,26 +249,34 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic ...@@ -237,26 +249,34 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset: if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset+size] recommend_topic_ids = have_read_topic_id_list[offset:offset + size]
else: else:
recommend_topic_ids = have_read_topic_id_list[0:size] recommend_topic_ids = have_read_topic_id_list[0:size]
else: else:
if usefulrecall != -1 and len(useful_tag_list)>0: if usefulrecall != -1 and len(useful_tag_list) > 0:
recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size, recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id,
query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall,useful_tag_list=useful_tag_list) tag_id=0, offset=0,
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids} size=size,
else: query_type=query_type,
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size, promote_topic_list=promote_topic_list,
query_type=query_type,promote_topic_list=promote_topic_list) usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids} return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]} return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []}
else: else:
return {"linucb_topic_ids": [], "rank_topic_ids": []} return {"linucb_topic_ids": [], "rank_topic_ids": []}
@bind("physical/search/discover_page") @bind("physical/search/discover_page")
...@@ -300,11 +320,12 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10) ...@@ -300,11 +320,12 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
if not isinstance(device_id, str): if not isinstance(device_id, str):
device_id = "" device_id = ""
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query) recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size: offset=offset, size=size, query=query)
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, offset=offset, size=size,
query=query,disable_collpase=True) query=query, disable_collpase=True)
return {"recommend_topic_ids": rank_topic_id_list} return {"recommend_topic_ids": rank_topic_id_list}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -326,9 +347,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -326,9 +347,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if not isinstance(user_id, int): if not isinstance(user_id, int):
user_id = -1 user_id = -1
if user_id > 0: if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id) redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id)
else: else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id) redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id)
if int(offset) == 0: if int(offset) == 0:
have_read_topic_list = list() have_read_topic_list = list()
# redis_dict = { # redis_dict = {
...@@ -341,7 +362,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -341,7 +362,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
have_read_topic_list = list() have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list) have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else [] have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id) have_read_topic_list.append(topic_id)
...@@ -349,28 +370,33 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -349,28 +370,33 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
topic_tag_result = list() topic_tag_result = list()
result = list() result = list()
if len(topic_tag_list) != 0: if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj, topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
topic_tag_size = 0
if topic_tag_size <size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj, size, es_cli_obj,
index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id index_type="topic", routing="3,4,5,6",
) topic_tag_list=topic_tag_list)
topic_user_size = len(topic_user_result) topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_user_result) have_read_topic_list.extend(topic_tag_result)
if topic_user_size < size: else:
size = size - topic_user_size topic_tag_size = 0
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, if topic_tag_size < size:
have_read_topic_list, size = size - topic_tag_size
size, es_cli_obj, if topic_user_id != -1:
index_type="topic", routing="4,5,6") topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list.extend(result) have_read_topic_list,
size, es_cli_obj,
index_type="topic",
routing="3,4,5,6",
topic_user_id=topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key) # have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else [] # have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
...@@ -446,8 +472,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10): ...@@ -446,8 +472,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。""" """帖子搜索。"""
try: try:
(topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters, (topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size) sorts_by=sorts_by, offset=offset, size=size)
return { return {
"topic_ids": topic_id_list, "topic_ids": topic_id_list,
...@@ -470,8 +496,10 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -470,8 +496,10 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try: try:
must_topic_id_list = list(topic_similarity_score_dict.keys()) must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size, topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,
must_topic_id_list=must_topic_id_list,index_type="topic",routing="4,5,6") single_size=size,
must_topic_id_list=must_topic_id_list, index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list} return {"recommend_topic_ids": topic_id_list}
except: except:
......
trans2es/mapping/topic-high-star.json
View file @ 4ae7e49f
{ {"dynamic":"strict",
"dynamic":"strict",
"_routing": {"required": true}, "_routing": {"required": true},
"properties": { "properties": {
"id":{"type":"long"}, "id":{"type":"long"},
...@@ -60,6 +59,7 @@ ...@@ -60,6 +59,7 @@
"type":"nested", "type":"nested",
"properties":{ "properties":{
"pictorial_id":{"type":"long"}, "pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"}, "real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}, "virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"} "total_vote_cnt":{"type":"long"}
...@@ -67,3 +67,6 @@ ...@@ -67,3 +67,6 @@
} }
} }
} }
trans2es/mapping/topic-star-routing.json
View file @ 4ae7e49f
...@@ -8,18 +8,18 @@ ...@@ -8,18 +8,18 @@
"vote_num":{"type":"long"}, "vote_num":{"type":"long"},
"total_vote_num":{"type":"long"}, "total_vote_num":{"type":"long"},
"reply_num":{"type":"long"}, "reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名 "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性 "useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"share_num":{"type":"long"}, "share_num":{"type":"long"},
"pick_id_list":{"type":"long"}, "pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分 "offline_score":{"type":"double"},//离线算分
...@@ -44,14 +44,14 @@ ...@@ -44,14 +44,14 @@
"platform": {"type": "long"}, "platform": {"type": "long"},
"platform_id": {"type": "long"}, "platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分 "drop_score":{"type": "long"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分 "sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID "pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称 "pictorial_name":{ // 所在组名称
"type": "text", "type": "text",
"analyzer": "gm_default_index", "analyzer": "keyword",
"search_analyzer": "gm_default_index" "search_analyzer": "keyword"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
...@@ -60,10 +60,14 @@ ...@@ -60,10 +60,14 @@
"type":"nested", "type":"nested",
"properties":{ "properties":{
"pictorial_id":{"type":"long"}, "pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"}, "real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}, "virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"} "total_vote_cnt":{"type":"long"}
} }
} }
} }
} }
\ No newline at end of file
trans2es/mapping/topic-star.json
View file @ 4ae7e49f
...@@ -44,7 +44,7 @@ ...@@ -44,7 +44,7 @@
"platform": {"type": "long"}, "platform": {"type": "long"},
"platform_id": {"type": "long"}, "platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分 "drop_score":{"type": "long"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分 "sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID "pictorial_id":{"type": "long"}, //所在组ID
......
trans2es/mapping/topic.json
View file @ 4ae7e49f
...@@ -8,18 +8,18 @@ ...@@ -8,18 +8,18 @@
"vote_num":{"type":"long"}, "vote_num":{"type":"long"},
"total_vote_num":{"type":"long"}, "total_vote_num":{"type":"long"},
"reply_num":{"type":"long"}, "reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名 "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性 "useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"share_num":{"type":"long"}, "share_num":{"type":"long"},
"pick_id_list":{"type":"long"}, "pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分 "offline_score":{"type":"double"},//离线算分
...@@ -44,14 +44,14 @@ ...@@ -44,14 +44,14 @@
"platform": {"type": "long"}, "platform": {"type": "long"},
"platform_id": {"type": "long"}, "platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分 "drop_score":{"type": "long"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分 "sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID "pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称 "pictorial_name":{ // 所在组名称
"type": "text", "type": "text",
"analyzer": "gm_default_index", "analyzer": "keyword",
"search_analyzer": "gm_default_index" "search_analyzer": "keyword"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
...@@ -60,6 +60,7 @@ ...@@ -60,6 +60,7 @@
"type":"nested", "type":"nested",
"properties":{ "properties":{
"pictorial_id":{"type":"long"}, "pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"}, "real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}, "virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"} "total_vote_cnt":{"type":"long"}
......
trans2es/models/topic.py
View file @ 4ae7e49f
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function from __future__ import unicode_literals, absolute_import, print_function
from django.conf import settings from django.conf import settings
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
import traceback import traceback
import logging import logging
import datetime
import time
from libs.es import ESPerform from libs.es import ESPerform
from django.db import models from django.db import models
import datetime
from alpha_types.venus import GRAP_PLATFORM from alpha_types.venus import GRAP_PLATFORM
from .pick_topic import PickTopic from .pick_topic import PickTopic
from .tag import TopicTag, Tag from .tag import TopicTag, Tag
from .user_extra import UserExtra from .user_extra import UserExtra
...@@ -203,7 +202,7 @@ class Topic(models.Model): ...@@ -203,7 +202,7 @@ class Topic(models.Model):
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id,
is_collection=1).values_list("tag_id", is_collection=1).values_list("tag_id",
flat=True) flat=True)
return list(tag_id_list) return list(tag_id_list)
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return []
...@@ -283,25 +282,29 @@ class Topic(models.Model): ...@@ -283,25 +282,29 @@ class Topic(models.Model):
def get_related_billboard(self): def get_related_billboard(self):
try: try:
pictorial_ids = Topic.get_pictorial_id(self) # pictorial_ids = Topic.get_pictorial_id(self)
pictorial_id_list = []
related_billboard_list = list() related_billboard_list = list()
for pictorial_id in pictorial_ids: pictorials = PictorialTopic.objects.filter(topic_id=self.id).values("pictorial_id", "create_time")
for pictorial_id in pictorials:
query_result = TopicBillBoard.objects.filter(pictorial_id=pictorial_id, query_result = TopicBillBoard.objects.filter(pictorial_id=pictorial_id.get('pictorial_id', None),
topic_id=self.id).values().first() topic_id=self.id).values().first()
if query_result == None: if query_result == None:
related_billboard_list.append({"pictorial_id": pictorial_id, "real_vote_cnt": 0, related_billboard_list.append({"pictorial_id": pictorial_id.get('pictorial_id', None), "real_vote_cnt": 0,
"virt_vote_cnt": 0, "total_vote_cnt": 0}) "virt_vote_cnt": 0, "total_vote_cnt": 0,
"topic_add_createtime": int(
time.mktime(pictorial_id.get("create_time", None).timetuple()))})
else: else:
total_vote_cnt = int(query_result["virt_vote_cnt"]) + int(query_result["real_vote_cnt"]) total_vote_cnt = int(query_result["virt_vote_cnt"]) + int(query_result["real_vote_cnt"])
related_billboard_list.append( related_billboard_list.append(
{"pictorial_id": query_result["pictorial_id"], "real_vote_cnt": query_result["real_vote_cnt"], {"pictorial_id": query_result["pictorial_id"], "real_vote_cnt": query_result["real_vote_cnt"],
"virt_vote_cnt": query_result["virt_vote_cnt"], "total_vote_cnt": total_vote_cnt}) "virt_vote_cnt": query_result["virt_vote_cnt"], "total_vote_cnt": total_vote_cnt,
"topic_add_createtime": int(time.mktime(pictorial_id.get("create_time", None).timetuple()))})
logging.info("product_brand_info" % related_billboard_list) logging.info("product_brand_info" % related_billboard_list)
...@@ -342,6 +345,7 @@ class PictorialTopic(models.Model): ...@@ -342,6 +345,7 @@ class PictorialTopic(models.Model):
topic_id = models.BigIntegerField(verbose_name=u'帖子ID') topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True) is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
is_deleted = models.BooleanField(verbose_name=u'是否删除') is_deleted = models.BooleanField(verbose_name=u'是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
class TopicExtra(models.Model): class TopicExtra(models.Model):
......
trans2es/utils/topic_transfer.py
View file @ 4ae7e49f
...@@ -8,6 +8,7 @@ from libs.tools import tzlc ...@@ -8,6 +8,7 @@ from libs.tools import tzlc
import time import time
import re import re
import datetime import datetime
from trans2es.models.user import User from trans2es.models.user import User
from trans2es.models.topic import ExcellentTopic, TopicHomeRecommend from trans2es.models.topic import ExcellentTopic, TopicHomeRecommend
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment