Skip to content

P
physical
Commit 49a7bf9d authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'revert-a3b0c854' into 'master' 

Revert "Revert "Merge branch 'test' into 'master'""

See merge request alpha/physical!353
parents d3d9138e 27b037ea
Hide whitespace changes
Inline Side-by-side
Showing with 279 additions and 158 deletions
injection/data_sync/tasks.py
View file @ 49a7bf9d
......@@ -60,12 +60,21 @@ def sync_face_similar_data_to_redis():
item_list = list()
for item in similar_result_items:
weight_score = int(item.similarity * 100)
item_list.append(
{
"contrast_user_id": item.contrast_user_id,
"similarity": item.similarity
"filter":{
"constant_score":{
"filter":{
"term":{"user_id": item.contrast_user_id}
}
}
},
"weight": weight_score*2
}
)
if len(item_list)>=100:
break
redis_client.set(redis_key, json.dumps(item_list))
logging.info("participant_user_id:%d set data done!" % participant_user_id)
......
search/utils/group.py
View file @ 49a7bf9d
......@@ -47,7 +47,7 @@ class GroupUtils(object):
return {"total_count": 0, "hits": []}
@classmethod
def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None,attention_tag_list=[]):
def get_hot_pictorial_recommend_result_list(cls, offset, size, es_cli_obj=None, attention_tag_list=[]):
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
......@@ -55,10 +55,10 @@ class GroupUtils(object):
functions_list = list()
for tag_id in attention_tag_list:
functions_list.append({
"filter":{
"constant_score":{
"filter":{
"term":{
"filter": {
"constant_score": {
"filter": {
"term": {
"tag_id": tag_id
}
}
......@@ -67,7 +67,7 @@ class GroupUtils(object):
"weight": 20
})
if len(functions_list)>=20:
if len(functions_list) >= 20:
break
functions_list.append(
......@@ -82,17 +82,17 @@ class GroupUtils(object):
}
)
q = {
"query":{
"function_score":{
"query":{
"bool":{
"filter":[
"query": {
"function_score": {
"query": {
"bool": {
"filter": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"effective": True}}
],
"must_not":[
{"term":{"is_default":1}}
"must_not": [
{"term": {"is_default": 1}}
]
}
},
......@@ -116,16 +116,16 @@ class GroupUtils(object):
}
},
{
"_score":{
"_score": {
"order": "desc"
}
}
]
q["_source"] = {
"includes":["id"]
"includes": ["id"]
}
result_dict = ESPerform.get_search_results(es_cli_obj,"pictorial",q,offset,size)
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
pictorial_ids_list = []
if len(result_dict["hits"]) > 0:
......@@ -153,7 +153,7 @@ class GroupUtils(object):
{"term": {"is_online": True}},
{"term": {"user_id": user_id}},
{"term": {"is_deleted": False}},
{"term":{"effective":True}}
{"term": {"effective": True}}
]
}
}
......@@ -171,7 +171,6 @@ class GroupUtils(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_pictorial_ids_by_aggs(cls, pictorial_ids_list, es_cli_obj=None):
"""
......
search/utils/topic.py
View file @ 49a7bf9d
......@@ -11,6 +11,9 @@ from libs.es import ESPerform
from .common import TopicDocumentField
from search.utils.common import *
from trans2es.models.pictorial import PictorialTopics
from libs.cache import redis_client
class TopicUtils(object):
......@@ -32,7 +35,7 @@ class TopicUtils(object):
}
q["_source"] = {
"include": ["tag_list", "attention_user_id_list", "pick_user_id_list", "same_pictorial_user_id_list"]
"include": ["tag_list","attention_user_id_list", "pick_user_id_list", "same_pictorial_user_id_list"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset, size)
......@@ -120,9 +123,11 @@ class TopicUtils(object):
return {}
@classmethod
def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,single_size,query=None,query_type=TopicPageType.HOME_RECOMMEND,
filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[],linucb_user_id_list = [],disable_collpase=False):
def get_recommend_topic_ids(cls, user_id, tag_id, offset, size, single_size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,
filter_topic_id_list=[], test_score=False, must_topic_id_list=[], recommend_tag_list=[],
user_similar_score_list=[], index_type="topic", routing=None, attention_tag_list=[],
linucb_user_id_list=[], disable_collpase=False):
"""
:remark:获取首页推荐帖子列表
:param user_id:
......@@ -136,14 +141,14 @@ class TopicUtils(object):
# pick_user_id_list = list()
# same_group_id_list = list()
user_tag_list = list()
if filter_topic_id_list == None:
filter_topic_id_list = []
result_dict = TopicUtils.get_related_user_info(user_id, 0, 1)
if len(result_dict["hits"]) == 0:
logging.warning("not find user_id:%d in es!" % int(user_id))
else:
attention_user_info_list = result_dict["hits"][0]["_source"]["attention_user_id_list"]
attention_user_id_list = [item["user_id"] for item in attention_user_info_list]
# pick_user_info_list = result_dict["hits"][0]["_source"]["pick_user_id_list"]
# pick_user_id_list = [item["user_id"] for item in pick_user_info_list]
......@@ -158,14 +163,6 @@ class TopicUtils(object):
q["query"] = dict()
functions_list = [
# {
# "filter": {
# "term": {
# "language_type": 1
# }
# },
# "weight": 60
# },
{
"gauss": {
"create_time": {
......@@ -174,38 +171,22 @@ class TopicUtils(object):
}
},
"weight": 60
},
# {
# "filter": {
# "constant_score":{
# "filter":{
# "term": {
# "content_level": 6
# }
# }
# }
# },
# "weight": 600
# }
}
]
# if len(user_similar_score_list) > 0:
# for item in user_similar_score_list[:100]:
# score_item = 2 + item[1]
# functions_list.append(
# {
# "filter": {"bool": {
# "should": {"term": {"user_id": item[0]}}}},
# "weight": score_item,
# }
# )
if user_id and user_id>0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list)>0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 30,
"filter": {"constant_score":{"filter":{"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
if len(attention_tag_list) > 0:
......@@ -216,13 +197,10 @@ class TopicUtils(object):
"weight": 100
}
)
query_function_score = {
"query": {
"bool": {
"filter": [
# {"term": {"content_level": 6}},
# {"term": {"has_image":True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
......@@ -244,7 +222,7 @@ class TopicUtils(object):
],
"minimum_should_match": 1,
"must_not": [
{"term": {"is_history":True}}
{"term": {"is_history": True}}
]
}
},
......@@ -260,10 +238,10 @@ class TopicUtils(object):
}
if len(filter_topic_id_list) > 0:
query_function_score["query"]["bool"]["must_not"] = [
{"terms":{"id":filter_topic_id_list}}
{"terms": {"id": filter_topic_id_list}}
]
if len(linucb_user_id_list)>0:
if len(linucb_user_id_list) > 0:
if "must_not" in query_function_score["query"]["bool"]:
query_function_score["query"]["bool"]["must_not"] += [
{"terms": {"user_id": linucb_user_id_list}}
......@@ -294,7 +272,7 @@ class TopicUtils(object):
]
query_function_score["query"]["bool"]["minimum_should_match"] = 1
query_function_score["query"]["bool"]["filter"].append(
{"range": {"content_level": {"gte":4,"lte":6}}}
{"range": {"content_level": {"gte":3,"lte":6}}}
)
else:
if "must_not" in query_function_score["query"]["bool"]:
......@@ -318,7 +296,6 @@ class TopicUtils(object):
q["collapse"] = {
"field": "user_id"
}
# "includes": ["id", "pictorial_id", "offline_score", "user_id", "edit_tag_list"]
q["_source"] = {
"includes": ["id"]
}
......@@ -347,7 +324,7 @@ class TopicUtils(object):
}
]
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size,routing=routing)
offset=offset, size=size, routing=routing)
topic_id_list = list()
......@@ -358,9 +335,78 @@ class TopicUtils(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
@classmethod
def userful_tag_topic_list(cls, user_id, have_read_topic_list, size,
index_type="topic-high-star",routing=None):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
:param topic_tag_list:
:param topic_group_id:
:param topic_user_id:
:param offset:
:param size:
:return:
"""
try:
es_cli_obj = ESPerform.get_cli()
useful_tag_list = list()
q = dict()
q["query"] = {
"term": {
"id": user_id
}
}
q["_source"] = {
"include": ["useful_tag_list"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, 0, 1)
if len(result_dict["hits"]) == 0:
logging.warning("not find user_id:%d in es!" % int(user_id))
else:
useful_tag_list = result_dict["hits"][0]["_source"]["useful_tag_list"]
if len(useful_tag_list) == 0:
return []
else:
q = dict()
q["query"] = dict()
# logging.warning("topic_tag_list:%s"%str(topic_tag_list))
query_function_score = {
"query": {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": useful_tag_list}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
}
}
}
}
}
q["_source"] = {
"includes": ["id"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size,
routing=routing)
topic_id_list = list()
for item in result_dict["hits"]:
topic_id_list.append(item["_source"]["id"])
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_topic_detail_recommend_list(cls, user_id, topic_id, topic_tag_list, topic_pictorial_id, topic_user_id,
filter_topic_user_id, have_read_topic_list, offset, size, es_cli_obj=None,index_type="topic",routing=None):
filter_topic_user_id, have_read_topic_list, offset, size, es_cli_obj=None,
index_type="topic", routing=None):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -437,7 +483,7 @@ class TopicUtils(object):
}
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q,
offset=offset, size=size,routing=routing)
offset=offset, size=size, routing=routing)
return result_dict["hits"]
except:
......@@ -445,9 +491,10 @@ class TopicUtils(object):
return []
@classmethod
def top_get_topic_detail_recommend_list(cls, user_id, topic_id,have_read_topic_list, size, es_cli_obj=None,
index_type="topic", routing=None,collection_topic_tag_list = [],topic_tag_list = [],
topic_user_id =-1):
def top_get_topic_detail_recommend_list(cls, user_id, topic_id, have_read_topic_list, size, es_cli_obj=None,
index_type="topic", routing=None, collection_topic_tag_list=[],
topic_tag_list=[],
topic_user_id=-1):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -491,26 +538,26 @@ class TopicUtils(object):
# "weight": 5000
# }
# )
if len(topic_tag_list) != 0 or topic_user_id!= -1:
query_function_score = {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 3, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
if len(topic_tag_list) != 0 or topic_user_id != -1:
query_function_score = {
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 3, "lte": 6}}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
}
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
else:
query_function_score = {
"query": {
......@@ -546,9 +593,9 @@ class TopicUtils(object):
}
q["query"]["function_score"] = query_function_score
if topic_user_id == -1:
q["collapse"] = {
"field": "user_id"
}
q["collapse"] = {
"field": "user_id"
}
q["_source"] = {
"includes": ["id", "pictorial_id", "user_id", "_score"]
}
......@@ -559,7 +606,8 @@ class TopicUtils(object):
# {"create_time": {"order": "desc"}}
# ]
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size, routing=routing)
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size,
routing=routing)
topic_id_list = list()
for item in result_dict["hits"]:
......@@ -702,7 +750,7 @@ class TopicUtils(object):
{"term": {"is_deleted": False}},
]
logging.info("get filters:%s"%filters)
logging.info("get filters:%s" % filters)
if not filters:
return f
......@@ -892,21 +940,23 @@ class TopicUtils(object):
query_body=q, offset=offset, size=size
)
if len(result_dict["hits"])>0:
if len(result_dict["hits"]) > 0:
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
return (topic_id_list,result_dict["total_count"])
elif offset==0 and "pictorial_id" in filters: # 防止帖子同步延迟，画报详情页为空
return (topic_id_list, result_dict["total_count"])
elif offset == 0 and "pictorial_id" in filters: # 防止帖子同步延迟，画报详情页为空
pictorial_id = int(filters["pictorial_id"])
topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=pictorial_id,is_online=True,is_deleted=False).values_list("topic_id", flat=True)[offset:size])
topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=pictorial_id, is_online=True,
is_deleted=False).values_list("topic_id",
flat=True)[
offset:size])
return (topic_id_list,len(topic_id_list))
return (topic_id_list, len(topic_id_list))
else:
return ([], 0)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([],0)
return ([], 0)
@classmethod
def business_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
......@@ -994,7 +1044,6 @@ class TopicUtils(object):
}
}
})
elif k == "drop_score":
if v == "0":
f.append({
......@@ -1009,7 +1058,6 @@ class TopicUtils(object):
}
}
})
else:
if isinstance(v, list):
......
search/views/group.py
View file @ 49a7bf9d
......@@ -10,6 +10,7 @@ from libs.es import ESPerform
from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics
@bind("physical/search/query_pictorial")
......@@ -54,11 +55,12 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
if sort_type == GroupSortTypes.HOT_RECOMMEND:
# 用户关注标签
attention_tag_list = list()
if user_id>0:
if user_id > 0:
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj,attention_tag_list)
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj,
attention_tag_list)
return {"pictorial_recommend_ids": pictorial_ids_list}
elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND:
......@@ -173,6 +175,13 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"term": {
"is_deleted": False
}
}, {
"range": {
"topic_id_list": {
"gte": 0
}
}
}]
}
}
......@@ -209,7 +218,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
@bind("physical/search/pictorial_topic_sort")
def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
"""
:remark 小组排序,缺少：前1天发评论人数＊x
:remark 画报排序 人气 部分
:param user_id:
:param sort_type:
:param offset:
......@@ -221,33 +230,33 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
user_id = -1
q = {
"query":{
"bool":{
"must":[
{"term":{"pictorial_id": pictorial_id}},
"query": {
"bool": {
"must": [
{"term": {"pictorial_id": pictorial_id}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
]
}
},
"sort":[
"sort": [
{"total_vote_num": {"order": "desc"}},
{"create_time": {"order": "desc"}}
]
}
pict_pictorial_ids_list =[]
pict_pictorial_ids_list = []
# 获取es链接对象
es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size)
logging.info("get pictorial_topic_sort res:%s"%result_dict)
logging.info("get pictorial_topic_sort res:%s" % result_dict)
for item in result_dict["hits"]:
topic_id = item["_source"]["id"]
pict_pictorial_ids_list.append(topic_id)
return {"pict_pictorial_ids_list":pict_pictorial_ids_list}
return {"pict_pictorial_ids_list": pict_pictorial_ids_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
search/views/topic.py
View file @ 49a7bf9d
......@@ -27,6 +27,8 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else []
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size,
query_type=query_type,
......@@ -46,16 +48,18 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False):
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1):
try:
topic_star_routing = "6"
index_type = "topic-high-star"
if query is None:
if user_id>0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
topic_star_routing = "4,5,6"
topic_star_routing = "3,4,5,6"
index_type = "topic"
if user_id>0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
else:
......@@ -74,37 +78,54 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
else:
if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list)
useful_topic_id_list = list()
recommend_topic_user_list = list()
attention_tag_list = list()
recommend_topic_list = list()
if query is None:
if user_id != -1:
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6")
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list()
recommend_topic_list = list()
if b"data" in recommend_topic_dict:
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
if linucb_recommend_topic_id_list == None:
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 6:
recommend_topic_list = recommend_topic_id_list[0:6]
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if len(recommend_topic_list) == 6 and linucb_recommend_topic_id_dict is not None:
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
# have_read_topic_id_list = list()
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
......@@ -119,12 +140,12 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
have_read_topic_id_list.extend(promote_topic_list)
topic_id_list = list()
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type="topic-high-star",routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
rank_topic_id_list = list()
if size>0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None:
# if (size < 11):
......@@ -139,7 +160,6 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(recommend_topic_list[3:6])
# else:
# topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list)-30000
......@@ -150,11 +170,16 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
redis_client.hmset(redis_key, redis_dict)
# 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60)
return recommend_topic_list,rank_topic_id_list
if usefulrecall != -1:
return recommend_topic_list,rank_topic_id_list,useful_topic_id_list
else:
return recommend_topic_list, rank_topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [],[]
if usefulrecall != -1:
return [],[],[]
else:
return [], []
@bind("physical/search/query_tag_id_by_topic")
......@@ -167,7 +192,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[]):
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
......@@ -184,6 +209,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
recommend_topic_ids = list()
rank_topic_ids = list()
useful_topic_ids = list()
es_node_load_high_flag = False
# try:
# es_node_load_high_flag = ESPerform.if_es_node_load_high(ESPerform.get_cli())
......@@ -204,13 +230,21 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
recommend_topic_ids = have_read_topic_id_list[0:size]
else:
if usefulrecall != -1:
recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids}
else:
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"linucb_topic_ids": [],"rank_topic_ids":[]}
if usefulrecall != -1:
return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]}
else:
return {"linucb_topic_ids": [], "rank_topic_ids": []}
@bind("physical/search/discover_page")
......
trans2es/mapping/topic-high-star.json
View file @ 49a7bf9d
......@@ -17,6 +17,7 @@
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
......
trans2es/mapping/topic-star-routing.json
View file @ 49a7bf9d
......@@ -16,6 +16,7 @@
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"useful_tag_list":{"type":"long"},//有用标签属性
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
......
trans2es/mapping/topic.json
View file @ 49a7bf9d
......@@ -17,6 +17,7 @@
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
......
trans2es/mapping/user.json
View file @ 49a7bf9d
......@@ -9,6 +9,7 @@
"is_online":{"type":"boolean"},//是否上线
"is_deleted":{"type":"boolean"},
"tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性
"city_id":{"type":"text"},
"country_id":{"type":"text"},
"is_recommend":{"type":"boolean"},//是否运营推荐用户
......
trans2es/models/tag.py
View file @ 49a7bf9d
......@@ -20,6 +20,7 @@ class TopicTag(models.Model):
topic_id = models.IntegerField(verbose_name=u"帖子ID")
is_online = models.BooleanField(verbose_name=u"是否在线")
is_collection = models.IntegerField(verbose_name=u"是否编辑标签")
is_body_esthetics = models.IntegerField(verbose_name=u"是否有用标签")
class AccountUserTag(models.Model):
......@@ -33,6 +34,7 @@ class AccountUserTag(models.Model):
is_deleted = models.BooleanField(verbose_name=u"是否删除")
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_body_esthetics = models.IntegerField(verbose_name=u"是否有用标签")
class Tag(models.Model):
......
trans2es/models/topic.py
View file @ 49a7bf9d
......@@ -191,6 +191,18 @@ class Topic(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([],[])
def get_topic_useful_tag_id_list(self):
try:
topic_useful_tag_id_list = list()
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id,is_online =True,is_body_esthetics = 1)
for tag_item in tag_id_list:
topic_useful_tag_id_list.append(tag_item.tag_id)
return topic_useful_tag_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_tag_name_list(self, tag_id_list):
try:
tag_name_list = list()
......@@ -227,10 +239,6 @@ class Topic(models.Model):
elif user_query_results[0].is_shadow:
user_is_shadow = True
# 是否官方推荐小组
# if self.group and self.group.is_recommend:
# offline_score += 4.0
# 帖子等级
if self.content_level == '5':
offline_score += 100.0 *3
......@@ -239,20 +247,8 @@ class Topic(models.Model):
elif self.content_level == '6':
offline_score += 200.0 *3
# is_excellent = self.judge_if_excellent_topic(self.id)
# if is_excellent:
# offline_score += 200.0
if self.language_type == 1:
offline_score += 60.0
# exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
# click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
# uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
#
# if exposure_count > 0:
# offline_score += click_count / exposure_count
# if uv_num > 0:
# offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
"""
1:马甲账号是否对总分降权？
......
trans2es/models/user.py
View file @ 49a7bf9d
......@@ -189,3 +189,16 @@ class User(models.Model):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_user_useful_tag_id_list(self):
try:
user_useful_tag_id_list = list()
query_results = AccountUserTag.objects.using(settings.SLAVE_DB_NAME).filter(user=self.user_id,is_body_esthetics =1, is_deleted=False)
for item in query_results:
user_useful_tag_id_list.append(item.tag_id)
return user_useful_tag_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
\ No newline at end of file
trans2es/utils/topic_transfer.py
View file @ 49a7bf9d
......@@ -75,6 +75,11 @@ class TopicTransfer(object):
end = time.time()
time4 = (end-begin)
# begin = time.time()
topic_useful_tag_id_list= instance.get_topic_useful_tag_id_list()
res["useful_tag_list"] = topic_useful_tag_id_list
# end = time.time()
# time5 = (end - begin)
# # 片假名
# re_jp_pian_words = re.compile(u"[\u30a0-\u30ff]+")
# m_pian = re_jp_pian_words.search(instance.content, 0)
......
trans2es/utils/user_transfer.py
View file @ 49a7bf9d
......@@ -73,6 +73,7 @@ class UserTransfer(object):
try:
res["tag_list"] = instance.get_user_tag_id_list()
res["useful_tag_list"] = instance.get_user_useful_tag_id_list()
res["attention_user_id_list"] = cls.get_follow_user_id_list(userInstance=instance)
# res["attention_group_id_list"] = instance.get_attention_group_id_list()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment