Commit e295a118 authored by 段英荣's avatar 段英荣

Merge branch 'duanyr' into 'master'

Duanyr

See merge request !68
parents cf40d59c 1a38d3ac
......@@ -167,7 +167,7 @@ class ESPerform(object):
@classmethod
def get_search_results(cls, es_cli,sub_index_name,query_body,offset=0,size=10,
auto_create_index=False,doc_type="_doc",aggregations_query=False,is_suggest_request=False):
auto_create_index=False,doc_type="_doc",aggregations_query=False,is_suggest_request=False,batch_search=False):
try:
assert (es_cli is not None)
......@@ -182,18 +182,25 @@ class ESPerform(object):
cls.put_index_mapping(es_cli,sub_index_name)
logging.info("duan add,query_body:%s" % str(query_body).encode("utf-8"))
res = es_cli.search(index=official_index_name,doc_type=doc_type,body=query_body,from_=offset,size=size)
if is_suggest_request:
return res
if not batch_search:
res = es_cli.search(index=official_index_name,doc_type=doc_type,body=query_body,from_=offset,size=size)
if is_suggest_request:
return res
else:
result_dict = {
"total_count":res["hits"]["total"],
"hits":res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
else:
result_dict = {
"total_count":res["hits"]["total"],
"hits":res["hits"]["hits"]
}
if aggregations_query:
result_dict["aggregations"] = res["aggregations"]
return result_dict
res = es_cli.msearch(body=query_body,index=official_index_name, doc_type=doc_type)
logging.info("duan add,msearch res:%s" % str(res))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]}
\ No newline at end of file
......@@ -11,6 +11,7 @@ PyMySQL==0.9.2
gunicorn==19.9.0
gevent==1.3.7
pypinyin==0.34.1
git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/helios.git@master
......@@ -19,5 +20,6 @@ git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git@v0.1.3#egg=gm-config==
git+ssh://git@git.wanmeizhensuo.com/backend/gm-protocol.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/gm-upload.git@master
git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@dev
git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git@master
......@@ -26,7 +26,12 @@ class TopicDocumentField(object):
IS_ONLINE = "is_online",
TAG_LIST = "tag_list"
class TopicPageType(object):
# 首页推荐
HOME_RECOMMEND=1
# 发现页面
FIND_PAGE=2
def time_consuming_decorator(func):
def time_consuming(*args, **kwargs):
......
......@@ -6,6 +6,8 @@ import traceback
import json
from libs.es import ESPerform
from .common import TopicDocumentField
from search.utils.common import *
class TopicUtils(object):
......@@ -88,7 +90,7 @@ class TopicUtils(object):
return []
@classmethod
def get_recommend_topic_ids(cls,user_id,offset,size,query=None):
def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[]):
"""
:需增加打散逻辑
:remark:获取首页推荐帖子列表
......@@ -172,18 +174,28 @@ class TopicUtils(object):
}
)
low_content_level = 4 if query_type==TopicPageType.FIND_PAGE else 3
query_function_score = {
"query": {
"bool": {
"must": {
"range": {"content_level": {"gte": 3, "lte": 5}}
}
"must": [
{"range": {"content_level": {"gte": low_content_level, "lte": 5}}},
{"term": {"has_image":True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
]
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
if query_type==TopicPageType.FIND_PAGE and len(filter_topic_id_list)>0:
query_function_score["query"]["bool"]["must_not"] = {
"terms":{
"id": filter_topic_id_list
}
}
if query is not None:#搜索帖子
multi_fields = {
......@@ -232,7 +244,7 @@ class TopicUtils(object):
return []
@classmethod
def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,offset,size):
def get_topic_detail_recommend_list(cls,user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -275,9 +287,9 @@ class TopicUtils(object):
query_function_score = {
"query":{
"bool":{
"must": {
"range": {"content_level": {"gte": 3, "lte": 5}}
},
"must": [
{"range": {"content_level": {"gte": 3, "lte": 5}}}
],
"must_not":{
"term":{
"id":topic_id
......@@ -290,6 +302,8 @@ class TopicUtils(object):
"functions": functions_list
}
if filter_topic_user_id:
query_function_score["query"]["bool"]["must"].append({"term": {"user_id": topic_user_id}})
if len(topic_tag_list)>0:
query_function_score["query"]["bool"]["should"]={
"terms":{
......
......@@ -5,10 +5,57 @@ import logging
import traceback
import json
import time
import copy
from libs.es import ESPerform
class UserUtils(object):
@classmethod
def get_batch_attention_user_list(cls,user_id_list,self_user_id):
"""
:remark 批量用户 关注的 用户列表
:param user_id_list:
:param self_user_id:
:return:
"""
try:
user_id_list.append(self_user_id)
q = dict()
q["query"] = {
"terms":{
"user_id":user_id_list
}
}
q["_source"] = {
"include":["attention_user_id_list","user_id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset=0, size=len(user_id_list))
self_attention_user_id_list = []
attention_user_dict_list = list()
ret_attention_user_id_list = list()
for hit_item in result_dict["hits"]:
attention_user_dict = dict()
user_id = hit_item["_source"]["user_id"]
if user_id == self_user_id:
self_attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
else:
attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
attention_user_dict[user_id] = attention_user_id_list
attention_user_dict_list.append(attention_user_dict)
ret_attention_user_id_list.append(user_id)
return [self_attention_user_id_list,attention_user_dict_list,ret_attention_user_id_list]
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([],[],[])
@classmethod
def get_attention_user_list(cls,user_id_list,self_user_id):
"""
......@@ -130,4 +177,107 @@ class UserUtils(object):
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
\ No newline at end of file
return []
@classmethod
def get_batch_recommend_user_dict(cls,need_filter_attention_user_id_list,attention_user_id_list,attention_user_dict_list,self_user_id,offset,size):
"""
:remark 获取批量推荐用户
:param need_filter_attention_user_id_list:
:param attention_user_dict:
:param self_user_id:
:param offset:
:param size:
:return:
"""
try:
batch_query_list = list()
for interesting_user_item_dict in attention_user_dict_list:
for interesting_user_id in interesting_user_item_dict:
if interesting_user_id != self_user_id:
logging.info("duan add,interesting_user_id:%d" % interesting_user_id)
filter_user_id_list = copy.deepcopy(need_filter_attention_user_id_list)
filter_user_id_list.append(interesting_user_id)
q = dict()
q["from"] = offset
q["size"] = size
q["query"] = dict()
functions_list = [
{
"gauss": {
"latest_topic_time_val": {
"origin": int(time.time()),
"scale": "600",
"decay": 0.1
}
}
}
]
if len(interesting_user_item_dict[interesting_user_id]) > 0:
functions_list.append(
{
"filter":{
"bool":{
"should":{"terms":{"user_id":interesting_user_item_dict[interesting_user_id]}}
}
},
"weight":10
}
)
query_function_score = {
"query": {
"bool": {
"must": [
{"term": {"is_recommend": True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"is_shadow": False}}
],
"must_not":{
"terms":{
"user_id":filter_user_id_list
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
q["query"]["function_score"] = query_function_score
q["_source"] = {
"include":["user_id"]
}
batch_query_list.append(q)
index_name = ESPerform.get_official_index_name("user","read")
search_header_dict = {'index': index_name, 'type': "_doc"}
query_body = ""
for query_item in batch_query_list:
query_body += "{}\n{}\n".format(json.dumps(search_header_dict),json.dumps(query_item))
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="user", query_body=query_body,
batch_search=True)
ret_dict = dict()
user_index = 0
for res_item in result_dict["responses"]:
recommend_user_list = list()
for item in res_item["hits"]["hits"]:
recommend_user_list.append(item["_source"]["user_id"])
logging.info("duan add,attention_user_id_list:%s,user_index:%d" % (str(attention_user_id_list), user_index))
ret_dict[str(attention_user_id_list[user_index])] = recommend_user_list
user_index += 1
logging.info("duan add,ret_dict:%s" % str(ret_dict))
return ret_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
......@@ -12,12 +12,47 @@ from libs.cache import redis_client
from search.utils.common import *
def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
def get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE):
try:
if user_id == -1:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id) + ":device_id:" + device_id
else:
redis_key = "physical:discover_page" + ":user_id:" + str(user_id)
redis_field_list = [b'have_read_topic_id']
redis_field_val_list = redis_client.hmget(redis_key,redis_field_list)
have_read_topic_id_list = json.loads(redis_field_val_list[0]) if redis_field_val_list[0] else []
recommend_topic_ids = []
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, offset=0, size=size,query_type=query_type,filter_topic_id_list=have_read_topic_id_list)
for item in topic_id_list:
recommend_topic_ids.append(item["id"])
have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = {
"have_read_topic_id": json.dumps(have_read_topic_id_list)
}
redis_client.hmset(redis_key,redis_dict)
return recommend_topic_ids
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_type=TopicPageType.HOME_RECOMMEND):
try:
if query is None:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":session_id:" + session_id
if user_id == -1:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":session_id:" + session_id + ":query:" + str(query)
if user_id == -1:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
redis_field_list = [b'last_offset_num', b'unread_topic_id']
for page_id in range(0,offset,size):
......@@ -27,7 +62,7 @@ def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
last_offset_num = int(redis_field_val_list[0]) if redis_field_val_list[0] else -1
recommend_topic_ids = []
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query)
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query,query_type=query_type)
have_read_group_id_set = set()
unread_topic_id_list = list()
have_read_topic_id_set = set()
......@@ -90,9 +125,32 @@ def get_home_recommend_topic_ids(user_id,session_id,offset,size,query=None):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@bind("physical/search/home_recommend")
def home_recommend(session_id="",user_id=-1,offset=0,size=10):
def home_recommend(device_id="",user_id=-1,offset=0,size=10,query_type=TopicPageType.HOME_RECOMMEND):
"""
:remark:首页推荐,目前只推荐日记
:param session_id:
:param user_id:
:param offset:
:param size:
:return:
"""
try:
if not user_id:
user_id=-1
if not isinstance(device_id,str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query_type=query_type)
return {"recommend_topic_ids":recommend_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
@bind("physical/search/discover_page")
def discover_page(device_id="",user_id=-1,size=10):
"""
:remark:首页推荐,目前只推荐日记
:param session_id:
......@@ -104,10 +162,10 @@ def home_recommend(session_id="",user_id=-1,offset=0,size=10):
try:
if not user_id:
user_id=-1
if not isinstance(session_id,str):
session_id = ""
if not isinstance(device_id,str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,session_id,offset,size)
recommend_topic_ids = get_discover_page_topic_ids(user_id,device_id,size,query_type=TopicPageType.FIND_PAGE)
return {"recommend_topic_ids":recommend_topic_ids}
except:
......@@ -116,7 +174,7 @@ def home_recommend(session_id="",user_id=-1,offset=0,size=10):
@bind("physical/search/home_query")
def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
def home_query(device_id="",user_id=-1,query="",offset=0,size=10):
"""
:remark:首页搜索,目前只推荐日记
:param session_id:
......@@ -129,10 +187,10 @@ def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
try:
if not user_id:
user_id=-1
if not isinstance(session_id,str):
session_id = ""
if not isinstance(device_id,str):
device_id = ""
recommend_topic_ids = get_home_recommend_topic_ids(user_id,session_id,offset,size,query)
recommend_topic_ids = get_home_recommend_topic_ids(user_id,device_id,offset,size,query)
return {"recommend_topic_ids":recommend_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -140,7 +198,7 @@ def home_query(session_id="",user_id=-1,query="",offset=0,size=10):
@bind("physical/search/topic_detail_page_recommend")
def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,offset=0,size=10):
def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,filter_topic_user_id=False,offset=0,size=10):
"""
:remark:帖子详情页推荐策略,缺少第一个卡片策略
:param user_id:
......@@ -156,7 +214,7 @@ def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_u
# 获取帖子标签列表
topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id)
result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,offset,size)
result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_id,topic_tag_list,topic_group_id,topic_user_id,filter_topic_user_id,offset,size)
recommend_topic_ids_list = list()
if len(result_list)>0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
......
......@@ -38,4 +38,33 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_user_ids":[]}
\ No newline at end of file
return []
@bind("physical/search/batch_recommend_user")
def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10):
"""
:remark 点关注推荐用户 - 批量接口
:param self_user_id:
:param interesting_user_id:
:param offset:
:param size:
:return:
"""
try:
if not isinstance(self_user_id,int):
self_user_id = -1
#获取关注用户列表
(need_filter_attention_user_id_list, attention_user_dict_list,attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list,self_user_id)
#去除自身及感兴趣的用户ID
need_filter_attention_user_id_list.append(self_user_id)
recommend_user_dict = UserUtils.get_batch_recommend_user_dict(need_filter_attention_user_id_list=need_filter_attention_user_id_list,attention_user_id_list=attention_user_id_list,attention_user_dict_list=attention_user_dict_list,self_user_id=self_user_id,offset=offset,size=size)
logging.info("duan add,recommend_user_dict:%s" % str(recommend_user_dict))
return recommend_user_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {}
\ No newline at end of file
......@@ -18,6 +18,7 @@
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
......
......@@ -27,32 +27,58 @@ class ActionSumAboutTopic(models.Model):
data_type = models.IntegerField(verbose_name=u"动作类型")
data_value = models.BigIntegerField(verbose_name=u"值")
class TopicImage(models.Model):
class Meta:
verbose_name = u'日记图片'
db_table = 'topic_image'
id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
topic_id = models.IntegerField(verbose_name=u'日记ID')
url = models.CharField(verbose_name=u'图片URL',max_length=300)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class Topic(models.Model):
class Meta:
verbose_name = u'日记'
db_table = 'topic'
id = models.IntegerField(verbose_name='日记ID',primary_key=True)
name = models.CharField(verbose_name='日记名称',max_length=100)
id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
name = models.CharField(verbose_name=u'日记名称',max_length=100)
#group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
group = models.ForeignKey(
Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
user_id = models.IntegerField(verbose_name='用户ID')
user_id = models.IntegerField(verbose_name=u'用户ID')
drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
description = models.CharField(verbose_name='日记本描述',max_length=200)
content = models.CharField(verbose_name='日记本内容',max_length=1000)
description = models.CharField(verbose_name=u'日记本描述',max_length=200)
content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
share_num = models.IntegerField(verbose_name='')
vote_num = models.IntegerField(verbose_name='点赞数')
reply_num = models.IntegerField(verbose_name='回复数')
vote_num = models.IntegerField(verbose_name=u'点赞数')
reply_num = models.IntegerField(verbose_name=u'回复数')
cover = models.CharField(verbose_name='',max_length=200)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
content_level = models.CharField(verbose_name='内容等级',max_length=3)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
def topic_has_image(self):
try:
has_image = False
query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
if len(query_list)>0:
has_image=True
return has_image
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
def get_pick_id_info(self):
try:
pick_id_list = list()
......
......@@ -38,6 +38,7 @@ class TopicTransfer(object):
res["offline_score"] = instance.get_topic_offline_score()
res["manual_score"] = instance.drop_score
res["has_image"] = instance.topic_has_image()
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment