Commit cffcec45 authored by zhanglu's avatar zhanglu

Merge branch 'master' into 'test'

Master

See merge request !99
parents 9204b809 2416b4b6
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*~
# C extensions
*.so
# Distribution / packaging
.Python
.vscode
env/
bin/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
.idea/
*.egg-info/
.installed.cfg
*.egg
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Rope
.ropeproject
# Django stuff:
*.log
*.pot
# Sphinx documentation
docs/_build/
# config
fabfile.py
settings.online.py
settings_local.py
media/
log/
crawldata/
conf/
/static
.vagrant/
Vagrantfile
*.DS_Store
dump.rdb
......@@ -11,6 +11,7 @@ PyMySQL==0.9.2
gunicorn==19.9.0
gevent==1.3.7
pypinyin==0.34.1
git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/helios.git@master
......@@ -19,5 +20,6 @@ git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git@v0.1.3#egg=gm-config==
git+ssh://git@git.wanmeizhensuo.com/backend/gm-protocol.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/gm-upload.git@master
git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@dev
git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git@master
......@@ -26,7 +26,12 @@ class TopicDocumentField(object):
IS_ONLINE = "is_online",
TAG_LIST = "tag_list"
class TopicPageType(object):
# 首页推荐
HOME_RECOMMEND=1
# 发现页面
FIND_PAGE=2
def time_consuming_decorator(func):
def time_consuming(*args, **kwargs):
......
......@@ -38,7 +38,7 @@ class GroupUtils(object):
}
}
q["_source"] = {
"include":["id"]
"includes":["id"]
}
return ESPerform.get_search_results(ESPerform.get_cli(), "group", q, offset, size)
......@@ -47,8 +47,11 @@ class GroupUtils(object):
return {"total_count":0, "hits":[]}
@classmethod
def get_hot_group_recommend_result_list(cls,offset,size):
def get_hot_group_recommend_result_list(cls,offset,size,es_cli_obj=None):
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
q = dict()
q["query"] = {
"bool":{
......@@ -62,10 +65,10 @@ class GroupUtils(object):
{"high_quality_topic_num":{"order":"desc"}}
]
q["_source"] = {
"include":["id"]
"includes":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"group",q,offset,size)
result_dict = ESPerform.get_search_results(es_cli_obj,"group",q,offset,size)
group_ids_list = []
if len(result_dict["hits"]) > 0:
......@@ -77,12 +80,15 @@ class GroupUtils(object):
return []
@classmethod
def get_user_attention_group_list(cls,user_id,offset=0,size=10):
def get_user_attention_group_list(cls,user_id,offset=0,size=10,es_cli_obj=None):
"""
:remark: 获取用户关注小组列表
:return:
"""
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
q = dict()
q["query"] = {
"bool":{
......@@ -94,10 +100,10 @@ class GroupUtils(object):
}
}
q["_source"] = {
"include":["attention_group_id_list"]
"includes":["attention_group_id_list"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"user",q,offset,size)
result_dict = ESPerform.get_search_results(es_cli_obj,"user",q,offset,size)
if len(result_dict["hits"])>0:
return result_dict["hits"][0]["_source"]["attention_group_id_list"]
else:
......@@ -107,13 +113,16 @@ class GroupUtils(object):
return []
@classmethod
def get_group_ids_by_aggs(cls,group_id_list):
def get_group_ids_by_aggs(cls,group_id_list,es_cli_obj=None):
"""
:remark:聚合查询获取小组列表
:param group_id_list:
:return:
"""
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
q = dict()
q["size"]=0
q["query"] = {
......@@ -136,7 +145,7 @@ class GroupUtils(object):
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),"topic",q,aggregations_query=True)
result_dict = ESPerform.get_search_results(es_cli_obj,"topic",q,aggregations_query=True)
buckets_list = result_dict["aggregations"]["group_ids"]["buckets"]
sorted_buckets_list = sorted(buckets_list,key=lambda item:item["max_date"]["value"],reverse=True)
......
......@@ -77,7 +77,7 @@ class PickUtils(object):
}
}
q["_source"] = {
"include": ["id"]
"includes": ["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
recommend_pick_celebrity_list = list()
......@@ -123,7 +123,7 @@ class PickUtils(object):
}
}
q["_source"] = {
"include":["id"]
"includes":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
......@@ -153,7 +153,7 @@ class PickUtils(object):
}
}
q["_source"] = {
"include":["id"]
"includes":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
......
This diff is collapsed.
......@@ -5,18 +5,72 @@ import logging
import traceback
import json
import time
import copy
from libs.es import ESPerform
class UserUtils(object):
@classmethod
def get_attention_user_list(cls,user_id_list,self_user_id):
def get_batch_attention_user_list(cls,user_id_list,self_user_id,es_cli_obj=None):
"""
:remark 批量用户 关注的 用户列表
:param user_id_list:
:param self_user_id:
:return:
"""
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
user_id_list.append(self_user_id)
q = dict()
q["query"] = {
"terms":{
"user_id":user_id_list
}
}
q["_source"] = {
"includes":["attention_user_id_list","user_id"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=len(user_id_list))
self_attention_user_id_list = []
attention_user_dict_list = list()
ret_attention_user_id_list = list()
for hit_item in result_dict["hits"]:
attention_user_dict = dict()
user_id = hit_item["_source"]["user_id"]
if user_id == self_user_id:
self_attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
else:
attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
attention_user_dict[user_id] = attention_user_id_list
attention_user_dict_list.append(attention_user_dict)
ret_attention_user_id_list.append(user_id)
return [self_attention_user_id_list,attention_user_dict_list,ret_attention_user_id_list]
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([],[],[])
@classmethod
def get_attention_user_list(cls,user_id_list,self_user_id,es_cli_obj=None):
"""
:remark 获取指定用户列表 关注的 用户列表
:param user_id:
:return:
"""
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
q = dict()
q["query"] = {
"terms":{
......@@ -24,10 +78,10 @@ class UserUtils(object):
}
}
q["_source"] = {
"include":["attention_user_id_list","user_id"]
"includes":["attention_user_id_list","user_id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset=0, size=len(user_id_list))
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=len(user_id_list))
self_attention_user_id_list = []
recursion_attention_user_id_list = []
......@@ -59,7 +113,7 @@ class UserUtils(object):
return []
@classmethod
def get_recommend_user_list(cls,self_attention_user_id_list,recursion_attention_user_id_list,offset,size):
def get_recommend_user_list(cls,self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj=None):
"""
:remark 获取推荐用户列表
:param attention_user_id_list:
......@@ -67,6 +121,9 @@ class UserUtils(object):
:return:
"""
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
q = dict()
q["query"] = dict()
......@@ -118,9 +175,9 @@ class UserUtils(object):
}
q["query"]["function_score"] = query_function_score
q["_source"] = {
"include":["user_id"]
"includes":["user_id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="user", query_body=q,
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="user", query_body=q,
offset=offset, size=size)
recommend_user_list = list()
......@@ -131,3 +188,109 @@ class UserUtils(object):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_batch_recommend_user_dict(cls,need_filter_attention_user_id_list,attention_user_id_list,attention_user_dict_list,self_user_id,offset,size,es_cli_obj=None):
"""
:remark 获取批量推荐用户
:param need_filter_attention_user_id_list:
:param attention_user_dict:
:param self_user_id:
:param offset:
:param size:
:return:
"""
try:
if not es_cli_obj:
es_cli_obj = ESPerform.get_cli()
batch_query_list = list()
for interesting_user_item_dict in attention_user_dict_list:
for interesting_user_id in interesting_user_item_dict:
if interesting_user_id != self_user_id:
logging.info("duan add,interesting_user_id:%d" % interesting_user_id)
filter_user_id_list = copy.deepcopy(need_filter_attention_user_id_list)
filter_user_id_list.append(interesting_user_id)
q = dict()
q["from"] = offset
q["size"] = size
q["query"] = dict()
functions_list = [
{
"gauss": {
"latest_topic_time_val": {
"origin": int(time.time()),
"scale": "600",
"decay": 0.1
}
}
}
]
if len(interesting_user_item_dict[interesting_user_id]) > 0:
functions_list.append(
{
"filter":{
"bool":{
"should":{"terms":{"user_id":interesting_user_item_dict[interesting_user_id]}}
}
},
"weight":10
}
)
query_function_score = {
"query": {
"bool": {
"must": [
{"term": {"is_recommend": True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"is_shadow": False}}
],
"must_not":{
"terms":{
"user_id":filter_user_id_list
}
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
q["query"]["function_score"] = query_function_score
q["_source"] = {
"includes":["user_id"]
}
batch_query_list.append(q)
index_name = ESPerform.get_official_index_name("user","read")
search_header_dict = {'index': index_name, 'type': "_doc"}
query_body = ""
for query_item in batch_query_list:
query_body += "{}\n{}\n".format(json.dumps(search_header_dict),json.dumps(query_item))
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name="user", query_body=query_body,
batch_search=True)
ret_dict = dict()
user_index = 0
for res_item in result_dict["responses"]:
recommend_user_list = list()
for item in res_item["hits"]["hits"]:
recommend_user_list.append(item["_source"]["user_id"])
logging.info("duan add,attention_user_id_list:%s,user_index:%d" % (str(attention_user_id_list), user_index))
ret_dict[str(attention_user_id_list[user_index])] = recommend_user_list
user_index += 1
logging.info("duan add,ret_dict:%s" % str(ret_dict))
return ret_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
......@@ -9,6 +9,7 @@ from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
@bind("physical/search/query_group")
......@@ -46,18 +47,21 @@ def group_sort(user_id=-1,sort_type=GroupSortTypes.HOT_RECOMMEND,offset=0,size=1
if not isinstance(user_id,int):
user_id = -1
#获取es链接对象
es_cli_obj = ESPerform.get_cli()
if sort_type==GroupSortTypes.HOT_RECOMMEND:
group_ids_list = GroupUtils.get_hot_group_recommend_result_list(offset,size)
group_ids_list = GroupUtils.get_hot_group_recommend_result_list(offset,size,es_cli_obj)
return {"group_recommend_ids":group_ids_list}
elif sort_type==GroupSortTypes.ATTENTION_RECOMMEND:
attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset=0,size=1)
attention_group_list = GroupUtils.get_user_attention_group_list(user_id,offset=0,size=1,es_cli_obj=es_cli_obj)
if len(attention_group_list)==0:
return {"group_recommend_ids": []}
else:
attention_group_id_list = [item["group_id"] for item in attention_group_list]
sorted_group_ids_list = GroupUtils.get_group_ids_by_aggs(attention_group_id_list)
sorted_group_ids_list = GroupUtils.get_group_ids_by_aggs(attention_group_id_list,es_cli_obj)
group_recommend_ids_list = sorted_group_ids_list
#if len(group_recommend_ids_list) < size and len(group_recommend_ids_list)<len(attention_group_list):
......
......@@ -113,7 +113,7 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
"_score"
]
q["_source"] = {
"include":["id"]
"includes":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
......
......@@ -46,7 +46,7 @@ def query_tag(query,offset,size):
{'_score': {"order": "desc"}}
],
"_source": {
"include": ["id", "name"]
"includes": ["id", "name"]
}
}
q["highlight"] = get_highlight(["name"])
......@@ -107,7 +107,7 @@ def query_by_tag_type(tag_type_id,offset,size):
{"near_new_topic_num":{"order":"desc"}}
],
"_source": {
"include": ["id", "name"]
"includes": ["id", "name"]
}
}
......
This diff is collapsed.
......@@ -10,6 +10,7 @@ from libs.es import ESPerform
from libs.cache import redis_client
from search.utils.user import UserUtils
from search.utils.common import *
from libs.es import ESPerform
@bind("physical/search/recommend_user")
......@@ -26,16 +27,51 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
if not isinstance(self_user_id,int):
self_user_id = -1
#获取es链接对象
es_cli_obj = ESPerform.get_cli()
#获取关注用户列表
(self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id)
(self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id,es_cli_obj)
#去除自身及感兴趣的用户ID
self_attention_user_id_list.append(self_user_id)
self_attention_user_id_list.append(interesting_user_id)
recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size)
recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj)
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_user_ids":[]}
\ No newline at end of file
return []
@bind("physical/search/batch_recommend_user")
def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10):
"""
:remark 点关注推荐用户 - 批量接口
:param self_user_id:
:param interesting_user_id:
:param offset:
:param size:
:return:
"""
try:
if not isinstance(self_user_id,int):
self_user_id = -1
#获取es链接对象
es_cli_obj = ESPerform.get_cli()
#获取关注用户列表
(need_filter_attention_user_id_list, attention_user_dict_list,attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list,self_user_id,es_cli_obj)
#去除自身及感兴趣的用户ID
need_filter_attention_user_id_list.append(self_user_id)
recommend_user_dict = UserUtils.get_batch_recommend_user_dict(need_filter_attention_user_id_list=need_filter_attention_user_id_list,attention_user_id_list=attention_user_id_list,attention_user_dict_list=attention_user_dict_list,self_user_id=self_user_id,offset=offset,size=size,es_cli_obj=es_cli_obj)
logging.info("duan add,recommend_user_dict:%s" % str(recommend_user_dict))
return recommend_user_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {}
\ No newline at end of file
......@@ -18,6 +18,7 @@
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
"manual_score":{"type":"double"},//人工赋分
"has_image":{"type":"boolean"},//是否有图
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
......
......@@ -27,32 +27,58 @@ class ActionSumAboutTopic(models.Model):
data_type = models.IntegerField(verbose_name=u"动作类型")
data_value = models.BigIntegerField(verbose_name=u"值")
class TopicImage(models.Model):
class Meta:
verbose_name = u'日记图片'
db_table = 'topic_image'
id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
topic_id = models.IntegerField(verbose_name=u'日记ID')
url = models.CharField(verbose_name=u'图片URL',max_length=300)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class Topic(models.Model):
class Meta:
verbose_name = u'日记'
db_table = 'topic'
id = models.IntegerField(verbose_name='日记ID',primary_key=True)
name = models.CharField(verbose_name='日记名称',max_length=100)
id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
name = models.CharField(verbose_name=u'日记名称',max_length=100)
#group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
group = models.ForeignKey(
Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
user_id = models.IntegerField(verbose_name='用户ID')
user_id = models.IntegerField(verbose_name=u'用户ID')
drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
description = models.CharField(verbose_name='日记本描述',max_length=200)
content = models.CharField(verbose_name='日记本内容',max_length=1000)
description = models.CharField(verbose_name=u'日记本描述',max_length=200)
content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
share_num = models.IntegerField(verbose_name='')
vote_num = models.IntegerField(verbose_name='点赞数')
reply_num = models.IntegerField(verbose_name='回复数')
vote_num = models.IntegerField(verbose_name=u'点赞数')
reply_num = models.IntegerField(verbose_name=u'回复数')
cover = models.CharField(verbose_name='',max_length=200)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
content_level = models.CharField(verbose_name='内容等级',max_length=3)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
def topic_has_image(self):
try:
has_image = False
query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
if len(query_list)>0:
has_image=True
return has_image
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
def get_pick_id_info(self):
try:
pick_id_list = list()
......
......@@ -38,6 +38,7 @@ class TopicTransfer(object):
res["offline_score"] = instance.get_topic_offline_score()
res["manual_score"] = instance.drop_score
res["has_image"] = instance.topic_has_image()
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment