Skip to content

P
physical
Commit aeefcc14 authored by 段英荣's avatar 段英荣
Browse files
Options

modify

parent 8d2d5945
Hide whitespace changes
Inline Side-by-side
Showing with 476 additions and 65 deletions
search/utils/common.py
View file @ aeefcc14
......@@ -7,4 +7,21 @@ class GroupSortTypes(object):
HOT_RECOMMEND=0
# 关注排序
ATTENTION_RECOMMEND=1
\ No newline at end of file
ATTENTION_RECOMMEND=1
class PickType(object):
# 明星榜
CELEBRITY_PICK=0
# 帖子榜
TOPIC_PICK=1
class TopicDocumentField(object):
"""
帖子索引字段信息
"""
ID="id",
IS_ONLINE = "is_online",
TAG_LIST = "tag_list"
search/utils/pick.py
View file @ aeefcc14
......@@ -28,4 +28,139 @@ class PickUtils(object):
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
\ No newline at end of file
return False
@classmethod
def ___get_filter_term_list(cls,ori_list):
try:
should_term_list = list()
for term_id in ori_list:
term_dict = {
"term":{
"id":{"value":term_id}
}
}
should_term_list.append(term_dict)
return should_term_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_recommend_celebrity_id_list(cls,pick_id,have_read_pick_celebrity_ids,offset,size):
"""
:remark 根据榜单ID推荐明星ID
:param pick_id:
:param have_read_pick_celebrity_ids:
:return:
"""
try:
filter_term_list = cls.___get_filter_term_list(have_read_pick_celebrity_ids)
q = dict()
q["query"] = {
"bool":{
"filter":filter_term_list,
"must":[
{"term":{"pick_id",pick_id}},
{"term":{"is_online":True}}
]
},
"_source":{
"include": ["id"]
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
recommend_pick_celebrity_list = [item["id"] for item in result_dict["hits"]]
return recommend_pick_celebrity_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_topic_pick_query(cls,query,pick_id,offset,size):
"""
:remark 查询榜单下帖子列表
:param query:
:param pick_id:
:param offset:
:param size:
:return:
"""
try:
q = dict()
multi_fields = {
'description': 2,
'name': 4,
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'cross_fields',
'operator': 'and',
'fields': query_fields,
}
q["query"] = {
"multi_match":multi_match,
"bool":{
"must":[
{"term":{"pick_id_list":pick_id}},
{"term":{"is_online":True}}
]
}
}
q["_source"] = {
"include":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,offset=offset,size=size)
recommend_topic_id_list = []
if len(result_dict["hits"]) > 0:
recommend_topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
return recommend_topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_celebrity_pick_query(cls,query,pick_id,offset,size):
try:
q = dict()
"""
multi_fields = {
'name': 2,
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'cross_fields',
'operator': 'and',
'fields': query_fields,
}
"""
q["query"] = {
#"multi_match":multi_match,
"match":{
"name":query
},
"bool":{
"must":[
{"term":{"pick_id_list":pick_id}},
{"term":{"is_online":True}}
]
}
}
q["_source"] = {
"include":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="celebrity",query_body=q,offset=offset,size=size)
recommend_celebrity_id_list = []
if len(result_dict["hits"]) > 0:
recommend_celebrity_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
return recommend_celebrity_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
search/utils/topic.py
View file @ aeefcc14
......@@ -3,7 +3,9 @@
import logging
import traceback
import json
from libs.es import ESPerform
from .common import TopicDocumentField
class TopicUtils(object):
......@@ -32,7 +34,7 @@ class TopicUtils(object):
return result_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
return {"total_count":0,"hits":[]}
@classmethod
......@@ -231,11 +233,11 @@ class TopicUtils(object):
return ([],[])
@classmethod
def get_topic_detail_recommend(cls,user_id,user_tag_list,topic_group_id,topic_user_id,offset,size):
def get_topic_detail_recommend_list(cls,user_id,topic_tag_list,topic_group_id,topic_user_id,offset,size):
"""
:remark 帖子详情页推荐策略,缺少按时间衰减
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
:param user_tag_list:
:param topic_tag_list:
:param topic_group_id:
:param topic_user_id:
:param offset:
......@@ -258,7 +260,7 @@ class TopicUtils(object):
"weight": 1000
}
]
query_tag_term_list = cls.___get_should_term_list(user_tag_list)
query_tag_term_list = cls.___get_should_term_list(topic_tag_list)
query_function_score = {
"query":{
"bool":{
......@@ -280,20 +282,36 @@ class TopicUtils(object):
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=offset, size=size)
return result_dict
return result_dict["hits"]
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_topic_detail_info(cls,topic_id):
def get_topic_tag_id_list(cls,topic_id):
"""
:remark 获取帖子标签列表
:param topic_id:
:return:
"""
try:
q = dict()
q["query"] = {
"term":{
""
"id": topic_id
}
}
q["_source"] = {
"include":[TopicDocumentField.TAG_LIST]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic",query_body=q,size=1)
tag_id_list = []
if len(result_dict["hits"])>0:
tag_id_list = json.loads(result_dict["hits"][0]["_source"][TopicDocumentField.TAG_LIST])
return tag_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
\ No newline at end of file
return list()
\ No newline at end of file
search/utils/user.py 0 → 100644
View file @ aeefcc14
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import traceback
import json
from libs.es import ESPerform
class UserUtils(object):
@classmethod
def get_attention_user_list(cls,use_id_list):
"""
:remark 获取指定用户列表 关注的 用户列表
:param user_id:
:return:
"""
try:
q = dict()
q["query"] = {
"terms":{
"user_id":use_id_list
}
}
q["_source"] = {
"include":["attention_user_id_list"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset=0, size=1)
attention_user_id_list = []
for hit_item in result_dict["hits"]:
attention_user_id_list = [item["user_id"] for item in hit_item["_source"]["attention_user_id_list"]]
return attention_user_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def ___get_should_term_list(cls,ori_list):
try:
should_term_list = list()
for term_id in ori_list:
term_dict = {
"term":{
"tag_list":{"value":term_id}
}
}
should_term_list.append(term_dict)
return should_term_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_recommend_user_list(cls,attention_user_id_list,recursion_attention_user_id_list,offset,size):
"""
:remark 获取推荐用户列表
:param attention_user_id_list:
:param recursion_attention_user_id_list:
:return:
"""
try:
q = dict()
q["query"] = dict()
recursion_attention_user_list = cls.___get_should_term_list(recursion_attention_user_id_list)
functions_list = [
{
"filter":{
"bool":{
"should":recursion_attention_user_list
}
},
"weight":10
}
]
query_function_score = {
"query": {
"bool": {
"should": [
{"term": {"is_recommend": True}},
{"term": {"is_online": True}},
],
"minimum_should_match":2,
"must_not":{
"terms":attention_user_id_list
}
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions": functions_list
}
q["query"]["function_score"] = query_function_score
q["_source"] = {
"include":["user_id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="user", query_body=q,
offset=offset, size=size)
recommend_user_list = list()
for item in result_dict["hits"]:
recommend_user_list.append(item["_source"]["user_id"])
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
\ No newline at end of file
search/views/pick.py
View file @ aeefcc14
......@@ -8,6 +8,7 @@ import json
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.pick import PickUtils
from search.utils.common import PickType
@bind("physical/search/pick_celebrity")
def pick_celebrity(user_id,pick_id,offset=0,size=10):
......@@ -23,32 +24,16 @@ def pick_celebrity(user_id,pick_id,offset=0,size=10):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
pass
q = dict()
q["query"] = {
"pick_id":pick_id,
"bool":{
"must_not":{
"terms":{
"id":have_read_pick_celebrity_ids
}
}
}
}
q["_source"] = {
"include":["id"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="pick_celebrity",query_body=q,offset=offset,size=size)
recommend_pick_celebrity_list = PickUtils.get_recommend_celebrity_id_list(pick_id,have_read_pick_celebrity_ids,offset,size)
recommend_pick_celebrity_list = [item["id"] for item in result_dict["hits"]]
key_exist = len(have_read_pick_celebrity_ids) > 0
PickUtils.refresh_redis_data(have_read_celebrity_list=have_read_pick_celebrity_ids.extend(recommend_pick_celebrity_list),
redis_cli=redis_client,redis_key=redis_key,key_exist=key_exist)
return result_dict
return {"recommend_pick_celebrity_ids": recommend_pick_celebrity_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]}
return {"recommend_pick_celebrity_ids":[]}
@bind("physical/search/pick_topic")
......@@ -65,10 +50,6 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
q = dict()
q["filter"] = {
"range":{"content_level":{"gte":3,"lte":5}}
}
query_function_score = dict()
functions_list = [
{
"filter": {"match": {"content_level": 5}},
......@@ -86,19 +67,25 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
query_function_score = {
"query":{
"pick_id_list":[pick_id]},
"bool":{
"must":[
{"term":{"pick_id_list":pick_id}},
{"range": {"content_level": {"gte": 3, "lte": 5}}}
],
"must_not": {
"terms": {
"id": have_read_pick_topic_ids
}
},
}
},
"score_mode": "sum",
"boost_mode": "sum",
"functions":functions_list
}
q["query"] = {
"funciton_score":query_function_score,
"bool":{
"must_not":{
"terms":{
"id":have_read_pick_topic_ids
}
}
}
"funciton_score":query_function_score
}
q["sort"] = [
"_score"
......@@ -114,11 +101,35 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
PickUtils.refresh_redis_data(have_read_celebrity_list=have_read_pick_topic_ids.extend(recommend_pick_topic_list),
redis_cli=redis_client,redis_key=redis_key,key_exist=key_exist)
return result_dict
return {"recommend_pick_topic_ids": recommend_pick_topic_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_pick_topic_ids":[]}
@bind("physical/search/pick_query")
def pick_query(query="",pick_id=-1,query_type=PickType.TOPIC_PICK,offset=0,size=10):
"""
:remark 榜单搜索
:param query:
:param pick_id:
:param query_type:
:param offset:
:param size:
:return:
"""
try:
pick_query_result_list = []
if query_type == PickType.TOPIC_PICK:
pick_query_result_list = PickUtils.get_topic_pick_query(query,pick_id,offset,size)
elif query_type == PickType.CELEBRITY_PICK:
pick_query_result_list = PickUtils.get_celebrity_pick_query(query,pick_id,offset,size)
return {"pick_query_result_list":pick_query_result_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]}
return {"pick_query_result_list":[]}
......
search/views/topic.py
View file @ aeefcc14
......@@ -128,7 +128,7 @@ def home_query(user_id=-1,query="",offset=0,size=10):
@bind("physical/search/topic_detail_page_recommend")
def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_user_id=-1,offset=0,size=10):
"""
:remark:帖子详情页推荐策略
:remark:帖子详情页推荐策略，缺少第一个卡片策略
:param user_id:
:param topic_id:
:param topic_group_id:
......@@ -136,13 +136,15 @@ def topic_detail_page_recommend(user_id=-1,topic_id=-1,topic_group_id=-1,topic_u
:return:
"""
try:
user_tag_list = list()
# 获取帖子标签列表
topic_tag_list = TopicUtils.get_topic_tag_id_list(topic_id)
result_list = TopicUtils.get_topic_detail_recommend_list(user_id,topic_tag_list,topic_group_id,topic_user_id,offset,size)
recommend_topic_ids_list = list()
if len(result_list)>0:
recommend_topic_ids_list = [item["_source"]["id"] for item in result_list]
result_dict = TopicUtils.get_topic_detail_recommend(user_id,user_tag_list,topic_group_id,topic_user_id,offset,size)
return {"recommend_topic_ids": []}
return {"recommend_topic_ids": recommend_topic_ids_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_ids": []}
......
search/views/user.py 0 → 100644
View file @ aeefcc14
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
from search.utils.topic import TopicUtils
from libs.es import ESPerform
from libs.cache import redis_client
from search.utils.user import UserUtils
@bind("physical/search/recommend_user")
def recommend_user(user_id=-1,offset=0,size=10):
"""
:remark: 点关注推荐用户,缺少按时间排序策略
:param user_id:
:param offset:
:param size:
:return:
"""
try:
#获取关注用户列表
attention_user_id_list = UserUtils.get_attention_user_list([user_id])
#关注的用户 关注用户列表
recursion_attention_user_id_list = UserUtils.get_attention_user_list(attention_user_id_list)
recommend_user_list = UserUtils.get_recommend_user_list(attention_user_id_list,recursion_attention_user_id_list,offset,size)
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_user_ids":[]}
\ No newline at end of file
trans2es/mapping/celebrity.json
View file @ aeefcc14
......@@ -9,8 +9,7 @@
"description":{"type":"text"},
"gender":{"type":"boolean"},
"city_id":{"type":"text"},
"user_id":{"type":"long"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"}
}
......
trans2es/mapping/user.json
View file @ aeefcc14
......@@ -10,6 +10,9 @@
"tag_list":{"type":"long"},//标签属性
"city_id":{"type":"long"},
"country_id":{"type":"long"},
"is_recommend":{"type":"boolean"},//是否运营推荐用户
"is_shadow":{"type":"boolean"},//是否马甲用户
"latest_topic_time_val":{"type":"long"},//最近发帖时间
"attention_user_id_list":{//关注用户列表
"type":"nested",
"properties":{
......
trans2es/models/celebrity.py 0 → 100644
View file @ aeefcc14
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
import traceback
import logging
from libs.es import ESPerform
from django.db import models
import datetime
from .pick_celebrity import PickCelebrity
class Celebrity(models.Model):
class Meta:
verbose_name = u'明星表'
db_table = 'celebrity'
id = models.IntegerField(verbose_name="主键ID",primary_key=True)
is_deleted = models.BooleanField(verbose_name="是否删除")
name = models.CharField(verbose_name=u'名称', max_length=300, default='')
portrait = models.CharField(verbose_name=u'肖像', max_length=100, default='')
gender = models.SmallIntegerField(verbose_name=u'性别')
city_id = models.IntegerField(verbose_name=u'城市id')
desc = models.IntegerField(verbose_name='',max_length=1000)
is_online = models.BooleanField(verbose_name="是否上线")
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
def get_pick_id_list(self):
try:
pick_id_list = list()
query_results = PickCelebrity.objects.filter(celebrity_id=self.id,is_deleted=False)
for data_item in query_results:
pick_id_list.append(data_item.pick_id)
return list(tuple(pick_id_list))
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
\ No newline at end of file
trans2es/models/pick_celebrity.py
View file @ aeefcc14
......@@ -16,6 +16,7 @@ class PickCelebrity(models.Model):
db_table = u"pickcelebrity"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
is_deleted = models.BooleanField(verbose_name="是否删除")
celebrity_id = models.IntegerField(verbose_name=u"明星ID")
pick_id = models.IntegerField(verbose_name=u"榜ID")
rank = models.IntegerField(verbose_name=u"排名")
......
trans2es/models/user.py
View file @ aeefcc14
......@@ -14,6 +14,7 @@ from libs.tools import tzlc
from .group_user_role import GroupUserRole
from .tag import AccountUserTag
from .topic import Topic
from .user_extra import UserExtra
class User(models.Model):
......@@ -32,6 +33,24 @@ class User(models.Model):
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_shadow = False
is_recommend = False
query_sql = UserExtra.objects.filter(user_id=user_id)
for record in query_sql:
is_recommend = record.is_recommend
is_shadow = record.is_shadow
# 获取该用户最新发帖时间
latest_topic_time_val = -1
topic_records = Topic.objects.filter(user_id=user_id).order_by(update_time)
check_index = 0
for record in topic_records:
topic_update_time = record.update_time
tzlc_topic_update_time = tzlc(topic_update_time)
latest_topic_time_val = int(time.mktime(tzlc_topic_update_time.timetuple()))
check_index += 1
if check_index >= 1:
break
def get_follow_user_id_list(self):
follow_user_id_list = list()
......
trans2es/type_info.py
View file @ aeefcc14
......@@ -12,11 +12,12 @@ import elasticsearch
import elasticsearch.helpers
import sys
from trans2es.models import topic,user,pick_celebrity,group
from trans2es.models import topic,user,pick_celebrity,group,celebrity
from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
from trans2es.utils.group_transfer import GroupTransfer
from trans2es.utils.topic_transfer import TopicTransfer
from trans2es.utils.celebrity_transfer import CelebrityTransfer
from libs.es import ESPerform
__es = None
......@@ -268,6 +269,17 @@ def get_type_info_map():
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name="celebrity", # 明星
type="celebrity",
model=celebrity.Celebrity,
# query_deferred=lambda:user.User.objects.all().query,
query_deferred=lambda: celebrity.Celebrity.objects.all().query,
get_data_func=CelebrityTransfer.get_celebrity_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name="group", # 小组
type="group",
......
trans2es/utils/celebrity_transfer.py
View file @ aeefcc14
......@@ -4,12 +4,12 @@ import os
import sys
import logging
import traceback
from libs.tools import tzlc
class CelebrityTransfer(object):
def __init__(self):
pass
def get_celebrity_data(self,instance):
@classmethod
def get_celebrity_data(cls,instance):
try:
res = dict()
......@@ -18,13 +18,18 @@ class CelebrityTransfer(object):
res["is_deleted"] = instance.is_deleted
res["portrait"] = instance.portrait
res["name"] = instance.name
res["description"] = instance.description
res["description"] = instance.desc
res["gender"] = instance.gender
res["city_id"] = instance.city_id
res["user_id"] = instance.user_id
res["share_num"] = instance.share_num
res["create_time"] = instance.create_time
res["update_time"] = instance.update_time
res["pick_id_list"] = instance.get_pick_id_list()
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
res["create_time"] = tzlc_create_time
update_time = instance.update_time
tzlc_udpate_time = tzlc(update_time)
res["update_time"] = tzlc_udpate_time
return res
except:
......
trans2es/utils/user_transfer.py
View file @ aeefcc14
......@@ -22,6 +22,9 @@ class UserTransfer(object):
res["city_id"] = instance.city_id
res["country_id"] = instance.country_id
res["is_online"] = instance.is_online
res["is_recommend"] = instance.is_recommend
res["is_shadow"] = instance.is_shadow
res["latest_topic_time_val"] = instance.latest_topic_time_val
tzlc_create_time = tzlc(instance.create_time)
res["create_time"] = tzlc_create_time
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment