Skip to content

P
physical
Commit 16b74f47 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'master' of git.wanmeizhensuo.com:alpha/physical

parents 5256d14e 2bcf8726
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 184 additions and 52 deletions
search/utils/topic.py
View file @ 16b74f47
......@@ -14,7 +14,6 @@ from trans2es.models.pictorial import PictorialTopics
from libs.cache import redis_client
class TopicUtils(object):
@classmethod
......@@ -35,7 +34,7 @@ class TopicUtils(object):
}
q["_source"] = {
"include": ["tag_list","attention_user_id_list", "pick_user_id_list", "same_pictorial_user_id_list"]
"include": ["tag_list", "attention_user_id_list", "pick_user_id_list", "same_pictorial_user_id_list"]
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), "user", q, offset, size)
......@@ -174,18 +173,18 @@ class TopicUtils(object):
}
]
if user_id and user_id>0:
if user_id and user_id > 0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list)>0:
if len(user_similar_list) > 0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"constant_score":{"filter":{"terms": {"user_id": attention_user_id_list}}}},
"filter": {"constant_score": {"filter": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
......@@ -272,7 +271,7 @@ class TopicUtils(object):
]
query_function_score["query"]["bool"]["minimum_should_match"] = 1
query_function_score["query"]["bool"]["filter"].append(
{"range": {"content_level": {"gte":3,"lte":6}}}
{"range": {"content_level": {"gte": 3, "lte": 6}}}
)
else:
if "must_not" in query_function_score["query"]["bool"]:
......@@ -421,7 +420,7 @@ class TopicUtils(object):
@classmethod
def userful_tag_topic_list(cls, user_id, have_read_topic_list, size,
index_type="topic-high-star",routing=None,useful_tag_list =[]):
index_type="topic-high-star", routing=None, useful_tag_list=[]):
"""
:remark 帖子详情页推荐列表,缺少按时间衰减
:param user_id:
......@@ -455,20 +454,20 @@ class TopicUtils(object):
else:
q = dict()
q["query"] = {
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"useful_tag_list": useful_tag_list}},
{"term":{"content_level":6}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
}
}
"bool": {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"useful_tag_list": useful_tag_list}},
{"term": {"content_level": 6}}
],
"must_not": {
"terms": {
"id": have_read_topic_list
}
}
}
}
# logging.warning("topic_tag_list:%s"%str(topic_tag_list))
# query_function_score = {
# "query": {
......@@ -490,7 +489,8 @@ class TopicUtils(object):
"includes": ["id"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size,
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q,
size=size,
routing=routing)
topic_id_list = list()
......@@ -959,7 +959,7 @@ class TopicUtils(object):
return nf
@classmethod
def process_sort(cls, sorts_by):
def process_sort(cls, sorts_by, pictorial_id):
"""处理排序部分。"""
sort_rule = []
......@@ -1012,6 +1012,62 @@ class TopicUtils(object):
"order": "desc"
},
})
elif sort_by == TOPIC_SEARCH_SORT.REAL_VOTE_AEC:
sort_rule.append({
"related_billboard.real_vote_cnt": {
"order": "asc",
"nested_path": "related_billboard",
"missing": "_last",
"nested_filter": {
"term": {
"related_billboard.pictorial_id": pictorial_id
}
}
},
})
elif sort_by == TOPIC_SEARCH_SORT.REAL_VOTE_DESC:
sort_rule.append({
"related_billboard.real_vote_cnt": {
"order": "desc",
"nested_path": "related_billboard",
# "missing": "_last",
"nested_filter": {
"term": {
"related_billboard.pictorial_id": pictorial_id
}
}
},
})
elif sort_by == TOPIC_SEARCH_SORT.VIRT_VOTE_AEC:
sort_rule.append({
"related_billboard.virt_vote_cnt": {
"order": "asc",
"nested_path": "related_billboard",
# "missing": "_last",
"nested_filter": {
"term": {
"related_billboard.pictorial_id": pictorial_id
}
}
},
})
elif sort_by == TOPIC_SEARCH_SORT.VIRT_VOTE_DESC:
sort_rule.append({
"related_billboard.virt_vote_cnt": {
"order": "desc",
"nested_path": "related_billboard",
# "missing": "_last",
"nested_filter": {
"term": {
"related_billboard.pictorial_id": pictorial_id
}
}
},
})
logging.info("get picotirial:%s" % sort_rule)
return sort_rule
......@@ -1029,7 +1085,7 @@ class TopicUtils(object):
}
if sorts_by:
sorts = cls.process_sort(sorts_by)
sorts = cls.process_sort(sorts_by, pictorial_id=None)
if sorts:
q["sort"] = sorts
......@@ -1060,7 +1116,6 @@ class TopicUtils(object):
def business_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
must = cls.business_filters(filters, filter_online=filter_online)
logging.info("get must:%s" % must)
q = {
"query": {
"bool": {
......@@ -1070,11 +1125,16 @@ class TopicUtils(object):
}
}
if sorts_by:
sorts = cls.process_sort(sorts_by)
if sorts:
q["sort"] = sorts
if 'pictorial_id' in filters.keys():
if sorts_by:
sorts = cls.process_sort(sorts_by, filters["pictorial_id"])
if sorts:
q["sort"] = sorts
else:
if sorts_by:
sorts = cls.process_sort(sorts_by, pictorial_id=None)
if sorts:
q["sort"] = sorts
try:
result_dict = ESPerform.get_search_results(
......@@ -1156,6 +1216,25 @@ class TopicUtils(object):
}
}
})
# elif k == "pictorial_id":
# f.append({
# "nested": {
# "path": "related_billboard",
# "query": {
# "bool": {
# "must": [
# {
# "term": {
# "related_billboard.pictorial_id": v
# }
# }
# ]
# }
# }
# }
# })
else:
if isinstance(v, list):
......
search/views/group.py
View file @ 16b74f47
......@@ -105,23 +105,23 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
q = {}
# 获取帖子从属的画报
q["query"] = {
"bool":{
"must":[
"bool": {
"must": [
{
"term": {
"id": topic_id
}
},
{
"term":{
"is_online":True
"term": {
"is_online": True
}
}
],
"must_not":[
"must_not": [
{
"term":{
"is_history":True
"term": {
"is_history": True
}
}
]
......@@ -237,6 +237,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
"""
:remark 画报排序 人气 部分
人气按照票数从大到小排序，相同票数按照图片票数更新时间由旧到新排序
:param user_id:
:param sort_type:
:param offset:
......@@ -251,18 +252,45 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
"query": {
"bool": {
"must": [
{"term": {"pictorial_id": pictorial_id}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"pictorial_id": pictorial_id}}
# {
# "nested": {
# "path": "related_billboard",
# "query": {
# "bool": {
# "must": [
# {
# "term": {
# "related_billboard.pictorial_id": pictorial_id
# }
# }
# ]
# }
# }
# }
# }
]
}
},
"sort": [
{"total_vote_num": {"order": "desc"}},
{"related_billboard.total_vote_cnt": {
"order": "desc",
"nested_path": "related_billboard",
"missing": "_last",
"nested_filter": {
"term": {
"related_billboard.pictorial_id": pictorial_id
}
}
}},
{"create_time": {"order": "desc"}}
]
}
logging.info("get qqqqqq:%s" % q)
pict_pictorial_ids_list = []
# 获取es链接对象
es_cli_obj = ESPerform.get_cli()
......
trans2es/mapping/topic-high-star.json
View file @ 16b74f47
......@@ -55,6 +55,15 @@
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
"is_history": {"type": "boolean"}, //是否历史数据
"related_billboard":{
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"}
}
}
}
}
trans2es/mapping/topic.json
View file @ 16b74f47
......@@ -55,6 +55,18 @@
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
"is_history": {"type": "boolean"}, //是否历史数据
"related_billboard":{
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"}
}
}
}
}
trans2es/models/topic.py
View file @ 16b74f47
This diff is collapsed.
trans2es/utils/topic_transfer.py
View file @ 16b74f47
......@@ -9,12 +9,13 @@ import time
import re
import datetime
from trans2es.models.user import User
from trans2es.models.topic import ExcellentTopic,TopicHomeRecommend
from trans2es.models.topic import ExcellentTopic, TopicHomeRecommend
class TopicTransfer(object):
@classmethod
def get_topic_data(cls,instance,is_excellect=False):
def get_topic_data(cls, instance, is_excellect=False):
try:
res = dict()
......@@ -47,25 +48,24 @@ class TopicTransfer(object):
begin = time.time()
res["pick_id_list"] = instance.get_pick_id_info()
end = time.time()
time0 = (end-begin)
time0 = (end - begin)
begin = time.time()
(topic_tag_id_list, edit_tag_id_list) = instance.get_topic_tag_id_list()
res["tag_list"] = topic_tag_id_list
res["edit_tag_list"] = edit_tag_id_list
res["edit_tag_list"] = instance.get_edit_tag_id_list()
end = time.time()
time1 = (end-begin)
time1 = (end - begin)
begin = time.time()
res["tag_name_list"] = instance.get_tag_name_list(res["tag_list"])
end = time.time()
time2 = (end-begin)
time2 = (end - begin)
begin = time.time()
res["offline_score"] = instance.get_topic_offline_score()
end = time.time()
time3 = (end-begin)
time3 = (end - begin)
begin = time.time()
res["manual_score"] = instance.drop_score
......@@ -73,10 +73,10 @@ class TopicTransfer(object):
res["has_video"] = instance.has_video
res["language_type"] = instance.language_type
end = time.time()
time4 = (end-begin)
time4 = (end - begin)
# begin = time.time()
topic_useful_tag_id_list= instance.get_topic_useful_tag_id_list()
topic_useful_tag_id_list = instance.get_topic_useful_tag_id_list()
res["useful_tag_list"] = topic_useful_tag_id_list
# end = time.time()
# time5 = (end - begin)
......@@ -99,7 +99,7 @@ class TopicTransfer(object):
res["virtual_content_level"] = instance.virtual_content_level
res["like_num_crawl"] = instance.like_num_crawl
res["comment_num_crawl"]= instance.comment_num_crawl
res["comment_num_crawl"] = instance.comment_num_crawl
res["is_crawl"] = instance.is_crawl
res["platform"] = instance.platform
......@@ -113,7 +113,6 @@ class TopicTransfer(object):
res["create_time"] = tzlc_create_time
res["create_time_val"] = int(time.mktime(tzlc_create_time.timetuple()))
update_time = instance.update_time
tzlc_update_time = tzlc(update_time)
res["update_time"] = tzlc_update_time
......@@ -137,7 +136,12 @@ class TopicTransfer(object):
if operation_home_recommend and operation_home_recommend.is_online and not operation_home_recommend.is_deleted:
res["is_operation_home_recommend"] = True
logging.info("test topic transfer time cost,time0:%d,time1:%d,time2:%d,time3:%d,time4:%d" % (time0,time1,time2,time3,time4))
logging.info("test topic transfer time cost,time0:%d,time1:%d,time2:%d,time3:%d,time4:%d" % (
time0, time1, time2, time3, time4))
# 榜单关联的投票
res["related_billboard"] = instance.get_related_billboard()
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment