Skip to content

P
physical
Commit 9563a254 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'master' into 'test' 

Master

See merge request alpha/physical!302
parents 757ba7f5 5f2f827c
Hide whitespace changes
Inline Side-by-side
Showing with 121 additions and 37 deletions
libs/es.py
View file @ 9563a254
......@@ -154,7 +154,9 @@ class ESPerform(object):
bulk_actions = []
if sub_index_name == "topic" or sub_index_name == "topic-star-routing":
if sub_index_name=="topic" or \
sub_index_name=="topic-star-routing" or \
sub_index_name=="topic-high-star":
for data in data_list:
if data:
bulk_actions.append({
......
search/utils/topic.py
View file @ 9563a254
......@@ -174,6 +174,14 @@ class TopicUtils(object):
}
},
"weight": 60
},
{
"filter": {
"term": {
"content_level": 6
}
},
"weight": 200
}
]
......@@ -308,24 +316,28 @@ class TopicUtils(object):
"includes": ["id", "pictorial_id", "offline_score", "user_id", "edit_tag_list"]
}
q["sort"] = [
{
"_script": {
"type": "number",
"script": {
"lang": "expression",
"source": "_score+doc['offline_score']"
# "lang":"painless",
# "source":"_score+params._source.offline_score"
},
"order": "desc"
}
},
# {
# "offline_score":{
# "_script": {
# "type": "number",
# "script": {
# "lang": "expression",
# "source": "_score+doc['offline_score']"
# # "lang":"painless",
# # "source":"_score+params._source.offline_score"
# },
# "order": "desc"
# }
# },
"_score"
{
"_score": {
"order": "desc"
}
},
{
"offline_score":{
"order": "desc"
}
}
]
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size,routing=routing)
......
search/views/topic.py
View file @ 9563a254
......@@ -114,7 +114,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
user_similar_score_list=user_similar_score_redis_list,index_type="topic",routing="4,5,6",attention_tag_list=attention_tag_list)
user_similar_score_list=user_similar_score_redis_list,index_type="topic-high-star",routing="4,5,6",attention_tag_list=attention_tag_list)
if len(recommend_topic_list) == 6 and query is None:
if (size < 11):
......@@ -290,7 +290,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
result = list()
if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
index_type="topic",routing="4,5,6",topic_tag_list = topic_tag_list)
index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
......@@ -300,17 +300,17 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6",topic_user_id = topic_user_id
index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
......
trans2es/mapping/topic-high-star.json
View file @ 9563a254
......@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
trans2es/mapping/topic-star-routing.json
View file @ 9563a254
......@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
trans2es/mapping/topic-star.json 0 → 100644
View file @ 9563a254
{
"dynamic":"strict",
"properties": {
"id":{"type":"long"},
"content_level":{"type":"text"}
}
}
trans2es/mapping/topic.json
View file @ 9563a254
......@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
}
}
trans2es/type_info.py
View file @ 9563a254
......@@ -84,7 +84,7 @@ class TypeInfo(object):
def bulk_get_data(self, instance_iterable):
data_list = []
# 4，5星帖子单独索引
# 4星以上帖子单独索引
topic_data_high_star_list = list()
if self.batch_get_data_func:
......@@ -144,23 +144,73 @@ class TypeInfo(object):
else:
if data:
if self.type == "topic":
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "include":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
# else:
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if ori_topic_star:
ori_topic_star = str(ori_topic_star, encoding="utf-8")
else:
q = {
"query":{
"term":{
"id": data["id"]
}
},
"_source":{
"includes":["content_level"]
}
}
search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
if len(search_results["hits"]) > 0:
ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
if not ori_topic_star:
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
else:
int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
# int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
int_ori_topic_star = int(ori_topic_star)
if int_ori_topic_star != data["content_level"]:
old_data = copy.deepcopy(data)
old_data["is_online"] = False
old_data["is_deleted"] = True
old_data["content_level"] = int_ori_topic_star
old_data["is_history"] = True
data_list.append(old_data)
if int_ori_topic_star>=4:
topic_data_high_star_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
data_list.append(data)
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
# if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
# topic_data_high_star_list.append(data)
if data["content_level"] and int(data["content_level"])>=4:
topic_data_high_star_list.append(data)
data_list.append(data)
return (data_list,topic_data_high_star_list)
......@@ -234,13 +284,13 @@ class TypeInfo(object):
es=es,
)
# # 同时写4星及以上的帖子
# if len(topic_data_high_star_list)>0:
# self.elasticsearch_bulk_insert_data(
# sub_index_name="topic-high-star",
# data_list=topic_data_high_star_list,
# es=es,
# )
# 同时写4星及以上的帖子
if len(topic_data_high_star_list)>0:
self.elasticsearch_bulk_insert_data(
sub_index_name="topic-high-star",
data_list=topic_data_high_star_list,
es=es,
)
end = time.time()
time3=end-begin
......@@ -301,6 +351,16 @@ def get_type_info_map():
return _get_type_info_map_result
type_info_list = [
TypeInfo(
name='topic-star',
type='topic-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,#假的
get_data_func=TopicTransfer.get_topic_data,#假的
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='topic-star-routing',
type='topic-star-routing',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment