Skip to content

P
physical
Commit 9563a254 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'master' into 'test' 

Master

See merge request !302
parents 757ba7f5 5f2f827c
Hide whitespace changes
Inline Side-by-side
Showing with 121 additions and 37 deletions
libs/es.py
View file @ 9563a254
...@@ -154,7 +154,9 @@ class ESPerform(object): ...@@ -154,7 +154,9 @@ class ESPerform(object):
bulk_actions = [] bulk_actions = []
if sub_index_name == "topic" or sub_index_name == "topic-star-routing": if sub_index_name=="topic" or \
sub_index_name=="topic-star-routing" or \
sub_index_name=="topic-high-star":
for data in data_list: for data in data_list:
if data: if data:
bulk_actions.append({ bulk_actions.append({
......
search/utils/topic.py
View file @ 9563a254
...@@ -174,6 +174,14 @@ class TopicUtils(object): ...@@ -174,6 +174,14 @@ class TopicUtils(object):
} }
}, },
"weight": 60 "weight": 60
},
{
"filter": {
"term": {
"content_level": 6
}
},
"weight": 200
} }
] ]
...@@ -308,24 +316,28 @@ class TopicUtils(object): ...@@ -308,24 +316,28 @@ class TopicUtils(object):
"includes": ["id", "pictorial_id", "offline_score", "user_id", "edit_tag_list"] "includes": ["id", "pictorial_id", "offline_score", "user_id", "edit_tag_list"]
} }
q["sort"] = [ q["sort"] = [
{
"_script": {
"type": "number",
"script": {
"lang": "expression",
"source": "_score+doc['offline_score']"
# "lang":"painless",
# "source":"_score+params._source.offline_score"
},
"order": "desc"
}
},
# { # {
# "offline_score":{ # "_script": {
# "type": "number",
# "script": {
# "lang": "expression",
# "source": "_score+doc['offline_score']"
# # "lang":"painless",
# # "source":"_score+params._source.offline_score"
# },
# "order": "desc" # "order": "desc"
# } # }
# }, # },
"_score" {
"_score": {
"order": "desc"
}
},
{
"offline_score":{
"order": "desc"
}
}
] ]
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q, result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name=index_type, query_body=q,
offset=offset, size=size,routing=routing) offset=offset, size=size,routing=routing)
......
search/views/topic.py
View file @ 9563a254
...@@ -114,7 +114,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -114,7 +114,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size, rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type, single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list, filter_topic_id_list=have_read_topic_id_list,
user_similar_score_list=user_similar_score_redis_list,index_type="topic",routing="4,5,6",attention_tag_list=attention_tag_list) user_similar_score_list=user_similar_score_redis_list,index_type="topic-high-star",routing="4,5,6",attention_tag_list=attention_tag_list)
if len(recommend_topic_list) == 6 and query is None: if len(recommend_topic_list) == 6 and query is None:
if (size < 11): if (size < 11):
...@@ -290,7 +290,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -290,7 +290,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
result = list() result = list()
if len(topic_tag_list) != 0: if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj, topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
index_type="topic",routing="4,5,6",topic_tag_list = topic_tag_list) index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_size = len(topic_tag_result) topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result) have_read_topic_list.extend(topic_tag_result)
else: else:
...@@ -300,17 +300,17 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -300,17 +300,17 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if topic_user_id != -1: if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list, topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj, size, es_cli_obj,
index_type="topic", routing="4,5,6",topic_user_id = topic_user_id index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
) )
topic_user_size = len(topic_user_result) topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result) have_read_topic_list.extend(topic_user_result)
if topic_user_size < size: if topic_user_size < size:
size = size - topic_user_size size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list, have_read_topic_list,
size, es_cli_obj, size, es_cli_obj,
index_type="topic", routing="4,5,6") index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result) have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key) # have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else [] # have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
......
trans2es/mapping/topic-high-star.json
View file @ 9563a254
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
} }
} }
trans2es/mapping/topic-star-routing.json
View file @ 9563a254
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
} }
} }
trans2es/mapping/topic-star.json 0 → 100644
View file @ 9563a254
{
"dynamic":"strict",
"properties": {
"id":{"type":"long"},
"content_level":{"type":"text"}
}
}
trans2es/mapping/topic.json
View file @ 9563a254
...@@ -52,6 +52,7 @@ ...@@ -52,6 +52,7 @@
"search_analyzer": "gm_default_index" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"} //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
} }
} }
trans2es/type_info.py
View file @ 9563a254
...@@ -84,7 +84,7 @@ class TypeInfo(object): ...@@ -84,7 +84,7 @@ class TypeInfo(object):
def bulk_get_data(self, instance_iterable): def bulk_get_data(self, instance_iterable):
data_list = [] data_list = []
# 4，5星帖子单独索引 # 4星以上帖子单独索引
topic_data_high_star_list = list() topic_data_high_star_list = list()
if self.batch_get_data_func: if self.batch_get_data_func:
...@@ -144,23 +144,73 @@ class TypeInfo(object): ...@@ -144,23 +144,73 @@ class TypeInfo(object):
else: else:
if data: if data:
if self.type == "topic": if self.type == "topic":
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "include":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
# else:
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"]) ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if ori_topic_star:
ori_topic_star = str(ori_topic_star, encoding="utf-8")
else:
q = {
"query":{
"term":{
"id": data["id"]
}
},
"_source":{
"includes":["content_level"]
}
}
search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
if len(search_results["hits"]) > 0:
ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
if not ori_topic_star: if not ori_topic_star:
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"]) redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
else: else:
int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8")) # int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
int_ori_topic_star = int(ori_topic_star)
if int_ori_topic_star != data["content_level"]: if int_ori_topic_star != data["content_level"]:
old_data = copy.deepcopy(data) old_data = copy.deepcopy(data)
old_data["is_online"] = False old_data["is_online"] = False
old_data["is_deleted"] = True old_data["is_deleted"] = True
old_data["content_level"] = int_ori_topic_star old_data["content_level"] = int_ori_topic_star
old_data["is_history"] = True
data_list.append(old_data) data_list.append(old_data)
if int_ori_topic_star>=4:
topic_data_high_star_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"]) redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
data_list.append(data) # data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
# if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
# topic_data_high_star_list.append(data) if data["content_level"] and int(data["content_level"])>=4:
topic_data_high_star_list.append(data)
data_list.append(data)
return (data_list,topic_data_high_star_list) return (data_list,topic_data_high_star_list)
...@@ -234,13 +284,13 @@ class TypeInfo(object): ...@@ -234,13 +284,13 @@ class TypeInfo(object):
es=es, es=es,
) )
# # 同时写4星及以上的帖子 # 同时写4星及以上的帖子
# if len(topic_data_high_star_list)>0: if len(topic_data_high_star_list)>0:
# self.elasticsearch_bulk_insert_data( self.elasticsearch_bulk_insert_data(
# sub_index_name="topic-high-star", sub_index_name="topic-high-star",
# data_list=topic_data_high_star_list, data_list=topic_data_high_star_list,
# es=es, es=es,
# ) )
end = time.time() end = time.time()
time3=end-begin time3=end-begin
...@@ -301,6 +351,16 @@ def get_type_info_map(): ...@@ -301,6 +351,16 @@ def get_type_info_map():
return _get_type_info_map_result return _get_type_info_map_result
type_info_list = [ type_info_list = [
TypeInfo(
name='topic-star',
type='topic-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,#假的
get_data_func=TopicTransfer.get_topic_data,#假的
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo( TypeInfo(
name='topic-star-routing', name='topic-star-routing',
type='topic-star-routing', type='topic-star-routing',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment