Commit 983ae5b2 authored by lixiaofang's avatar lixiaofang

add

parents 8c26d279 906b3056
...@@ -416,6 +416,7 @@ class ESPerform(object): ...@@ -416,6 +416,7 @@ class ESPerform(object):
"sort": [ "sort": [
{"_score": {"order": "desc"}}, {"_score": {"order": "desc"}},
{"latest_reply_time":{"order":"desc"}}, {"latest_reply_time":{"order":"desc"}},
{"create_time_val": {"order": "desc"}}, {"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}}, {"language_type": {"order": "asc"}},
], ],
......
...@@ -94,7 +94,7 @@ class CollectData(object): ...@@ -94,7 +94,7 @@ class CollectData(object):
# } # }
# redis_client.hmset(click_recommend_redis_key, click_redis_data_dict) # redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
tag_id_list = recommend_tag_list[0:100] tag_id_list = recommend_tag_list[0:20]
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id) topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
# redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key) # redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
...@@ -159,7 +159,6 @@ class CollectData(object): ...@@ -159,7 +159,6 @@ class CollectData(object):
click_topic_tag_list = list() click_topic_tag_list = list()
if "on_click_feed_topic_card" == raw_val_dict["type"]: if "on_click_feed_topic_card" == raw_val_dict["type"]:
topic_id = raw_val_dict["params"]["topic_id"]
device_id = raw_val_dict["device"]["device_id"] device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
......
...@@ -173,29 +173,6 @@ class TopicUtils(object): ...@@ -173,29 +173,6 @@ class TopicUtils(object):
} }
] ]
if user_id and user_id > 0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list) > 0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"constant_score": {"filter": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
if len(attention_tag_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"tag_list": attention_tag_list}}}},
"weight": 100
}
)
query_function_score = { query_function_score = {
"query": { "query": {
"bool": { "bool": {
...@@ -225,7 +202,7 @@ class TopicUtils(object): ...@@ -225,7 +202,7 @@ class TopicUtils(object):
] ]
} }
}, },
"score_mode": "sum", "score_mode": "max",
"boost_mode": "sum", "boost_mode": "sum",
"functions": functions_list "functions": functions_list
} }
...@@ -250,18 +227,11 @@ class TopicUtils(object): ...@@ -250,18 +227,11 @@ class TopicUtils(object):
{"terms": {"user_id": linucb_user_id_list}} {"terms": {"user_id": linucb_user_id_list}}
] ]
if query is not None: # 搜索帖子 if query is not None: # 搜索帖子
multi_fields = {
# 'description': 200,
'content': 300,
# 'name': 400,
'tag_name_list': 300,
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = { multi_match = {
'query': query, 'query': query,
'type': 'best_fields', 'type': 'best_fields',
'operator': 'or', 'operator': 'or',
'fields': query_fields, 'fields': ["content", "tag_name_list"],
} }
query_function_score["query"]["bool"]["should"] = [ query_function_score["query"]["bool"]["should"] = [
...@@ -290,16 +260,41 @@ class TopicUtils(object): ...@@ -290,16 +260,41 @@ class TopicUtils(object):
query_function_score["query"]["bool"]["filter"].append( query_function_score["query"]["bool"]["filter"].append(
{"term": {"content_level": 6}} {"term": {"content_level": 6}}
) )
q["query"]["function_score"] = query_function_score
if not disable_collpase:
q["collapse"] = {
"field": "user_id"
}
q["_source"] = { q["_source"] = {
"includes": ["id"] "includes": ["id"]
} }
if query is None: if query is None:
if user_id and user_id > 0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list) > 0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"constant_score": {"filter": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
if len(attention_tag_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"tag_list": attention_tag_list}}}},
"weight": 100
}
)
query_function_score["functions"] = functions_list
q["query"]["function_score"] = query_function_score
if not disable_collpase:
q["collapse"] = {
"field": "user_id"
}
q["sort"] = [ q["sort"] = [
# { # {
# "_script": { # "_script": {
...@@ -342,54 +337,84 @@ class TopicUtils(object): ...@@ -342,54 +337,84 @@ class TopicUtils(object):
} }
functions_list += [ functions_list += [
{ {
"weight": 300, "weight": 400,
"filter":{
"constant_score":{
"filter":{
"term": {"user_nick_name_pre": query.lower()}
}
}
}
},
{
"weight": 400,
"filter": { "filter": {
"bool": { "constant_score":{
"must": { "filter":{
"term": {"content_level": 6}, "bool": {
}, "must": {
"minimum_should_match": 1, "term": {"content_level": 6},
"should": [ },
{'multi_match': multi_match}, "minimum_should_match": 1,
{"term": {"tag_list": tag_id}}, "should": [
{"term": {"user_nick_name_pre": query.lower()}} {'match_phrase': {"content":query}},
] {'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
} }
} }
}, },
{ {
"weight": 200, "weight": 400,
"filter": { "filter": {
"bool": { "constant_score":{
"must": { "filter":{
"term": {"content_level": 5}, "bool": {
}, "must": {
"minimum_should_match": 1, "term": {"content_level": 5},
"should": [ },
{'multi_match': multi_match}, "minimum_should_match": 1,
{"term": {"tag_list": tag_id}}, "should": [
{"term": {"user_nick_name_pre": query.lower()}} {'match_phrase': {"content": query}},
] {'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
} }
} }
}, },
{ {
"weight": 100, "weight": 400,
"filter": { "filter": {
"bool": { "constant_score":{
"must": { "filter":{
"term": {"content_level": 4}, "bool": {
}, "must": {
"minimum_should_match": 1, "term": {"content_level": 4},
"should": [ },
{'multi_match': multi_match}, "minimum_should_match": 1,
{"term": {"tag_list": tag_id}}, "should": [
{"term": {"user_nick_name_pre": query.lower()}} {'match_phrase': {"content": query}},
] {'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
} }
} }
} }
] ]
query_function_score["functions"] = functions_list
q["query"]["function_score"] = query_function_score
q["sort"] = [ q["sort"] = [
# { # {
# "_script": { # "_script": {
...@@ -404,12 +429,17 @@ class TopicUtils(object): ...@@ -404,12 +429,17 @@ class TopicUtils(object):
# } # }
# }, # },
{ {
"offline_score": { "_score": {
"order": "desc" "order": "desc"
} }
}, },
{ {
"_score": { "latest_reply_time":{
"order": "desc"
}
},
{
"offline_score": {
"order": "desc" "order": "desc"
} }
} }
...@@ -709,10 +739,11 @@ class TopicUtils(object): ...@@ -709,10 +739,11 @@ class TopicUtils(object):
} }
# "includes": ["id", "pictorial_id", "user_id", "_score", "create_time", "content_level"] # "includes": ["id", "pictorial_id", "user_id", "_score", "create_time", "content_level"]
# q['sort'] = [ q['sort'] = [
{"latest_reply_time": {"order": "desc"}},
# {"content_level": {"order": "desc"}}, # {"content_level": {"order": "desc"}},
# {"create_time": {"order": "desc"}} # {"create_time": {"order": "desc"}}
# ] ]
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size, result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size,
routing=routing) routing=routing)
......
...@@ -40,7 +40,9 @@ def query_pictorial(query="", offset=0, size=10): ...@@ -40,7 +40,9 @@ def query_pictorial(query="", offset=0, size=10):
return {"group_ids": []} return {"group_ids": []}
def pictorial_sort1(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, size=10, device_id=""):
@bind("physical/search/pictorial_sort")
def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, size=10, device_id=""):
""" """
:remark 小组排序,缺少:前1天发评论人数*x :remark 小组排序,缺少:前1天发评论人数*x
:param user_id: :param user_id:
...@@ -678,7 +680,6 @@ def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1): ...@@ -678,7 +680,6 @@ def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
q["sort"] = process_sort(sort_type) q["sort"] = process_sort(sort_type)
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size) result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
result_dicts = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
if len(result_dict["hits"]) > 0: if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]: for item in result_dict["hits"]:
...@@ -692,6 +693,7 @@ def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1): ...@@ -692,6 +693,7 @@ def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
pictorial_ids_list_all = json.loads(str(redis_field_val_list, encoding="utf-8")) pictorial_ids_list_all = json.loads(str(redis_field_val_list, encoding="utf-8"))
pictorial_ids_list = pictorial_ids_list_all[offset:offset + size] pictorial_ids_list = pictorial_ids_list_all[offset:offset + size]
return {"pictorial_activity_sort": pictorial_ids_list} return {"pictorial_activity_sort": pictorial_ids_list}
except: except:
...@@ -833,6 +835,7 @@ def get_pictorial_activeity_sort(activity_id=-1, pictorial_id=-1): ...@@ -833,6 +835,7 @@ def get_pictorial_activeity_sort(activity_id=-1, pictorial_id=-1):
pictorial_ids_list = pictorial_activity_sort(activity_id=activity_id, size=999, offset=0, sort_type=1) pictorial_ids_list = pictorial_activity_sort(activity_id=activity_id, size=999, offset=0, sort_type=1)
data = pictorial_ids_list.get("pictorial_activity_sort", []) data = pictorial_ids_list.get("pictorial_activity_sort", [])
if data: if data:
pictorial_ids = [] pictorial_ids = []
for item in data: for item in data:
......
...@@ -29,6 +29,30 @@ def query_tag(query,offset,size): ...@@ -29,6 +29,30 @@ def query_tag(query,offset,size):
try: try:
if query: if query:
query = query.lower() query = query.lower()
pre_q = {
"query":{
"bool": {
"must":[
{"term":{"name_pre": query}},
{"term":{"is_online": True}}
]
}
},
"_source": {
"include": ["id", "name", "is_deleted", "is_online"]
}
}
ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=pre_q,offset=0,size=1)
if len(result_dict["hits"])>0:
hitLight = u'<ems>%s</ems>' % query
result_dict["hits"][0]["_source"]["highlight"] = result_dict["hits"][0]["_source"]["name"].replace(query, hitLight)
ret_list.append(result_dict["hits"][0]["_source"])
size -= 1
q = { q = {
"suggest":{ "suggest":{
"tips-suggest":{ "tips-suggest":{
...@@ -51,7 +75,6 @@ def query_tag(query,offset,size): ...@@ -51,7 +75,6 @@ def query_tag(query,offset,size):
} }
} }
ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=q,offset=offset,size=size,is_suggest_request=True) result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=q,offset=offset,size=size,is_suggest_request=True)
for tips_item in result_dict["suggest"]["tips-suggest"]: for tips_item in result_dict["suggest"]["tips-suggest"]:
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
] ]
}, },
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"name_pre":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"tag_type":{"type":"long"}, "tag_type":{"type":"long"},
"collection":{"type":"long"}, "collection":{"type":"long"},
"is_ai":{"type":"long"}, "is_ai":{"type":"long"},
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
] ]
}, },
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"name_pre":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"tag_type":{"type":"long"}, "tag_type":{"type":"long"},
"collection":{"type":"long"}, "collection":{"type":"long"},
"is_ai":{"type":"long"}, "is_ai":{"type":"long"},
...@@ -28,4 +29,4 @@ ...@@ -28,4 +29,4 @@
"is_deleted":{"type":"keyword"}, "is_deleted":{"type":"keyword"},
"near_new_topic_num":{"type":"long","store": true} "near_new_topic_num":{"type":"long","store": true}
} }
} }
\ No newline at end of file
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名 "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"useful_tag_list":{"type":"long"},//有用标签属性 "useful_tag_list":{"type":"long"},//有用标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"}, "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
......
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名 "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"useful_tag_list":{"type":"long"},//有用标签属性 "useful_tag_list":{"type":"long"},//有用标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"}, "latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
......
...@@ -420,3 +420,4 @@ class Reply(models.Model): ...@@ -420,3 +420,4 @@ class Reply(models.Model):
create_time = models.DateTimeField(verbose_name=u'创建时间') create_time = models.DateTimeField(verbose_name=u'创建时间')
is_deleted = models.BooleanField(verbose_name=u'是否删除') is_deleted = models.BooleanField(verbose_name=u'是否删除')
is_online = models.BooleanField(verbose_name=u'是否上线') is_online = models.BooleanField(verbose_name=u'是否上线')
...@@ -144,49 +144,13 @@ class TypeInfo(object): ...@@ -144,49 +144,13 @@ class TypeInfo(object):
else: else:
if data: if data:
if self.type == "topic": if self.type == "topic":
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "include":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
# else:
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"]) ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if ori_topic_star: if ori_topic_star:
ori_topic_star = str(ori_topic_star, encoding="utf-8") ori_topic_star = str(ori_topic_star, encoding="utf-8")
# else:
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "includes":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
if not ori_topic_star: if not ori_topic_star:
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"]) redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
else: else:
# int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
int_ori_topic_star = int(ori_topic_star) int_ori_topic_star = int(ori_topic_star)
if int_ori_topic_star != data["content_level"]: if int_ori_topic_star != data["content_level"]:
old_data = copy.deepcopy(data) old_data = copy.deepcopy(data)
...@@ -199,17 +163,15 @@ class TypeInfo(object): ...@@ -199,17 +163,15 @@ class TypeInfo(object):
if int_ori_topic_star >= 4: if int_ori_topic_star >= 4:
topic_data_high_star_list.append(old_data) topic_data_high_star_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"]) redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
if data["content_level"] and int(data["content_level"]) >= 4: if data["content_level"] and int(data["content_level"]) >= 4:
topic_data_high_star_list.append(data) topic_data_high_star_list.append(data)
data_list.append(data) elif self.type == "tag" or self.type == "tag_v1":
(res, begin_res) = data
data_list.append(res)
data_list.append(begin_res)
else:
data_list.append(data)
return (data_list, topic_data_high_star_list) return (data_list, topic_data_high_star_list)
......
...@@ -11,7 +11,7 @@ from trans2es.models.topic import Topic ...@@ -11,7 +11,7 @@ from trans2es.models.topic import Topic
from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation
import datetime import datetime
from django.conf import settings from django.conf import settings
import copy
class TagTransfer(object): class TagTransfer(object):
...@@ -33,21 +33,27 @@ class TagTransfer(object): ...@@ -33,21 +33,27 @@ class TagTransfer(object):
res["id"] = instance.id res["id"] = instance.id
begin_tag_name_terms_list = list()
tag_name_terms_list = list() tag_name_terms_list = list()
for i in range(len(instance.name)): for i in range(len(instance.name)):
for j in range(i, len(instance.name) + 1): for j in range(i, len(instance.name) + 1):
name_term = instance.name[i:j].strip() name_term = instance.name[i:j].strip()
if name_term: if name_term:
tag_name_terms_list.append(name_term.lower()) if i == 0:
begin_tag_name_terms_list.append(name_term.lower())
else:
tag_name_terms_list.append(name_term.lower())
res["suggest"] = { res["suggest"] = {
"input": tag_name_terms_list, "input":tag_name_terms_list,
"contexts": { "weight": 1,
"contexts":{
"is_online": [instance.is_online], "is_online": [instance.is_online],
"is_deleted": [instance.is_deleted] "is_deleted": [instance.is_deleted]
} }
} }
res["name"] = instance.name res["name"] = instance.name
res["name_pre"] = instance.name
res["is_online"] = instance.is_online res["is_online"] = instance.is_online
res["is_deleted"] = instance.is_deleted res["is_deleted"] = instance.is_deleted
...@@ -82,7 +88,13 @@ class TagTransfer(object): ...@@ -82,7 +88,13 @@ class TagTransfer(object):
res["collection"] = instance.collection res["collection"] = instance.collection
res["is_ai"] = instance.is_ai res["is_ai"] = instance.is_ai
res["is_own"] = instance.is_own res["is_own"] = instance.is_own
return res
begin_res = copy.deepcopy(res)
begin_res["id"] = begin_res["id"] + 600000000
begin_res["suggest"]["input"] = begin_tag_name_terms_list
begin_res["suggest"]["weight"] = 10
return (res, begin_res)
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict() return (dict(), dict())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment