Skip to content

P
physical
Commit 983ae5b2 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parents 8c26d279 906b3056
Show whitespace changes
Inline Side-by-side
Showing with 136 additions and 100 deletions
libs/es.py
View file @ 983ae5b2
......@@ -416,6 +416,7 @@ class ESPerform(object):
"sort": [
{"_score": {"order": "desc"}},
{"latest_reply_time":{"order":"desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
......
linucb/views/collect_data.py
View file @ 983ae5b2
......@@ -94,7 +94,7 @@ class CollectData(object):
# }
# redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
tag_id_list = recommend_tag_list[0:100]
tag_id_list = recommend_tag_list[0:20]
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
# redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
......@@ -159,7 +159,6 @@ class CollectData(object):
click_topic_tag_list = list()
if "on_click_feed_topic_card" == raw_val_dict["type"]:
topic_id = raw_val_dict["params"]["topic_id"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
......
search/utils/topic.py
View file @ 983ae5b2
......@@ -173,29 +173,6 @@ class TopicUtils(object):
}
]
if user_id and user_id > 0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list) > 0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"constant_score": {"filter": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
if len(attention_tag_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"tag_list": attention_tag_list}}}},
"weight": 100
}
)
query_function_score = {
"query": {
"bool": {
......@@ -225,7 +202,7 @@ class TopicUtils(object):
]
}
},
"score_mode": "sum",
"score_mode": "max",
"boost_mode": "sum",
"functions": functions_list
}
......@@ -250,18 +227,11 @@ class TopicUtils(object):
{"terms": {"user_id": linucb_user_id_list}}
]
if query is not None: # 搜索帖子
multi_fields = {
# 'description': 200,
'content': 300,
# 'name': 400,
'tag_name_list': 300,
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'best_fields',
'operator': 'or',
'fields': query_fields,
'fields': ["content", "tag_name_list"],
}
query_function_score["query"]["bool"]["should"] = [
......@@ -290,16 +260,41 @@ class TopicUtils(object):
query_function_score["query"]["bool"]["filter"].append(
{"term": {"content_level": 6}}
)
q["query"]["function_score"] = query_function_score
if not disable_collpase:
q["collapse"] = {
"field": "user_id"
}
q["_source"] = {
"includes": ["id"]
}
if query is None:
if user_id and user_id > 0:
redis_key_prefix = "physical:user_similar:participant_user_id:"
similar_redis_key = redis_key_prefix + str(user_id)
redis_user_similar_data = redis_client.get(similar_redis_key)
user_similar_list = json.loads(redis_user_similar_data) if redis_user_similar_data else []
if len(user_similar_list) > 0:
functions_list.extend(user_similar_list)
if len(attention_user_id_list) > 0:
functions_list.append(
{
"filter": {"constant_score": {"filter": {"terms": {"user_id": attention_user_id_list}}}},
"weight": 100,
}
)
if len(attention_tag_list) > 0:
functions_list.append(
{
"filter": {"bool": {
"should": {"terms": {"tag_list": attention_tag_list}}}},
"weight": 100
}
)
query_function_score["functions"] = functions_list
q["query"]["function_score"] = query_function_score
if not disable_collpase:
q["collapse"] = {
"field": "user_id"
}
q["sort"] = [
# {
# "_script": {
......@@ -342,54 +337,84 @@ class TopicUtils(object):
}
functions_list += [
{
"weight": 300,
"weight": 400,
"filter":{
"constant_score":{
"filter":{
"term": {"user_nick_name_pre": query.lower()}
}
}
}
},
{
"weight": 400,
"filter": {
"constant_score":{
"filter":{
"bool": {
"must": {
"term": {"content_level": 6},
},
"minimum_should_match": 1,
"should": [
{'multi_match': multi_match},
{'match_phrase': {"content":query}},
{'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
}
}
},
{
"weight": 200,
"weight": 400,
"filter": {
"constant_score":{
"filter":{
"bool": {
"must": {
"term": {"content_level": 5},
},
"minimum_should_match": 1,
"should": [
{'multi_match': multi_match},
{'match_phrase': {"content": query}},
{'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
}
}
},
{
"weight": 100,
"weight": 400,
"filter": {
"constant_score":{
"filter":{
"bool": {
"must": {
"term": {"content_level": 4},
},
"minimum_should_match": 1,
"should": [
{'multi_match': multi_match},
{'match_phrase': {"content": query}},
{'match_phrase': {"tag_name_list": query}},
# {'multi_match': multi_match},
{"term": {"tag_list": tag_id}},
{"term": {"user_nick_name_pre": query.lower()}}
]
}
}
}
}
}
]
query_function_score["functions"] = functions_list
q["query"]["function_score"] = query_function_score
q["sort"] = [
# {
# "_script": {
......@@ -404,12 +429,17 @@ class TopicUtils(object):
# }
# },
{
"offline_score": {
"_score": {
"order": "desc"
}
},
{
"_score": {
"latest_reply_time":{
"order": "desc"
}
},
{
"offline_score": {
"order": "desc"
}
}
......@@ -709,10 +739,11 @@ class TopicUtils(object):
}
# "includes": ["id", "pictorial_id", "user_id", "_score", "create_time", "content_level"]
# q['sort'] = [
q['sort'] = [
{"latest_reply_time": {"order": "desc"}},
# {"content_level": {"order": "desc"}},
# {"create_time": {"order": "desc"}}
# ]
]
result_dict = ESPerform.get_search_results(es_cli_obj, sub_index_name=index_type, query_body=q, size=size,
routing=routing)
......
search/views/group.py
View file @ 983ae5b2
......@@ -40,7 +40,9 @@ def query_pictorial(query="", offset=0, size=10):
return {"group_ids": []}
def pictorial_sort1(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, size=10, device_id=""):
@bind("physical/search/pictorial_sort")
def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, size=10, device_id=""):
"""
:remark 小组排序,缺少：前1天发评论人数＊x
:param user_id:
......@@ -678,7 +680,6 @@ def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
q["sort"] = process_sort(sort_type)
es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
result_dicts = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]:
......@@ -692,6 +693,7 @@ def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
pictorial_ids_list_all = json.loads(str(redis_field_val_list, encoding="utf-8"))
pictorial_ids_list = pictorial_ids_list_all[offset:offset + size]
return {"pictorial_activity_sort": pictorial_ids_list}
except:
......@@ -833,6 +835,7 @@ def get_pictorial_activeity_sort(activity_id=-1, pictorial_id=-1):
pictorial_ids_list = pictorial_activity_sort(activity_id=activity_id, size=999, offset=0, sort_type=1)
data = pictorial_ids_list.get("pictorial_activity_sort", [])
if data:
pictorial_ids = []
for item in data:
......
search/views/tag.py
View file @ 983ae5b2
......@@ -29,6 +29,30 @@ def query_tag(query,offset,size):
try:
if query:
query = query.lower()
pre_q = {
"query":{
"bool": {
"must":[
{"term":{"name_pre": query}},
{"term":{"is_online": True}}
]
}
},
"_source": {
"include": ["id", "name", "is_deleted", "is_online"]
}
}
ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=pre_q,offset=0,size=1)
if len(result_dict["hits"])>0:
hitLight = u'<ems>%s</ems>' % query
result_dict["hits"][0]["_source"]["highlight"] = result_dict["hits"][0]["_source"]["name"].replace(query, hitLight)
ret_list.append(result_dict["hits"][0]["_source"])
size -= 1
q = {
"suggest":{
"tips-suggest":{
......@@ -51,7 +75,6 @@ def query_tag(query,offset,size):
}
}
ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=q,offset=offset,size=size,is_suggest_request=True)
for tips_item in result_dict["suggest"]["tips-suggest"]:
......
trans2es/mapping/tag.json
View file @ 983ae5b2
......@@ -20,6 +20,7 @@
]
},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"name_pre":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"tag_type":{"type":"long"},
"collection":{"type":"long"},
"is_ai":{"type":"long"},
......
trans2es/mapping/tag_v1.json
View file @ 983ae5b2
......@@ -20,6 +20,7 @@
]
},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"name_pre":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"tag_type":{"type":"long"},
"collection":{"type":"long"},
"is_ai":{"type":"long"},
......
trans2es/mapping/topic-high-star.json
View file @ 983ae5b2
......@@ -17,6 +17,7 @@
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"useful_tag_list":{"type":"long"},//有用标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/mapping/topic.json
View file @ 983ae5b2
......@@ -17,6 +17,7 @@
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"useful_tag_list":{"type":"long"},//有用标签属性
"latest_reply_time":{"type":"date", "format":"date_time_no_millis"},
"edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/models/topic.py
View file @ 983ae5b2
......@@ -420,3 +420,4 @@ class Reply(models.Model):
create_time = models.DateTimeField(verbose_name=u'创建时间')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
is_online = models.BooleanField(verbose_name=u'是否上线')
trans2es/type_info.py
View file @ 983ae5b2
......@@ -144,49 +144,13 @@ class TypeInfo(object):
else:
if data:
if self.type == "topic":
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "include":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
# else:
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if ori_topic_star:
ori_topic_star = str(ori_topic_star, encoding="utf-8")
# else:
# q = {
# "query":{
# "term":{
# "id": data["id"]
# }
# },
# "_source":{
# "includes":["content_level"]
# }
# }
# search_results = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="topic-star",query_body=q,offset=0,size=1)
# if len(search_results["hits"]) > 0:
# ori_topic_star = search_results["hits"][0]["_source"]["content_level"]
if not ori_topic_star:
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
else:
# int_ori_topic_star = int(str(ori_topic_star, encoding="utf-8"))
int_ori_topic_star = int(ori_topic_star)
if int_ori_topic_star != data["content_level"]:
old_data = copy.deepcopy(data)
......@@ -199,16 +163,14 @@ class TypeInfo(object):
if int_ori_topic_star >= 4:
topic_data_high_star_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
# data_list = [
# {
# "id": data["id"],
# "content_level": data["content_level"]
# }
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
if data["content_level"] and int(data["content_level"]) >= 4:
topic_data_high_star_list.append(data)
elif self.type == "tag" or self.type == "tag_v1":
(res, begin_res) = data
data_list.append(res)
data_list.append(begin_res)
else:
data_list.append(data)
return (data_list, topic_data_high_star_list)
......
trans2es/utils/tag_transfer.py
View file @ 983ae5b2
......@@ -11,7 +11,7 @@ from trans2es.models.topic import Topic
from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation
import datetime
from django.conf import settings
import copy
class TagTransfer(object):
......@@ -33,21 +33,27 @@ class TagTransfer(object):
res["id"] = instance.id
begin_tag_name_terms_list = list()
tag_name_terms_list = list()
for i in range(len(instance.name)):
for j in range(i, len(instance.name) + 1):
name_term = instance.name[i:j].strip()
if name_term:
if i == 0:
begin_tag_name_terms_list.append(name_term.lower())
else:
tag_name_terms_list.append(name_term.lower())
res["suggest"] = {
"input": tag_name_terms_list,
"contexts": {
"input":tag_name_terms_list,
"weight": 1,
"contexts":{
"is_online": [instance.is_online],
"is_deleted": [instance.is_deleted]
}
}
res["name"] = instance.name
res["name_pre"] = instance.name
res["is_online"] = instance.is_online
res["is_deleted"] = instance.is_deleted
......@@ -82,7 +88,13 @@ class TagTransfer(object):
res["collection"] = instance.collection
res["is_ai"] = instance.is_ai
res["is_own"] = instance.is_own
return res
begin_res = copy.deepcopy(res)
begin_res["id"] = begin_res["id"] + 600000000
begin_res["suggest"]["input"] = begin_tag_name_terms_list
begin_res["suggest"]["weight"] = 10
return (res, begin_res)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
return (dict(), dict())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment