Skip to content

P
physical
Commit 30991868 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'test' into 'master' 

Test

See merge request !228
parents cad53f5b 099832ab
Hide whitespace changes
Inline Side-by-side
Showing with 135 additions and 42 deletions
.idea/codeStyles/Project.xml
View file @ 30991868
...@@ -24,5 +24,29 @@ ...@@ -24,5 +24,29 @@
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" /> <option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings> </formatting-settings>
</DBN-SQL> </DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme> </code_scheme>
</component> </component>
\ No newline at end of file
search/utils/group.py
View file @ 30991868
...@@ -9,7 +9,7 @@ from libs.es import ESPerform ...@@ -9,7 +9,7 @@ from libs.es import ESPerform
class GroupUtils(object): class GroupUtils(object):
@classmethod @classmethod
def get_group_query_result(cls,query,offset,size): def get_group_query_result(cls, query, offset, size):
try: try:
q = dict() q = dict()
...@@ -38,49 +38,84 @@ class GroupUtils(object): ...@@ -38,49 +38,84 @@ class GroupUtils(object):
} }
} }
q["_source"] = { q["_source"] = {
"includes":["id"] "includes": ["id"]
} }
return ESPerform.get_search_results(ESPerform.get_cli(), "group", q, offset, size) return ESPerform.get_search_results(ESPerform.get_cli(), "group", q, offset, size)
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0, "hits":[]} return {"total_count": 0, "hits": []}
@classmethod @classmethod
def get_hot_pictorial_recommend_result_list(cls,offset,size,es_cli_obj=None): def get_hot_pictorial_recommend_result_list(cls, offset, size, es_cli_obj=None):
try: try:
if not es_cli_obj: if not es_cli_obj:
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
q = dict() q = dict()
q["query"] = { q["query"] = {
"bool":{ "bool": {
"must":[ "must": [
{"term": {"is_online": True}}, {"term": {"is_online": True}},
{"term":{"is_deleted": False}} {"term": {"is_deleted": False}}
] ]
} }
} }
q["sort"] = [ q["sort"] = [
{"high_quality_topic_num":{"order":"desc"}} {"high_quality_topic_num": {"order": "desc"}}
] ]
q["_source"] = { q["_source"] = {
"includes":["id"] "includes": ["id"]
} }
MAX_LOAD = 1000
result_dict = ESPerform.get_search_results(es_cli_obj,"pictorial",q,offset,size) sizes = max(size, MAX_LOAD)
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset=0, size=sizes)
pictorial_ids_list = [] pictorial_ids_list = []
if len(result_dict["hits"]) > 0: if len(result_dict["hits"]) > 0:
pictorial_ids_list = [item["_source"]["id"] for item in result_dict["hits"]] pictorial_ids_list = [item["_source"]["id"] for item in result_dict["hits"]]
return pictorial_ids_list pictorial_del_ids_list = []
if len(pictorial_ids_list) > 0:
for id in pictorial_ids_list:
q = {
"query": {
"bool": {
"must": [{
"term": {
"pictorial_id": id
}
}, {
"terms": {
"content_level": [3, 4, 5, 0]
}
}, {
"term": {
"is_online": True
}
}]
}
}
}
topic_ids_list = []
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset=0, size=sizes)
if len(result_dict["hits"]) > 0:
topic_ids_list = [item["_source"]["id"] for item in result_dict["hits"]]
if len(topic_ids_list) > 5:
pictorial_del_ids_list.append(id)
return pictorial_del_ids_list[offset:(offset+size)]
else:
return []
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return []
@classmethod @classmethod
def get_user_attention_pictorial_list(cls,user_id,offset=0,size=10,es_cli_obj=None): def get_user_attention_pictorial_list(cls, user_id, offset=0, size=10, es_cli_obj=None):
""" """
:remark: 获取用户关注小组列表 :remark: 获取用户关注小组列表
:return: :return:
...@@ -91,29 +126,67 @@ class GroupUtils(object): ...@@ -91,29 +126,67 @@ class GroupUtils(object):
q = dict() q = dict()
q["query"] = { q["query"] = {
"bool":{ "bool": {
"must":[ "must": [
{"term":{"is_online": True}}, {"term": {"is_online": True}},
{"term":{"user_id":user_id}}, {"term": {"user_id": user_id}},
{"term":{"is_deleted":False}} {"term": {"is_deleted": False}}
] ]
} }
} }
q["_source"] = { q["_source"] = {
"includes":["attention_pictorial_id_list"] "includes": ["attention_pictorial_id_list"]
} }
MAX_LOAD = 1000
sizes = max(size, MAX_LOAD)
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset=0, size=sizes)
pictorial_ids_list = []
if len(result_dict["hits"]) > 0:
pictorial_ids_list = result_dict["hits"][0]["_source"]["attention_pictorial_id_list"]
pictorial_del_ids_list = []
if len(pictorial_ids_list) > 0:
for id in pictorial_ids_list:
q = {
"query": {
"bool": {
"must": [{
"term": {
"pictorial_id": id
}
}, {
"terms": {
"content_level": [3, 4, 5, 0]
}
}, {
"term": {
"is_online": True
}
}]
}
}
}
topic_ids_list = []
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset=0, size=sizes)
if len(result_dict["hits"]) > 0:
topic_ids_list = [item["_source"]["id"] for item in result_dict["hits"]]
if len(topic_ids_list) > 5:
pictorial_del_ids_list.append(id)
return pictorial_del_ids_list[offset:(offset+size)]
result_dict = ESPerform.get_search_results(es_cli_obj,"user",q,offset,size)
if len(result_dict["hits"])>0:
return result_dict["hits"][0]["_source"]["attention_pictorial_id_list"]
else: else:
return [] return []
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return []
@classmethod @classmethod
def get_pictorial_ids_by_aggs(cls,pictorial_ids_list,es_cli_obj=None): def get_pictorial_ids_by_aggs(cls, pictorial_ids_list, es_cli_obj=None):
""" """
:remark:聚合查询获取小组列表 :remark:聚合查询获取小组列表
:param group_id_list: :param group_id_list:
...@@ -124,34 +197,34 @@ class GroupUtils(object): ...@@ -124,34 +197,34 @@ class GroupUtils(object):
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
q = dict() q = dict()
q["size"]=0 q["size"] = 0
q["query"] = { q["query"] = {
"terms":{ "terms": {
"pictorial_id":pictorial_ids_list "pictorial_id": pictorial_ids_list
} }
} }
q["aggs"] = { q["aggs"] = {
"pictorial_ids":{ "pictorial_ids": {
"terms":{ "terms": {
"field":"pictorial_id" "field": "pictorial_id"
}, },
"aggs":{ "aggs": {
"max_date":{ "max_date": {
"max":{ "max": {
"field":"update_time_val" "field": "update_time_val"
} }
} }
} }
} }
} }
result_dict = ESPerform.get_search_results(es_cli_obj,"topic",q,aggregations_query=True) result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, aggregations_query=True)
buckets_list = result_dict["aggregations"]["pictorial_ids"]["buckets"] buckets_list = result_dict["aggregations"]["pictorial_ids"]["buckets"]
sorted_buckets_list = sorted(buckets_list,key=lambda item:item["max_date"]["value"],reverse=True) sorted_buckets_list = sorted(buckets_list, key=lambda item: item["max_date"]["value"], reverse=True)
sorted_pictorial_id_list = [item["key"] for item in sorted_buckets_list] sorted_pictorial_id_list = [item["key"] for item in sorted_buckets_list]
return sorted_pictorial_id_list return sorted_pictorial_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return []
\ No newline at end of file
search/views/group.py
View file @ 30991868
...@@ -53,7 +53,6 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, ...@@ -53,7 +53,6 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
if sort_type == GroupSortTypes.HOT_RECOMMEND: if sort_type == GroupSortTypes.HOT_RECOMMEND:
pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj) pictorial_ids_list = GroupUtils.get_hot_pictorial_recommend_result_list(offset, size, es_cli_obj)
return {"pictorial_recommend_ids": pictorial_ids_list} return {"pictorial_recommend_ids": pictorial_ids_list}
elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND: elif sort_type == GroupSortTypes.ATTENTION_RECOMMEND:
...@@ -61,6 +60,7 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, ...@@ -61,6 +60,7 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
es_cli_obj=es_cli_obj) es_cli_obj=es_cli_obj)
if len(attention_pictorial_list) == 0: if len(attention_pictorial_list) == 0:
return {"pictorial_recommend_ids": []} return {"pictorial_recommend_ids": []}
else: else:
attention_pictorial_id_list = [item["pictorial_id"] for item in attention_pictorial_list] attention_pictorial_id_list = [item["pictorial_id"] for item in attention_pictorial_list]
sorted_pictorial_ids_list = GroupUtils.get_pictorial_ids_by_aggs(attention_pictorial_id_list, sorted_pictorial_ids_list = GroupUtils.get_pictorial_ids_by_aggs(attention_pictorial_id_list,
...@@ -73,7 +73,6 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0, ...@@ -73,7 +73,6 @@ def pictorial_sort(user_id=-1, sort_type=GroupSortTypes.HOT_RECOMMEND, offset=0,
for item in sorted_attention_pictorial_list: for item in sorted_attention_pictorial_list:
if item["pictorial_id"] not in pictorial_recommend_ids_list: if item["pictorial_id"] not in pictorial_recommend_ids_list:
pictorial_recommend_ids_list.append(item["pictorial_id"]) pictorial_recommend_ids_list.append(item["pictorial_id"])
return {"pictorial_recommend_ids": pictorial_recommend_ids_list[offset:(offset + size)]} return {"pictorial_recommend_ids": pictorial_recommend_ids_list[offset:(offset + size)]}
except: except:
...@@ -106,7 +105,6 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -106,7 +105,6 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"includes": ["id", "pictorial_id", "tag_list"] "includes": ["id", "pictorial_id", "tag_list"]
} }
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size) result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size)
logging.info("get result_dict:%s" % result_dict)
pict_pictorial_ids_list = [] pict_pictorial_ids_list = []
topic_tag_list = [] topic_tag_list = []
pictorial_id_list = [] pictorial_id_list = []
......
trans2es/mapping/pictorial.json
View file @ 30991868
...@@ -16,6 +16,5 @@ ...@@ -16,6 +16,5 @@
"tag_id":{"type":"long"}, "tag_id":{"type":"long"},
"tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"topic_id_list":{"type":"long"} "topic_id_list":{"type":"long"}
} }
} }
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment