Skip to content

P
physical
Commit 5b5e6032 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parents 7c9943d3 5e5123c2
master deploy/like-prod deploy/like-stage deploy/like-test dev like-pre/r01 test
1 merge request!366Test
Show whitespace changes
Inline Side-by-side
Showing with 594 additions and 180 deletions
.idea/codeStyles/Project.xml
View file @ 5b5e6032
......@@ -24,5 +24,29 @@
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme>
</component>
\ No newline at end of file
app_conf.xml
View file @ 5b5e6032
......@@ -17,5 +17,6 @@
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
<element value="search.views.search_hotword"/>
</config>
</gm_rpcd_config>
libs/es.py
View file @ 5b5e6032
......@@ -155,9 +155,9 @@ class ESPerform(object):
bulk_actions = []
if sub_index_name=="topic" or \
sub_index_name=="topic-star-routing" or \
sub_index_name=="topic-high-star":
if sub_index_name == "topic" or \
sub_index_name == "topic-star-routing" or \
sub_index_name == "topic-high-star":
for data in data_list:
if data:
bulk_actions.append({
......@@ -232,7 +232,7 @@ class ESPerform(object):
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls,es_cli, sub_index_name, query_body):
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
......@@ -242,7 +242,7 @@ class ESPerform(object):
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
res = es_cli.indices.analyze(index=official_index_name,body=query_body)
res = es_cli.indices.analyze(index=official_index_name, body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -298,8 +298,8 @@ class ESPerform(object):
functions_list += [
{
"filter": {
"constant_score":{
"filter":{
"constant_score": {
"filter": {
"term": {"content_level": 6}}
}
},
......@@ -307,8 +307,8 @@ class ESPerform(object):
},
{
"filter": {
"constant_score":{
"filter":{
"constant_score": {
"filter": {
"term": {"content_level": 5}}
}
},
......@@ -316,8 +316,8 @@ class ESPerform(object):
},
{
"filter": {
"constant_score":{
"filter":{
"constant_score": {
"filter": {
"term": {"content_level": 4}}
}
},
......@@ -411,7 +411,7 @@ class ESPerform(object):
}
},
"_source": {
"include": ["id","user_id"]
"include": ["id", "user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
......@@ -429,7 +429,8 @@ class ESPerform(object):
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q,
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
......@@ -441,7 +442,7 @@ class ESPerform(object):
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list,topic_id_dict
return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
search/utils/group.py
View file @ 5b5e6032
......@@ -109,8 +109,6 @@ class GroupUtils(object):
"script": {
"lang": "expression",
"source": "_score+doc['offline_score']"
# "lang":"painless",
# "source":"_score+params._source.offline_score"
},
"order": "desc"
}
......@@ -214,3 +212,74 @@ class GroupUtils(object):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@classmethod
def get_search_pictorial_topic(cls, query, offset, size):
try:
q = dict()
multi_fields = {
'name': 4,
'description': 4,
'tag_name': 4
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'cross_fields',
'operator': 'and',
'fields': query_fields,
}
q['query'] = {
'bool': {
"must": [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"is_default": 0}},
{"range": {"topic_id_list": {"gte": 0}}},
{"term": {"is_cover": True}}
],
"should": [
{'multi_match': multi_match}
],
"minimum_should_match": 1
}
}
q["_source"] = {
"includes": ["id", "is_online", "is_deleted", "is_default", "topic_id_list", "is_cover", "offline_score",
"is_default"]
}
q["sort"] = [
{
"_script": {
"type": "number",
"script": {
"lang": "expression",
"source": "_score+doc['offline_score']"
},
"order": "desc"
}
},
{
"_score": {
"order": "desc"
}
}
]
q = {}
q["query"] = {
"term": {
"is_online": True
}
}
logging.info("get qqqqqqqq:%s" % q)
es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
logging.info("get result_dict:%s" % result_dict)
return result_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
search/views/group.py
View file @ 5b5e6032
......@@ -261,3 +261,24 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"pict_pictorial_ids_list": []}
@bind("physical/search/pictorial_topic_sort")
def search_physical(query="", offset=0, size=10):
"""
搜索页面搜索画报
:param query:
:param offset:
:param size:
:return:
1. query识别：允许模糊匹配
2. 召回部分：去掉无封面、无帖子、下线的、默认画报
3. rank部分：（和之前一样，下面只是粘贴过来）
"""
try:
result_dicts = GroupUtils.get_search_pictorial_topic(query, offset, size)
logging.info("result_dicts:%s" % result_dicts)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"search_topic_ids": []}
search/views/search_hotword.py 0 → 100644
View file @ 5b5e6032
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics
from trans2es.models.tag import RegisterShowTag, Tag
import random
@bind("physical/search/search_hotword")
def search_hotword(device_id=-1):
"""
:remark:搜索页的热门灵感
内容组成：搜索推荐热词register_show_tag
个性化标签physical:linucb:tag_recommend:device_id:
核心词，需要去重community_tag collection 1
①先从后台随机取最多6个搜索推荐热词，不够6个则有多少取多少
②然后随机取和推荐热词一样数量的linUCB标签，取不够数则取核心标签，linUCB+核心词去重后的数量要和搜索推荐热词的数量一样
③最后打乱顺序填入到热门灵感中，超过3行的标签弃掉，不够3行不用管
:param query:
:param offset:
:param size:
:return:
"""
try:
all_tag_name_list = set()
# 获取搜索推荐热词
results_registr_tag = RegisterShowTag.objects.filter(is_deleted=False, is_online=1).count()
results_tag = Tag.objects.filter(is_online=True, collection=1).count()
tag_id = random.sample(range(1, results_registr_tag), 1)
results_tag_chose = Tag.objects.filter(id__in=tag_id).values_list("name", flat=True)
for i in results_tag_chose:
all_tag_name_list.add(i)
# 获取个性化标签
linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
linucb_recommend_tag_data = redis_client.get(tag_recommend_redis_key)
linucb_recommend_tag_list = json.loads(linucb_recommend_tag_data) if linucb_recommend_tag_data else []
for item in linucb_recommend_tag_list:
results_tag_recommend = Tag.objects.filter(id=item, is_online=True).values_list("name", flat=True)
all_tag_name_list.add(results_tag_recommend[0])
if len(all_tag_name_list) == 12:
return {"recommend_tag_name": list(all_tag_name_list)}
logging.info("get all_tag_name_list02:%s" % all_tag_name_list)
# 取不够数则取核心标签
if len(all_tag_name_list) < 12:
for i in range(0, 12):
tag_id = random.sample(range(1, results_tag), 12 - len(all_tag_name_list))
results_tag_hexin = Tag.objects.filter(id__in=tag_id).values_list("name", flat=True)
if results_tag_hexin not in all_tag_name_list:
all_tag_name_list.add(results_tag_hexin[0])
if len(all_tag_name_list) >= 12:
return {"recommend_tag_name": list(all_tag_name_list)}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_tag_name": []}
search/views/topic.py
View file @ 5b5e6032
......@@ -15,7 +15,6 @@ from libs.es import ESPerform
from django.conf import settings
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try:
if user_id == -1:
......@@ -30,10 +29,11 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size,
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,
single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,index_type="topic",routing="4,5,6")
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = {
......@@ -48,27 +48,31 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1,useful_tag_list =[]):
query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1, useful_tag_list=[]):
try:
topic_star_routing = "6"
index_type = "topic-high-star"
device_redis_key = ""
if query is None:
if user_id>0:
if user_id > 0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(
query_type)
else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
topic_star_routing = "3,4,5,6"
index_type = "topic"
if user_id>0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
if user_id > 0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
......@@ -80,18 +84,17 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset>0: # 首次搜索时不需要过滤已读
if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
elif len(device_redis_key)>0:
elif len(device_redis_key) > 0:
redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list)
if redis_field_val_list[0]:
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset>0: # 首次搜索时不需要过滤已读
if offset > 0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list)
......@@ -104,7 +107,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6",useful_tag_list=useful_tag_list)
"topic-high-star", "6",
useful_tag_list=useful_tag_list)
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
......@@ -124,18 +128,20 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
logging.warning(
"type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0:
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
......@@ -143,8 +149,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list)>0:
size = size-len(recommend_topic_list)
if len(recommend_topic_list) > 0:
size = size - len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list()
......@@ -155,11 +161,15 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list()
if size>0:
if size > 0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size,query=query, query_type=query_type,
single_size=size, query=query,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
index_type=index_type, routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None:
# if (size < 11):
......@@ -176,7 +186,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list)-30000
cut_len = len(have_read_topic_id_list) - 30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list),
......@@ -185,13 +195,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60)
if usefulrecall != -1:
return recommend_topic_list,rank_topic_id_list,useful_topic_id_list
return recommend_topic_list, rank_topic_id_list, useful_topic_id_list
else:
return recommend_topic_list, rank_topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return [],[],[]
return [], [], []
else:
return [], []
......@@ -206,7 +216,10 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1,useful_tag_list=[]):
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[]):
"""
:remark:首页推荐，目前只推荐日记
:param session_id:
......@@ -239,25 +252,50 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset+size]
recommend_topic_ids = have_read_topic_id_list[offset:offset + size]
else:
recommend_topic_ids = have_read_topic_id_list[0:size]
else:
if usefulrecall != -1 and len(useful_tag_list)>0:
recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall,useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids}
else:
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]}
if usefulrecall != -1 and len(useful_tag_list) > 0:
recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
== == == =
if usefulrecall != -1:
recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id,
tag_id=0, offset=0,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
>> >> >> > hotword
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []}
else:
return {"linucb_topic_ids": [], "rank_topic_ids": []}
......@@ -302,11 +340,12 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
if not isinstance(device_id, str):
device_id = ""
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, query=query)
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size,
query=query,disable_collpase=True)
query=query, disable_collpase=True)
return {"recommend_topic_ids": rank_topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -328,9 +367,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if not isinstance(user_id, int):
user_id = -1
if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id)
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id)
else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id)
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id)
if int(offset) == 0:
have_read_topic_list = list()
# redis_dict = {
......@@ -343,7 +382,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list)
have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id)
......@@ -351,18 +390,23 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
topic_tag_result = list()
result = list()
if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="3,4,5,6",
topic_tag_list=topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
topic_tag_size = 0
if topic_tag_size <size:
if topic_tag_size < size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
index_type="topic",
routing="3,4,5,6",
topic_user_id=topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
......@@ -448,7 +492,7 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
(topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
(topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
return {
......@@ -472,10 +516,23 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try:
must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
must_topic_id_list=must_topic_id_list,index_type="topic",routing="4,5,6")
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,
single_size=size,
must_topic_id_list=must_topic_id_list, index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_topic_id": []}
@bind("physical/search/query_topic_search")
def query_topic_search(query=""):
"""
召回搜索帖子
当搜索的内容 完全匹配 用户昵称，且该用户为推荐用户时，帖子tab中也会展示用户栏
:param query:
:return:
"""
# 召回完全匹配的用户
search/views/user.py
View file @ 5b5e6032
......@@ -14,7 +14,7 @@ from libs.es import ESPerform
@bind("physical/search/recommend_user")
def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
def recommend_user(self_user_id, interesting_user_id, offset=0, size=10):
"""
:remark 点关注推荐用户
:param self_user_id:
......@@ -29,25 +29,29 @@ def recommend_user(self_user_id,interesting_user_id,offset=0,size=10):
if not isinstance(interesting_user_id, int):
interesting_user_id = -1
#获取es链接对象
# 获取es链接对象
es_cli_obj = ESPerform.get_cli()
#获取关注用户列表
(self_attention_user_id_list,recursion_attention_user_id_list) = UserUtils.get_attention_user_list([self_user_id,interesting_user_id],self_user_id,es_cli_obj)
# 获取关注用户列表
(self_attention_user_id_list, recursion_attention_user_id_list) = UserUtils.get_attention_user_list(
[self_user_id, interesting_user_id], self_user_id, es_cli_obj)
#去除自身及感兴趣的用户ID
# 去除自身及感兴趣的用户ID
self_attention_user_id_list.append(self_user_id)
self_attention_user_id_list.append(interesting_user_id)
recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,recursion_attention_user_id_list,offset,size,es_cli_obj)
recommend_user_list = UserUtils.get_recommend_user_list(self_attention_user_id_list,
recursion_attention_user_id_list, offset, size,
es_cli_obj)
return recommend_user_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
@bind("physical/search/batch_recommend_user")
def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10):
def batch_recommend_user(self_user_id, interesting_user_id_list, offset=0, size=10):
"""
:remark 点关注推荐用户 - 批量接口
:param self_user_id:
......@@ -57,23 +61,129 @@ def batch_recommend_user(self_user_id,interesting_user_id_list,offset=0,size=10)
:return:
"""
try:
if not isinstance(self_user_id,int):
if not isinstance(self_user_id, int):
self_user_id = -1
#获取es链接对象
# 获取es链接对象
es_cli_obj = ESPerform.get_cli()
#获取关注用户列表
(need_filter_attention_user_id_list, attention_user_dict_list,attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list,self_user_id,es_cli_obj)
# 获取关注用户列表
(need_filter_attention_user_id_list, attention_user_dict_list,
attention_user_id_list) = UserUtils.get_batch_attention_user_list(interesting_user_id_list, self_user_id,
es_cli_obj)
#去除自身及感兴趣的用户ID
# 去除自身及感兴趣的用户ID
need_filter_attention_user_id_list.append(self_user_id)
recommend_user_dict = UserUtils.get_batch_recommend_user_dict(need_filter_attention_user_id_list=need_filter_attention_user_id_list,attention_user_id_list=attention_user_id_list,attention_user_dict_list=attention_user_dict_list,self_user_id=self_user_id,offset=offset,size=size,es_cli_obj=es_cli_obj)
recommend_user_dict = UserUtils.get_batch_recommend_user_dict(
need_filter_attention_user_id_list=need_filter_attention_user_id_list,
attention_user_id_list=attention_user_id_list, attention_user_dict_list=attention_user_dict_list,
self_user_id=self_user_id, offset=offset, size=size, es_cli_obj=es_cli_obj)
logging.info("duan add,recommend_user_dict:%s" % str(recommend_user_dict))
return recommend_user_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {}
@bind("physical/search/search_user")
def search_user(query="", offset=0, size=10):
"""
:remark 搜索用户
:param query:
:param offset:
:param size:
:return:
搜索域：用户名称
1.识别query：允许模糊匹配
2.召回部分：全部 线上状态用户
3.rank部分：一级-是否推荐，二级-发帖数
"""
try:
es_cli_obj = ESPerform.get_cli()
q = {}
# 先获取完全匹配的用户ID
sheer_user_id = []
q["query"] = {
"bool": {
"must": [
{"term": {
"nick_name_pre": query
}
}, {
"term": {
"is_recommend": True
}
}
]
}
}
q["sort"] = {
"count_topic": {
"order": "desc"
}
}
# que = {"query": {"term": {"nick_name_pre": query}}}
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, 0, 10)
res = result_dict["hits"]
if len(res) > 0:
user_id = [item["_source"]["user_id"] for item in res]
sheer_user_id.append(user_id)
logging.info("get res:%s" % res)
# 获取模糊匹配的用户ID
multi_match = {
"fields": ["nick_name"],
"type": "cross_fields",
"operator": "and",
"query": query
}
q = {}
q["query"] = {
"bool": {
"must": [{
"multi_match": multi_match
}, {
"term": {
"is_online": True
}
}
]
}
}
q["sort"] = {
"is_recommend": {
"order": "desc"
},
"count_topic": {
"order": "desc"
},
}
logging.info("get q:%s" % q)
result_dict = ESPerform.get_search_results(es_cli_obj, "user", q, offset, size)
logging.info("get result_dict:%s" % result_dict)
search_user_id = []
res = result_dict["hits"]
if len(res) > 0:
search_user_id = [item["_source"]["user_id"] for item in res]
return {"sheer_user_id": sheer_user_id, "search_user_id": search_user_id}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"sheer_user_id": [], "search_user_id": []}
trans2es/mapping/pictorial.json
View file @ 5b5e6032
......@@ -18,6 +18,7 @@
"topic_id_list":{"type":"long"},
"effective":{"type":"boolean"},
"offline_score":{"type":"long"},
"is_default":{"type":"long"}
"is_default":{"type":"long"},
"is_cover":{"type":"boolean"}
}
}
\ No newline at end of file
trans2es/mapping/user.json
View file @ 5b5e6032
......@@ -4,6 +4,7 @@
"id":{"type":"long"},
"user_id":{"type":"long"},
"nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, //昵称
"nick_name_pre":{"type":"text","analyzer":"not_analyzed"}, //昵称
"profile_pic":{"type":"text"}, //头像
"gender":{"type":"integer"},
"is_online":{"type":"boolean"},//是否上线
......@@ -60,6 +61,7 @@
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
"update_time_val":{"type":"long"}
"update_time_val":{"type":"long"},
"count_topic":{"type":"long"}
}
}
\ No newline at end of file
trans2es/models/pictorial.py
View file @ 5b5e6032
......@@ -63,27 +63,27 @@ class Pictorial(models.Model):
def get_topic_id(self):
try:
topic_id_list = list(PictorialTopics.objects.filter(pictorial_id=self.id,is_online=True,is_deleted=False).values_list("topic_id", flat=True))
topic_id_list = list(
PictorialTopics.objects.filter(pictorial_id=self.id, is_online=True, is_deleted=False).values_list(
"topic_id", flat=True))
return topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_effective(self,topic_id_list):
def get_effective(self, topic_id_list):
try:
effective_num = 0
ret = False
for topic_id in topic_id_list:
topic_id_object = Topic.objects.filter(id=int(topic_id)).first()
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0,3,4,5]:
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5]:
effective_num += 1
if effective_num >= 5:
ret = True
break
return ret
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -109,7 +109,8 @@ class Pictorial(models.Model):
def get_tag_by_id(self):
try:
tag_id_list = list(PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True))
tag_id_list = list(
PictorialTag.objects.filter(pictorial_id=self.id, is_online=True).values_list("tag_id", flat=True))
return tag_id_list
except:
......@@ -125,6 +126,25 @@ class Pictorial(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_is_cover(self, topic_id_list):
try:
effective_num = 0
ret = False
for topic_id in topic_id_list:
topic_id_object = Topic.objects.filter(id=int(topic_id)).first()
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [3, 4, 5, 6]:
effective_num += 1
if effective_num >= 5:
ret = True
break
return ret
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
class PictorialTag(models.Model):
"""画报关注标签"""
......
trans2es/models/tag.py
View file @ 5b5e6032
......@@ -12,10 +12,10 @@ import datetime
class TopicTag(models.Model):
class Meta:
verbose_name=u"帖子标签"
db_table="community_topictag"
verbose_name = u"帖子标签"
db_table = "community_topictag"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
tag_id = models.IntegerField(verbose_name=u"标签ID")
topic_id = models.IntegerField(verbose_name=u"帖子ID")
is_online = models.BooleanField(verbose_name=u"是否在线")
......@@ -25,73 +25,84 @@ class TopicTag(models.Model):
class AccountUserTag(models.Model):
class Meta:
verbose_name=u"用户标签"
db_table="account_user_tag"
verbose_name = u"用户标签"
db_table = "account_user_tag"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
tag_id = models.IntegerField(verbose_name=u"标签ID")
user = models.IntegerField(verbose_name=u"用户ID")
is_deleted = models.BooleanField(verbose_name=u"是否删除")
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
is_body_esthetics = models.IntegerField(verbose_name=u"是否有用标签")
class Tag(models.Model):
class Meta:
verbose_name=u"标签"
db_table="community_tag"
verbose_name = u"标签"
db_table = "community_tag"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
is_deleted = models.BooleanField(verbose_name=u"是否删除")
is_online = models.BooleanField(verbose_name=u"是否上线")
name = models.CharField(verbose_name=u"标签名称",max_length=128)
name = models.CharField(verbose_name=u"标签名称", max_length=128)
description = models.TextField(verbose_name=u"标签描述")
icon_url=models.CharField(verbose_name=u"icon_url",max_length=120)
icon_url = models.CharField(verbose_name=u"icon_url", max_length=120)
collection = models.IntegerField(verbose_name=u"是否编辑")
is_ai = models.IntegerField(verbose_name=u"是否ai")
is_own = models.IntegerField(verbose_name=u"是否ins上自带")
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class CommunityTagFollow(models.Model):
class Meta:
verbose_name=u"用户关注标签"
db_table="community_tag_follow"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
verbose_name = u"用户关注标签"
db_table = "community_tag_follow"
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
is_deleted = models.BooleanField(verbose_name=u"是否删除")
is_online = models.BooleanField(verbose_name=u"是否上线")
user_id = models.IntegerField(verbose_name=u"用户ID")
tag_id = models.IntegerField(verbose_name=u"帖子ID")
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class CommunityTagTypeRelation(models.Model):
class Meta:
verbose_name=u"标签类型对应关系"
db_table="community_tag_type_relation"
verbose_name = u"标签类型对应关系"
db_table = "community_tag_type_relation"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
is_deleted = models.BooleanField(verbose_name=u"是否删除")
is_online = models.BooleanField(verbose_name=u"是否上线")
tag_id = models.IntegerField(verbose_name=u"标签ID")
tag_type_id = models.IntegerField(verbose_name=u"标签类型ID")
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class CommunityTagType(models.Model):
class Meta:
verbose_name=u"标签类型"
db_table="community_tag_type"
verbose_name = u"标签类型"
db_table = "community_tag_type"
id = models.IntegerField(primary_key=True,verbose_name=u"主键ID")
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
is_deleted = models.BooleanField(verbose_name=u"是否删除")
is_online = models.BooleanField(verbose_name=u"是否上线")
name = models.CharField(verbose_name=u"名称",max_length=64)
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
name = models.CharField(verbose_name=u"名称", max_length=64)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class RegisterShowTag(models.Model):
class Meta:
verbose_name = "搜索热词"
db_table = "register_show_tag"
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
is_deleted = models.IntegerField(verbose_name=u"是否删除")
is_online = models.IntegerField(verbose_name=u"是否在线")
tag_id = models.IntegerField(verbose_name="标签ID")
trans2es/models/user.py
View file @ 5b5e6032
......@@ -16,6 +16,8 @@ from .tag import AccountUserTag
from .topic import Topic
from .user_extra import UserExtra
from .pictorial import PictorialFollow
from .topic import Topic
class User(models.Model):
class Meta:
......@@ -35,9 +37,10 @@ class User(models.Model):
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
@classmethod
def get_user_nick_name(cls,user_id):
def get_user_nick_name(cls, user_id):
try:
nick_name = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id=user_id).values_list("nick_name").first()
nick_name = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id=user_id).values_list(
"nick_name").first()
return nick_name[0]
except:
......@@ -47,7 +50,8 @@ class User(models.Model):
def get_is_recommend_flag(self):
is_shadow = False
is_recommend = False
query_sql = UserExtra.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id, is_deleted=False, is_online=True)
query_sql = UserExtra.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id, is_deleted=False,
is_online=True)
for record in query_sql:
is_recommend = record.is_recommend
is_shadow = record.is_shadow
......@@ -58,7 +62,8 @@ class User(models.Model):
latest_topic_time_val = -1
# 获取该用户最新发帖时间
topic_records = Topic.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",
topic_records = Topic.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id).order_by(
"-update_time").values_list("update_time",
flat=True).first()
if topic_records:
tzlc_topic_update_time = tzlc(topic_records)
......@@ -75,7 +80,8 @@ class User(models.Model):
logging.info("get follow_user_id_list :%s" % follow_user_id_list)
for i in range(0, len(follow_user_id_list), 1000):
logging.info("get follow_user_id_list :%s" % follow_user_id_list[i:i + 1000])
sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=follow_user_id_list[i:i + 1000])
sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(
user_id__in=follow_user_id_list[i:i + 1000])
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
......@@ -88,7 +94,8 @@ class User(models.Model):
def get_attention_group_id_list(self):
try:
attention_group_id_list = list()
query_results = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True, user_id=self.user_id)
query_results = GroupUserRole.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True,
user_id=self.user_id)
for item in query_results:
item_dict = {
"group_id": item.group_id,
......@@ -104,15 +111,16 @@ class User(models.Model):
def get_attention_pictorial_id_list(self):
try:
attention_pictorial_id_list = list()
query_results = PictorialFollow.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True, user_id=self.user_id)
logging.info("get PictorialFollow:%s"%query_results)
query_results = PictorialFollow.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True,
user_id=self.user_id)
logging.info("get PictorialFollow:%s" % query_results)
for item in query_results:
item_dict = {
"pictorial_id": item.pictorial_id,
"update_time_val": time.mktime(tzlc(item.update_time).timetuple())
}
attention_pictorial_id_list.append(item_dict)
logging.info("get user_id:%s" %self.user_id)
logging.info("get user_id:%s" % self.user_id)
logging.info("get same_pictorial_user_id_list:%s" % attention_pictorial_id_list)
......@@ -149,25 +157,29 @@ class User(models.Model):
def get_same_pictorial_user_id_list(self):
#todo 有的同组数据过大，导致celery cpu过高，暂时限制同组的数据大小，后续可能会去掉同组的数据
# todo 有的同组数据过大，导致celery cpu过高，暂时限制同组的数据大小，后续可能会去掉同组的数据
same_pictorial_user_id_list = list()
pictorial_items_list = list(PictorialFollow.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id).values_list("pictorial_id",flat=True))
pictorial_items_list = list(
PictorialFollow.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id).values_list(
"pictorial_id", flat=True))
for pictorial_id in pictorial_items_list:
user_items_list = list(PictorialFollow.objects.using(settings.SLAVE_DB_NAME).filter(pictorial_id=pictorial_id).values_list("user_id",flat=True))
user_items_list = list(
PictorialFollow.objects.using(settings.SLAVE_DB_NAME).filter(pictorial_id=pictorial_id).values_list(
"user_id", flat=True))
for user_id in user_items_list:
same_pictorial_user_id_list.append(user_id)
if len(same_pictorial_user_id_list)>=100:
if len(same_pictorial_user_id_list) >= 100:
break
if len(same_pictorial_user_id_list)>=100:
if len(same_pictorial_user_id_list) >= 100:
break
logging.info("get same user_id:%s"%self.user_id)
logging.info("get same_pictorial_user_id_list:%s"%same_pictorial_user_id_list)
logging.info("get same user_id:%s" % self.user_id)
logging.info("get same_pictorial_user_id_list:%s" % same_pictorial_user_id_list)
same_pictorial_detail_list = list()
for i in range(0, len(same_pictorial_user_id_list), 200):
sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=same_pictorial_user_id_list[i:i + 1000])
sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(
user_id__in=same_pictorial_user_id_list[i:i + 1000])
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
......@@ -181,7 +193,8 @@ class User(models.Model):
try:
user_tag_id_list = list()
query_results = AccountUserTag.objects.using(settings.SLAVE_DB_NAME).filter(user=self.user_id, is_deleted=False)
query_results = AccountUserTag.objects.using(settings.SLAVE_DB_NAME).filter(user=self.user_id,
is_deleted=False)
for item in query_results:
user_tag_id_list.append(item.tag_id)
......@@ -194,7 +207,9 @@ class User(models.Model):
try:
user_useful_tag_id_list = list()
query_results = AccountUserTag.objects.using(settings.SLAVE_DB_NAME).filter(user=self.user_id,is_body_esthetics =1, is_deleted=False)
query_results = AccountUserTag.objects.using(settings.SLAVE_DB_NAME).filter(user=self.user_id,
is_body_esthetics=1,
is_deleted=False)
for item in query_results:
user_useful_tag_id_list.append(item.tag_id)
......@@ -202,3 +217,11 @@ class User(models.Model):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_topic_number(self):
try:
number = Topic.objects.filter(user_id=self.user_id).count()
return number
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
trans2es/type_info.py
View file @ 5b5e6032
......@@ -14,7 +14,7 @@ import sys
from libs.cache import redis_client
import copy
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar,pictorial
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar, pictorial
from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
from trans2es.utils.group_transfer import GroupTransfer
......@@ -196,7 +196,7 @@ class TypeInfo(object):
old_data["is_history"] = True
data_list.append(old_data)
if int_ori_topic_star>=4:
if int_ori_topic_star >= 4:
topic_data_high_star_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
# data_list = [
......@@ -207,12 +207,11 @@ class TypeInfo(object):
# ]
# ESPerform.es_helpers_bulk(ESPerform.get_cli(), data_list, "topic-star")
if data["content_level"] and int(data["content_level"])>=4:
if data["content_level"] and int(data["content_level"]) >= 4:
topic_data_high_star_list.append(data)
data_list.append(data)
return (data_list,topic_data_high_star_list)
return (data_list, topic_data_high_star_list)
def elasticsearch_bulk_insert_data(self, sub_index_name, data_list, es=None):
......@@ -255,17 +254,17 @@ class TypeInfo(object):
else:
qs = self.model.objects.all()
end = time.time()
time0=end-begin
time0 = end - begin
begin = time.time()
instance_list = qs.filter(pk__in=pk_list)
end = time.time()
time1=end-begin
time1 = end - begin
begin = time.time()
data_list, topic_data_high_star_list = self.bulk_get_data(instance_list)
end = time.time()
time2=end-begin
time2 = end - begin
begin = time.time()
# logging.info("get sub_index_name:%s"%sub_index_name)
......@@ -277,7 +276,7 @@ class TypeInfo(object):
es=es,
)
if sub_index_name=="topic":
if sub_index_name == "topic":
self.elasticsearch_bulk_insert_data(
sub_index_name="topic-star-routing",
data_list=data_list,
......@@ -285,7 +284,7 @@ class TypeInfo(object):
)
# 同时写4星及以上的帖子
if len(topic_data_high_star_list)>0:
if len(topic_data_high_star_list) > 0:
self.elasticsearch_bulk_insert_data(
sub_index_name="topic-high-star",
data_list=topic_data_high_star_list,
......@@ -293,9 +292,8 @@ class TypeInfo(object):
)
end = time.time()
time3=end-begin
logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0,time1,time2,time3))
time3 = end - begin
logging.info("duan add,insert_table_by_pk_list time cost:%ds,%ds,%ds,%ds" % (time0, time1, time2, time3))
def insert_table_chunk(self, sub_index_name, table_chunk, es=None):
try:
......@@ -317,7 +315,7 @@ class TypeInfo(object):
auto_create_index=True
)
logging.info("es_helpers_bulk,sub_index_name:%s,data_list len:%d" % (sub_index_name,len(data_list)))
logging.info("es_helpers_bulk,sub_index_name:%s,data_list len:%d" % (sub_index_name, len(data_list)))
stage_3_time = time.time()
end_clock = time.clock()
......@@ -355,8 +353,8 @@ def get_type_info_map():
name='topic-star',
type='topic-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,#假的
get_data_func=TopicTransfer.get_topic_data,#假的
query_deferred=lambda: topic.Topic.objects.all().query, # 假的
get_data_func=TopicTransfer.get_topic_data, # 假的
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
......@@ -525,4 +523,3 @@ def get_type_info_map():
_get_type_info_map_result = type_info_map
return type_info_map
trans2es/utils/pictorial_transfer.py
View file @ 5b5e6032
......@@ -7,26 +7,27 @@ import traceback
from libs.tools import tzlc
from trans2es.models.topic import Topic
class PictorialTransfer(object):
def __init__(self):
pass
@classmethod
def get_offline_score(cls,instance,topic_id_list):
def get_offline_score(cls, instance, topic_id_list):
try:
total_offline_score = 0
topic_image_num = 0
for topic_id in topic_id_list:
topic_image_num += Topic.get_topic_image_num(topic_id)
if topic_image_num>=6 and topic_image_num<=10:
if topic_image_num >= 6 and topic_image_num <= 10:
total_offline_score += 30
elif topic_image_num>10 and topic_image_num<=20:
elif topic_image_num > 10 and topic_image_num <= 20:
total_offline_score += 60
elif topic_image_num>20 and topic_image_num<=50:
elif topic_image_num > 20 and topic_image_num <= 50:
total_offline_score += 80
elif topic_image_num>50:
elif topic_image_num > 50:
total_offline_score += 100
total_offline_score += instance.add_score
......@@ -62,10 +63,11 @@ class PictorialTransfer(object):
tag_id_list = instance.get_tag_by_id()
res["tag_id"] = tag_id_list
res["tag_name"] = instance.get_tag_by_name(tag_id_list)
res["topic_id_list"] =instance.get_topic_id()
res["topic_id_list"] = instance.get_topic_id()
res["effective"] = instance.get_effective(res["topic_id_list"])
res["offline_score"] = cls.get_offline_score(instance,res["topic_id_list"])
res["offline_score"] = cls.get_offline_score(instance, res["topic_id_list"])
res["is_default"] = instance.is_default
res["is_cover"] = instance.get_is_cover(res["topic_id_list"])
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/tag_transfer.py
View file @ 5b5e6032
......@@ -8,16 +8,15 @@ import logging
import traceback
from libs.tools import tzlc
from trans2es.models.topic import Topic
from trans2es.models.tag import TopicTag,CommunityTagType,CommunityTagTypeRelation
from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation
import datetime
from django.conf import settings
class TagTransfer(object):
@classmethod
def get_tag_name_data(cls,instance):
def get_tag_name_data(cls, instance):
try:
res = dict()
res["name"] = instance.name
......@@ -28,7 +27,7 @@ class TagTransfer(object):
return dict()
@classmethod
def get_tag_data(cls,instance):
def get_tag_data(cls, instance):
try:
res = dict()
......@@ -36,14 +35,14 @@ class TagTransfer(object):
tag_name_terms_list = list()
for i in range(len(instance.name)):
for j in range(i,len(instance.name)+1):
for j in range(i, len(instance.name) + 1):
name_term = instance.name[i:j].strip()
if name_term:
tag_name_terms_list.append(name_term.lower())
res["suggest"] = {
"input":tag_name_terms_list,
"contexts":{
"input": tag_name_terms_list,
"contexts": {
"is_online": [instance.is_online],
"is_deleted": [instance.is_deleted]
}
......@@ -54,24 +53,27 @@ class TagTransfer(object):
topic_num = 0
res["near_new_topic_num"] = topic_num
if instance.is_online==True and instance.is_deleted==False:
if instance.is_online == True and instance.is_deleted == False:
topic_id_list = list()
sql_result_results = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(
tag_id=instance.id).values_list("topic_id", "is_online"))
for topic_id,is_online in sql_result_results:
for topic_id, is_online in sql_result_results:
if is_online:
topic_id_list.append(topic_id)
time_base_val = datetime.datetime.strftime(datetime.datetime.now()+datetime.timedelta(-7), "%Y-%m-%d")
time_base_val = datetime.datetime.strftime(datetime.datetime.now() + datetime.timedelta(-7), "%Y-%m-%d")
for topic_begin_index in range(0,len(topic_id_list),100):
cur_topic_num = Topic.objects.using(settings.SLAVE_DB_NAME).filter(id__in=topic_id_list[topic_begin_index:topic_begin_index+100],create_time__gte=time_base_val).count()
for topic_begin_index in range(0, len(topic_id_list), 100):
cur_topic_num = Topic.objects.using(settings.SLAVE_DB_NAME).filter(
id__in=topic_id_list[topic_begin_index:topic_begin_index + 100],
create_time__gte=time_base_val).count()
topic_num += cur_topic_num
res["near_new_topic_num"] = topic_num
tag_type_sql_list = CommunityTagTypeRelation.objects.using(settings.SLAVE_DB_NAME).filter(tag_id=instance.id).values_list("tag_type_id",flat=True)
tag_type_sql_list = CommunityTagTypeRelation.objects.using(settings.SLAVE_DB_NAME).filter(
tag_id=instance.id).values_list("tag_type_id", flat=True)
tag_type_list = list()
for tag_type_id in tag_type_sql_list:
tag_type_list.append(tag_type_id)
......
trans2es/utils/user_transfer.py
View file @ 5b5e6032
......@@ -21,7 +21,8 @@ class UserTransfer(object):
follow_user_detail_list = list()
for i in range(0, len(follow_user_id_list), 1000):
sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(user_id__in=follow_user_id_list[i:i + 1000], is_online=True,
sql_data_list = User.objects.using(settings.SLAVE_DB_NAME).filter(
user_id__in=follow_user_id_list[i:i + 1000], is_online=True,
is_deleted=False)
for detail_data in sql_data_list:
item = {
......@@ -40,13 +41,14 @@ class UserTransfer(object):
res["id"] = instance.id
res["user_id"] = instance.user_id
res["nick_name"] = instance.nick_name
res["nick_name_pre"] = instance.nick_name
res["profile_pic"] = instance.profile_pic
res["gender"] = instance.gender
res["city_id"] = instance.city_id
res["country_id"] = instance.country_id
res["is_online"] = instance.is_online
res["is_deleted"] = instance.is_deleted
res["count_topic"] = instance.get_topic_number()
try:
(is_recommend, is_shadow) = instance.get_is_recommend_flag()
res["is_recommend"] = is_recommend
......@@ -95,7 +97,6 @@ class UserTransfer(object):
res["attention_pictorial_id_list"] = []
res["same_pictorial_user_id_list"] = []
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment