Skip to content

P
physical
Commit efc03161 authored by lixiaofang's avatar lixiaofang
Browse files
Options

add

parents 1907bb51 91465585
Show whitespace changes
Inline Side-by-side
Showing with 190 additions and 38 deletions
.idea/codeStyles/Project.xml
View file @ efc03161
......@@ -24,5 +24,29 @@
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme>
</component>
\ No newline at end of file
develop_server.py 0 → 100644
View file @ efc03161
import sys
from gm_rpcd.commands.utils import add_cwd_to_path
from gm_rpcd.internals.utils import serve
def main(args):
add_cwd_to_path()
from gm_rpcd.internals.configuration import config
config.is_develop_mode = True
config.freeze()
host = '127.0.0.1'
port = 9000
try:
first_arg = args[0]
except IndexError:
pass
else:
if ':' in first_arg:
host, port = first_arg.split(':')
port = int(port)
else:
port = int(first_arg)
print('Serving on {}:{}'.format(host, port))
serve(host=host, port=port)
if __name__ == '__main__':
main(sys.argv[1:])
injection/data_sync/tasks.py
View file @ efc03161
......@@ -113,12 +113,10 @@ def sync_user_similar_score():
def get_tag_count():
try:
# 获取搜索推荐热词
results_registr_tag = list(
set(RegisterShowTag.objects.filter(is_deleted=False, is_online=1).values_list("tag_id", flat=True)))
results_registr_tag = list(set(RegisterShowTag.objects.filter(is_deleted=False, is_online=1).values_list("tag_id", flat=True)))
# 获取符合条件的核心词
results_tag = list(
set(Tag.objects.filter(is_online=True, is_deleted=False, collection=1).values_list("id", flat=True)))
results_tag = list(set(Tag.objects.filter(is_online=True, is_deleted=False, collection=1).values_list("id", flat=True)))
redis_registr_tag = "physical:search_hotword:results_registr_tag"
redis_tag = "physical:search_hotword:results_tag"
......
search/utils/group.py
View file @ efc03161
......@@ -238,14 +238,19 @@ class GroupUtils(object):
{"term": {"is_deleted": False}},
{"term": {"is_default": 0}},
{"range": {"topic_id_list": {"gte": 0}}},
{"term": {"is_cover": True}},
{"term": {"is_cover": True}}
],
"should": [
{'multi_match': multi_match}
]
],
"minimum_should_match": 1
}
}
q["_source"] = {
"includes": ["id", "is_online", "is_deleted", "is_default", "name", "description", "tag_name",
"is_cover", "offline_score", "is_default"]
"includes": ["id", "is_online", "is_deleted", "is_default", "name", "tag_name", "description",
"is_cover",
"offline_score",
"is_default"]
}
q["sort"] = [
{
......@@ -265,11 +270,9 @@ class GroupUtils(object):
}
]
logging.info("get get_search_pictorial_topic:%s" % q)
es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
logging.info("get qqqqqq:%s" % q)
logging.info("get result_dict:%s" % result_dict)
return result_dict
except:
......
search/utils/topic.py
View file @ efc03161
......@@ -260,7 +260,7 @@ class TopicUtils(object):
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'cross_fields',
'type': 'best_fields',
'operator': 'and',
'fields': query_fields,
}
......
search/views/group.py
View file @ efc03161
......@@ -104,9 +104,27 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
q = {}
# 获取帖子从属的画报
q["query"] = {
"bool": {
"must": [
{
"term": {
"id": topic_id
}
},
{
"term": {
"is_online": True
}
}
],
"must_not": [
{
"term": {
"is_history": True
}
}
]
}
}
q["_source"] = {
"includes": ["id", "pictorial_id", "tag_list"]
......@@ -218,6 +236,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
"""
:remark 画报排序 人气 部分
人气按照票数从大到小排序，相同票数按照图片票数更新时间由旧到新排序
:param user_id:
:param sort_type:
:param offset:
......@@ -235,6 +254,22 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10):
{"term": {"pictorial_id": pictorial_id}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
# {
# "nested": {
# "path": "related_billboard",
# "query": {
# "bool": {
# "must": [
# {
# "term": {
# "pictorial_id": pictorial_id
# }
# }
# ]
# }
# }
# }
# }
]
}
},
......
search/views/search_hotword.py
View file @ efc03161
......@@ -51,6 +51,7 @@ def search_hotword(device_id=-1):
if len(all_tag_name_list) == 6 or num == results_tag:
break
logging.info("get all_tag_name_list:%s" % all_tag_name_list)
# 获取个性化标签
linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
......@@ -61,6 +62,7 @@ def search_hotword(device_id=-1):
set(Tag.objects.filter(id=item, is_online=True).values_list("name", flat=True)))
if results_tag_recommend:
all_tag_name_list.add(results_tag_recommend[0])
logging.info("get all_tag_name_list:%s" % all_tag_name_list)
if len(all_tag_name_list) == 12:
return {"recommend_tag_name": list(all_tag_name_list)}
......@@ -74,6 +76,8 @@ def search_hotword(device_id=-1):
if results_tag_hexin:
if results_tag_hexin[0] not in all_tag_name_list:
all_tag_name_list.add(results_tag_hexin[0])
logging.info("get all_tag_name_list:%s" % all_tag_name_list)
if len(all_tag_name_list) >= 12:
return {"recommend_tag_name": list(all_tag_name_list)}
......
search/views/user.py
View file @ efc03161
......@@ -114,7 +114,7 @@ def search_user(query="", offset=0, size=10):
"bool": {
"must": [
{"term": {
"nick_name_pre": query
"nick_pre": query
}
}, {
"term": {
......
trans2es/mapping/topic-star.json
View file @ efc03161
......@@ -14,7 +14,7 @@
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","index":"not_analyzed"}, //不切词的用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"edit_tag_list":{"type":"long"},//编辑标签
......
trans2es/mapping/topic.json
View file @ efc03161
......@@ -55,6 +55,14 @@
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
"is_history": {"type": "boolean"} //是否历史数据
"is_history": {"type": "boolean"}, //是否历史数据,
"related_billboard":{
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}
}
}
}
}
trans2es/mapping/user.json
View file @ efc03161
......@@ -4,7 +4,8 @@
"id":{"type":"long"},
"user_id":{"type":"long"},
"nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, //昵称
"nick_name_pre":{"type":"text","index":"not_analyzed"}, //昵称
"nick_pre":{"type":"text","analyzer":"keyword"}, //昵称
"nick_name_pre":{"type":"text"}, //昵称
"profile_pic":{"type":"text"}, //头像
"gender":{"type":"integer"},
"is_online":{"type":"boolean"},//是否上线
......
trans2es/models/topic.py
View file @ efc03161
......@@ -17,6 +17,7 @@ from .tag import TopicTag, Tag
from .user_extra import UserExtra
from .group import Group
class ActionSumAboutTopic(models.Model):
class Meta:
verbose_name = u"帖子埋点数据汇总"
......@@ -43,6 +44,7 @@ class TopicImage(models.Model):
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class ExcellentTopic(models.Model):
"""优质帖子"""
......@@ -51,13 +53,14 @@ class ExcellentTopic(models.Model):
db_table = "excellent_topic"
id = models.IntegerField(verbose_name=u'ID', primary_key=True)
topic_id = models.IntegerField(verbose_name=u"帖子ID",db_index=True)
topic_id = models.IntegerField(verbose_name=u"帖子ID", db_index=True)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
excellent_type = models.IntegerField(verbose_name=u"优质类型",db_index=True)
excellent_type = models.IntegerField(verbose_name=u"优质类型", db_index=True)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class Topic(models.Model):
class Meta:
verbose_name = u'日记'
......@@ -109,8 +112,8 @@ class Topic(models.Model):
def get_pictorial_id(self):
try:
pictorial_id_list =[]
pictorial_id = PictorialTopic.objects.filter(topic_id=self.id).values_list("pictorial_id",flat=True)
pictorial_id_list = []
pictorial_id = PictorialTopic.objects.filter(topic_id=self.id).values_list("pictorial_id", flat=True)
for i in pictorial_id:
pictorial_id_list.append(i)
return pictorial_id_list
......@@ -131,14 +134,16 @@ class Topic(models.Model):
return False
@classmethod
def get_topic_image_num(cls,topic_id):
def get_topic_image_num(cls, topic_id):
"""
:remark 获取指定帖子的图片数量
:param topic_id:
:return:
"""
try:
query_list = list(TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id, is_deleted=False, is_online=True).values_list("url",flat=True))
query_list = list(
TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id, is_deleted=False,
is_online=True).values_list("url", flat=True))
return len(query_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -148,7 +153,8 @@ class Topic(models.Model):
try:
has_image = False
query_list = TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_deleted=False, is_online=True)
query_list = TopicImage.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_deleted=False,
is_online=True)
if len(query_list) > 0:
has_image = True
......@@ -174,12 +180,13 @@ class Topic(models.Model):
topic_tag_id_list = list()
edit_tag_id_list = list()
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id).values_list("tag_id", flat=True)
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id).values_list("tag_id",
flat=True)
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=tag_id_list)
for tag_item in tag_query_results:
is_online=tag_item.is_online
is_deleted=tag_item.is_deleted
collection=tag_item.collection
is_online = tag_item.is_online
is_deleted = tag_item.is_deleted
collection = tag_item.collection
if is_online and not is_deleted:
topic_tag_id_list.append(tag_item.id)
......@@ -189,12 +196,13 @@ class Topic(models.Model):
return (topic_tag_id_list, edit_tag_id_list)
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return ([],[])
return ([], [])
def get_topic_useful_tag_id_list(self):
try:
topic_useful_tag_id_list = list()
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id,is_online =True,is_body_esthetics = 1)
tag_id_list = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, is_online=True,
is_body_esthetics=1)
for tag_item in tag_id_list:
topic_useful_tag_id_list.append(tag_item.tag_id)
return topic_useful_tag_id_list
......@@ -202,12 +210,13 @@ class Topic(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_tag_name_list(self, tag_id_list):
try:
tag_name_list = list()
for i in range(0, len(tag_id_list), 100):
query_terms_list = list(Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=tag_id_list[i:i + 100],is_online=True,is_deleted=False).values_list("name",flat=True))
query_terms_list = list(
Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=tag_id_list[i:i + 100], is_online=True,
is_deleted=False).values_list("name", flat=True))
tag_name_list.extend(query_terms_list)
return tag_name_list
except:
......@@ -234,17 +243,17 @@ class Topic(models.Model):
user_query_results = UserExtra.objects.using(settings.SLAVE_DB_NAME).filter(user_id=self.user_id)
if user_query_results.count() > 0:
if user_query_results[0].is_recommend:
offline_score += 2.0 *10
offline_score += 2.0 * 10
elif user_query_results[0].is_shadow:
user_is_shadow = True
# 帖子等级
if self.content_level == '5':
offline_score += 100.0 *3
offline_score += 100.0 * 3
elif self.content_level == '4':
offline_score += 60.0 *3
offline_score += 60.0 * 3
elif self.content_level == '6':
offline_score += 200.0 *3
offline_score += 200.0 * 3
if self.language_type == 1:
offline_score += 60.0
......@@ -262,6 +271,25 @@ class Topic(models.Model):
return 0.0
def get_related_billboard(self):
try:
related_billboard_list = list()
query_results = TopicBillBoard.objects.filter(id=self.id)
for items in query_results:
related_billboard_list.append({"pictorial_id": items.pictorial_id, "real_vote_cnt": items.real_vote_cnt,
"virt_vote_cnt": items.virt_vote_cnt})
logging.info("product_brand_info" % related_billboard_list)
return related_billboard_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
class TopicComplaint(models.Model):
"""帖子举报"""
......@@ -273,7 +301,8 @@ class TopicComplaint(models.Model):
id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
user_id = models.BigIntegerField(verbose_name=u'用户ID', db_index=True)
topic = models.ForeignKey(
Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE, related_name='complaints')
Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE,
related_name='complaints')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
......@@ -302,7 +331,7 @@ class TopicExtra(models.Model):
db_table = 'topic_extra'
id = models.IntegerField(verbose_name=u'ID', primary_key=True)
topic_id = models.IntegerField(verbose_name=u"帖子ID",db_index=True)
topic_id = models.IntegerField(verbose_name=u"帖子ID", db_index=True)
virtual_vote_num = models.IntegerField(verbose_name="帖子虚拟点赞")
......@@ -312,7 +341,21 @@ class TopicHomeRecommend(models.Model):
class Meta:
verbose_name = "运营位帖子"
db_table = "topic_home_recommend"
id = models.IntegerField(verbose_name=u"id",primary_key=True)
id = models.IntegerField(verbose_name=u"id", primary_key=True)
topic_id = models.IntegerField(verbose_name=u"帖子ID")
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
class TopicBillBoard(models.Model):
"""榜单投票"""
class Meta:
verbose_name = "榜单投票"
db_table = "topic_vote_cnt"
topic_id = models.IntegerField(verbose_name=u"帖子ID")
pictorial_id = models.IntegerField(verbose_name=u"榜单ID")
real_vote_cnt = models.IntegerField(verbose_name=u"真实赞")
virt_vote_cnt = models.IntegerField(verbose_name=u"虚拟赞")
trans2es/utils/topic_transfer.py
View file @ efc03161
......@@ -138,6 +138,9 @@ class TopicTransfer(object):
res["is_operation_home_recommend"] = True
logging.info("test topic transfer time cost,time0:%d,time1:%d,time2:%d,time3:%d,time4:%d" % (time0,time1,time2,time3,time4))
#榜单关联的投票
res["related_billboard"] = instance.get_related_billboard()
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/user_transfer.py
View file @ efc03161
......@@ -42,6 +42,7 @@ class UserTransfer(object):
res["user_id"] = instance.user_id
res["nick_name"] = instance.nick_name
res["nick_name_pre"] = instance.nick_name
res["nick_pre"] = instance.nick_name
res["profile_pic"] = instance.profile_pic
res["gender"] = instance.gender
res["city_id"] = instance.city_id
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment