Skip to content

P
physical
Commit 6d35c57b authored by lixiaofang's avatar lixiaofang
Browse files
Options

change list

parent 6bd762aa
Hide whitespace changes
Inline Side-by-side
Showing with 179 additions and 98 deletions
search/utils/pick.py
View file @ 6d35c57b
......@@ -90,7 +90,7 @@ class PickUtils(object):
return []
@classmethod
def get_topic_pick_query(cls,query,pick_id,offset,size):
def get_topic_pick_query(cls,query,pick_id,offset,size):
"""
:remark 查询榜单下帖子列表
:param query:
......
search/utils/topic.py
View file @ 6d35c57b
......@@ -90,7 +90,7 @@ class TopicUtils(object):
return []
@classmethod
def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[]):
def get_recommend_topic_ids(cls,user_id,read_id_list,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[]):
"""
:需增加打散逻辑
:remark:获取首页推荐帖子列表
......@@ -182,7 +182,7 @@ class TopicUtils(object):
{"range": {"content_level": {"gte": low_content_level, "lte": 5}}},
{"term": {"has_image":True}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}
{"term": {"is_deleted": False}},
]
}
},
......@@ -191,11 +191,11 @@ class TopicUtils(object):
"functions": functions_list
}
if query_type==TopicPageType.FIND_PAGE and len(filter_topic_id_list)>0:
query_function_score["query"]["bool"]["must_not"] = {
query_function_score["query"]["bool"]["must_not"] =[{
"terms":{
"id": filter_topic_id_list
}
}
},{"terms":{"topic_id":read_id_list}}]
if query is not None:#搜索帖子
multi_fields = {
......
search/views/topic.py
View file @ 6d35c57b
......@@ -62,7 +62,6 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
last_offset_num = int(redis_field_val_list[0]) if redis_field_val_list[0] else -1
recommend_topic_ids = []
topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id, offset, size*size,query,query_type=query_type)
have_read_group_id_set = set()
unread_topic_id_dict = dict()
......@@ -84,6 +83,7 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
cur_page_user_num = 0
for topic_id in topic_id_dict:
#
if topic_id_dict[topic_id] in have_read_group_id_set:
unread_topic_id_dict[topic_id] = topic_id_dict[topic_id]
else:
......@@ -102,7 +102,6 @@ def get_home_recommend_topic_ids(user_id,device_id,offset,size,query=None,query_
if len(recommend_topic_ids) >= size:
break
offi_unread_topic_id_dict = dict()
if len(recommend_topic_ids) < size and len(unread_topic_id_dict)>0:
for unread_topic_id in unread_topic_id_dict:
......
trans2es/models/group.py
View file @ 6d35c57b
#!/usr/bin/env python
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.conf import settings
......
trans2es/models/topic.py
View file @ 6d35c57b
......@@ -10,34 +10,36 @@ from django.db import models
import datetime
from .pick_topic import PickTopic
from .tag import TopicTag,Tag
from .tag import TopicTag, Tag
from .user_extra import UserExtra
from .group import Group
from .it_list import it_lsit
class ActionSumAboutTopic(models.Model):
class Meta:
verbose_name=u"帖子埋点数据汇总"
db_table="action_sum_about_topic"
verbose_name = u"帖子埋点数据汇总"
db_table = "action_sum_about_topic"
partiton_date = models.CharField(verbose_name=u"日期",max_length=20)
device_id = models.CharField(verbose_name=u"用户设备号",max_length=50)
topic_id = models.CharField(verbose_name=u"帖子ID",max_length=50)
user_id = models.CharField(verbose_name=u"用户ID",max_length=50)
partiton_date = models.CharField(verbose_name=u"日期", max_length=20)
device_id = models.CharField(verbose_name=u"用户设备号", max_length=50)
topic_id = models.CharField(verbose_name=u"帖子ID", max_length=50)
user_id = models.CharField(verbose_name=u"用户ID", max_length=50)
data_type = models.IntegerField(verbose_name=u"动作类型")
data_value = models.BigIntegerField(verbose_name=u"值")
class TopicImage(models.Model):
class Meta:
verbose_name = u'日记图片'
db_table = 'topic_image'
id = models.IntegerField(verbose_name='日记图片ID',primary_key=True)
id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
topic_id = models.IntegerField(verbose_name=u'日记ID')
url = models.CharField(verbose_name=u'图片URL',max_length=300)
url = models.CharField(verbose_name=u'图片URL', max_length=300)
is_online = models.BooleanField(verbose_name='是否上线')
is_deleted = models.BooleanField(verbose_name='是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
......@@ -46,33 +48,33 @@ class Topic(models.Model):
verbose_name = u'日记'
db_table = 'topic'
id = models.IntegerField(verbose_name=u'日记ID',primary_key=True)
name = models.CharField(verbose_name=u'日记名称',max_length=100)
#group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
name = models.CharField(verbose_name=u'日记名称', max_length=100)
# group_id = models.IntegerField(verbose_name='用户所在组ID',default=-1)
group = models.ForeignKey(
Group, verbose_name=u"关联的小组", related_name=u"group_topics",null=True, blank=True, default=None, on_delete=models.CASCADE)
Group, verbose_name=u"关联的小组", related_name=u"group_topics", null=True, blank=True, default=None,
on_delete=models.CASCADE)
user_id = models.IntegerField(verbose_name=u'用户ID')
drop_score = models.IntegerField(verbose_name=u'人工赋分',default=0)
description = models.CharField(verbose_name=u'日记本描述',max_length=200)
content = models.CharField(verbose_name=u'日记本内容',max_length=1000)
drop_score = models.IntegerField(verbose_name=u'人工赋分', default=0)
description = models.CharField(verbose_name=u'日记本描述', max_length=200)
content = models.CharField(verbose_name=u'日记本内容', max_length=1000)
share_num = models.IntegerField(verbose_name='')
vote_num = models.IntegerField(verbose_name=u'点赞数')
reply_num = models.IntegerField(verbose_name=u'回复数')
cover = models.CharField(verbose_name='',max_length=200)
cover = models.CharField(verbose_name='', max_length=200)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
content_level = models.CharField(verbose_name=u'内容等级',max_length=3)
create_time = models.DateTimeField(verbose_name=u'日记创建时间',default=datetime.datetime.fromtimestamp(0))
content_level = models.CharField(verbose_name=u'内容等级', max_length=3)
create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
def topic_has_image(self):
try:
has_image = False
query_list = TopicImage.objects.filter(topic_id=self.id,is_deleted=False,is_online=True)
if len(query_list)>0:
has_image=True
query_list = TopicImage.objects.filter(topic_id=self.id, is_deleted=False, is_online=True)
if len(query_list) > 0:
has_image = True
return has_image
except:
......@@ -82,7 +84,7 @@ class Topic(models.Model):
def get_pick_id_info(self):
try:
pick_id_list = list()
query_list = PickTopic.objects.filter(topic_id=self.id,is_deleted=False)
query_list = PickTopic.objects.filter(topic_id=self.id, is_deleted=False)
for item in query_list:
pick_id_list.append(item.pick_id)
......@@ -104,12 +106,26 @@ class Topic(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_tag_name_list(self,tag_id_list):
def get_tag_name_list(self, tag_id_list):
try:
tag_name_list = list()
query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name",flat=True)
for item in query_results:
tag_name_list.append(item)
if len(tag_id_list) > 100:
lens = int(len(tag_id_list) / 100)
for i in range(lens+1):
if i != 0:
tag_id_100 = tag_id_list[i * 100:i * 100 + 100]
if i == 0:
tag_id_100 = tag_id_list[i:i + 100]
if i * 100 == lens * 100 and len(tag_id_list[lens * 100:]) > 0:
tag_id_100 = tag_id_list[lens * 100:]
if tag_id_100:
query_results = Tag.objects.filter(id__in=tag_id_100).values_list("name", flat=True)
for item in query_results:
tag_name_list.append(item)
else:
query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name", flat=True)
for item in query_results:
tag_name_list.append(item)
return tag_name_list
except:
......@@ -121,7 +137,7 @@ class Topic(models.Model):
offline_score = 0.0
user_is_shadow = False
#是否官方推荐用户
# 是否官方推荐用户
user_query_results = UserExtra.objects.filter(user_id=self.user_id)
if user_query_results.count() > 0:
if user_query_results[0].is_recommend:
......@@ -133,7 +149,7 @@ class Topic(models.Model):
if self.group and self.group.is_recommend:
offline_score += 4.0
#帖子等级
# 帖子等级
if self.content_level == '5':
offline_score += 5.0
elif self.content_level == '4':
......@@ -141,21 +157,21 @@ class Topic(models.Model):
elif self.content_level == '3':
offline_score += 2.0
exposure_count = ActionSumAboutTopic.objects.filter(topic_id=self.id,data_type=1).count()
exposure_count = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=1).count()
click_count = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=2).count()
uv_num = ActionSumAboutTopic.objects.filter(topic_id=self.id,data_type=3).count()
uv_num = ActionSumAboutTopic.objects.filter(topic_id=self.id, data_type=3).count()
if exposure_count>0:
offline_score += click_count/exposure_count
if uv_num>0:
offline_score += (self.vote_num/uv_num + self.reply_num/uv_num)
if exposure_count > 0:
offline_score += click_count / exposure_count
if uv_num > 0:
offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
"""
1:马甲账号是否对总分降权？
"""
if user_is_shadow:
offline_score = offline_score*0.5
offline_score = offline_score * 0.5
return offline_score
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0.0
\ No newline at end of file
return 0.0
trans2es/models/user.py
View file @ 6d35c57b
......@@ -22,7 +22,7 @@ class User(models.Model):
verbose_name = u'用户'
db_table = 'account_user'
id = models.IntegerField(verbose_name="主键ID",primary_key=True)
id = models.IntegerField(verbose_name="主键ID", primary_key=True)
user_id = models.BigIntegerField(verbose_name=u'用户id', unique=True)
nick_name = models.CharField(verbose_name=u'昵称', max_length=255, default='')
profile_pic = models.CharField(verbose_name=u'头像', max_length=300)
......@@ -31,56 +31,73 @@ class User(models.Model):
country_id = models.CharField(verbose_name=u'国家id', max_length=40)
is_online = models.BooleanField(verbose_name="是否上线")
is_deleted = models.BooleanField(verbose_name='是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
def get_is_recommend_flag(self):
is_shadow = False
is_recommend = False
query_sql = UserExtra.objects.filter(user_id=self.user_id,is_deleted=False,is_online=True)
query_sql = UserExtra.objects.filter(user_id=self.user_id, is_deleted=False, is_online=True)
for record in query_sql:
is_recommend = record.is_recommend
is_shadow = record.is_shadow
return (is_recommend,is_shadow)
return (is_recommend, is_shadow)
def get_latest_topic_time_val(self):
latest_topic_time_val = -1
# 获取该用户最新发帖时间
topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",flat=True).first()
topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",
flat=True).first()
if topic_records:
tzlc_topic_update_time = tzlc(topic_records)
latest_topic_time_val = int(time.mktime(tzlc_topic_update_time.timetuple()))
return latest_topic_time_val
def get_follow_user_id_list(self):
follow_user_id_list = list()
user_follows = self.userfollow.filter(is_online=True)
for user_follow in user_follows:
follow_user_id_list.append(user_follow.follow_id)
follow_user_detail_list = list()
sql_data_list = User.objects.filter(user_id__in=follow_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id":detail_data.user_id,
"country_id":detail_data.country_id
}
follow_user_detail_list.append(item)
if len(follow_user_id_list) > 100:
lens = int(len(follow_user_id_list) / 100)
for i in range(lens+1):
if i != 0:
follow_user_id_list_100 = follow_user_id_list[i * 100:i * 100 + 100]
if i == 0:
follow_user_id_list_100 = follow_user_id_list[i:i + 100]
if i * 100 == lens * 100 and len(follow_user_id_list[lens * 100:]) > 0:
follow_user_id_list_100 = follow_user_id_list[lens * 100:]
if follow_user_id_list_100:
sql_data_list = User.objects.filter(user_id__in=follow_user_id_list_100).values("user_id","country_id")
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
follow_user_detail_list.append(item)
else:
sql_data_list = User.objects.filter(user_id__in=follow_user_id_list).values("user_id","country_id")
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
follow_user_detail_list.append(item)
return follow_user_detail_list
def get_attention_group_id_list(self):
try:
attention_group_id_list = list()
query_results = GroupUserRole.objects.filter(is_online=True,user_id=self.user_id)
query_results = GroupUserRole.objects.filter(is_online=True, user_id=self.user_id)
for item in query_results:
item_dict = {
"group_id": item.group_id,
"update_time_val":time.mktime(tzlc(item.update_time).timetuple())
"update_time_val": time.mktime(tzlc(item.update_time).timetuple())
}
attention_group_id_list.append(item_dict)
......@@ -91,24 +108,58 @@ class User(models.Model):
def get_pick_user_id_list(self):
pick_topic_id_list = list()
user_picks = self.user_pick.filter(is_deleted=False,is_pick=True)
user_picks = self.user_pick.filter(is_deleted=False, is_pick=True)
for user_pick in user_picks:
pick_topic_id_list.append(user_pick.picktopic_id)
pick_user_id_list = []
topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list)
for topic_data in topic_sql_list:
pick_user_id_list.append(topic_data.user_id)
if len(pick_topic_id_list) > 100:
lens = int(len(pick_topic_id_list) / 100)
for i in range(lens+1):
if i != 0:
pick_topic_id_list_100 = pick_topic_id_list[i * 100:i * 100 + 100]
if i == 0:
pick_topic_id_list_100 = pick_topic_id_list[i:i + 100]
if i * 100 == lens * 100 and len(pick_topic_id_list[lens * 100:]) > 0:
pick_topic_id_list_100 = pick_topic_id_list[lens * 100:]
if pick_topic_id_list_100:
topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list_100).values('user_id')
for topic_data in topic_sql_list:
pick_user_id_list.append(topic_data.user_id)
else:
topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list).values('user_id')
for topic_data in topic_sql_list:
pick_user_id_list.append(topic_data.user_id)
pick_user_id_list = tuple(pick_user_id_list)
pick_user_detail_list = list()
sql_data_list = User.objects.filter(user_id__in=pick_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id":detail_data.user_id,
"country_id":detail_data.country_id
}
pick_user_detail_list.append(item)
if len(pick_user_id_list) > 100:
lens = int(len(pick_user_id_list) / 100)
for i in range(lens+1):
if i != 0:
pick_user_id_list_100 = pick_user_id_list[i * 100:i * 100 + 100]
if i == 0:
pick_user_id_list_100 = pick_user_id_list[i:i + 100]
if i * 100 == lens* 100 and len(pick_user_id_list[lens * 100:]) > 0:
pick_user_id_list_100 = pick_user_id_list[lens * 100:]
if pick_topic_id_list_100:
sql_data_list = User.objects.filter(user_id__in=pick_user_id_list_100).values("user_id","country_id")
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
pick_user_detail_list.append(item)
else:
sql_data_list = User.objects.filter(user_id__in=pick_user_id_list).values("user_id","country_id")
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
pick_user_detail_list.append(item)
return pick_user_detail_list
......@@ -122,22 +173,39 @@ class User(models.Model):
same_group_user_id_list.append(user_items_list.user_id)
same_group_detail_list = list()
sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id":detail_data.user_id,
"country_id":detail_data.country_id
}
same_group_detail_list.append(item)
if len(same_group_user_id_list) > 100:
lens = int(len(same_group_user_id_list) / 100)
for i in range(lens+1):
if i != 0:
same_group_user_id_list_100 = same_group_user_id_list[i * 100:i * 100 + 100]
if i == 0:
same_group_user_id_list_100 = same_group_user_id_list[i:i + 100]
if i * 100 == (lens - 1) * 100:
same_group_user_id_list_100 = same_group_user_id_list[lens * 100:]
if same_group_user_id_list_100:
sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list_100).values("user_id","country_id")
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
same_group_detail_list.append(item)
else:
sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list).values("user_id","country_id")
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
same_group_detail_list.append(item)
return same_group_detail_list
def get_user_tag_id_list(self):
try:
user_tag_id_list = list()
query_results = AccountUserTag.objects.filter(user=self.user_id,is_deleted=False)
query_results = AccountUserTag.objects.filter(user=self.user_id, is_deleted=False).values("name")
for item in query_results:
user_tag_id_list.append(item.tag_id)
......@@ -145,4 +213,3 @@ class User(models.Model):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
trans2es/utils/tag_transfer.py
View file @ 6d35c57b
......@@ -8,13 +8,14 @@ import logging
import traceback
from libs.tools import tzlc
from trans2es.models.topic import Topic
from trans2es.models.tag import TopicTag,CommunityTagType,CommunityTagTypeRelation
from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation
import datetime
class TagTransfer(object):
@classmethod
def get_tag_data(cls,instance):
def get_tag_data(cls, instance):
try:
res = dict()
......@@ -22,14 +23,14 @@ class TagTransfer(object):
tag_name_terms_list = list()
for i in range(len(instance.name)):
for j in range(i,len(instance.name)+1):
for j in range(i, len(instance.name) + 1):
name_term = instance.name[i:j].strip()
if name_term:
tag_name_terms_list.append(name_term)
res["suggest"] = {
"input":tag_name_terms_list,
"contexts":{
"input": tag_name_terms_list,
"contexts": {
"is_online": [instance.is_online],
"is_deleted": [instance.is_deleted]
}
......@@ -39,19 +40,19 @@ class TagTransfer(object):
res["is_deleted"] = instance.is_deleted
res["near_new_topic_num"] = 0
if instance.is_online==True and instance.is_deleted==False:
if instance.is_online == True and instance.is_deleted == False:
topic_id_list = list()
sql_result_list = TopicTag.objects.filter(tag_id=instance.id).values_list("topic_id",flat=True)
sql_result_list = TopicTag.objects.filter(tag_id=instance.id).values_list("topic_id", flat=True)
for item_id in sql_result_list:
topic_id_list.append(item_id)
time_base_val = datetime.datetime.strftime(datetime.datetime.now()+datetime.timedelta(-7), "%Y-%m-%d")
topic_num = Topic.objects.filter(id__in=topic_id_list,create_time__gte=time_base_val).count()
time_base_val = datetime.datetime.strftime(datetime.datetime.now() + datetime.timedelta(-7), "%Y-%m-%d")
topic_num = Topic.objects.filter(id__in=topic_id_list, create_time__gte=time_base_val).count()
res["near_new_topic_num"] = topic_num
tag_type_sql_list = CommunityTagTypeRelation.objects.filter(tag_id=instance.id).values_list("tag_type_id",flat=True)
tag_type_sql_list = CommunityTagTypeRelation.objects.filter(tag_id=instance.id).values_list("tag_type_id",
flat=True)
tag_type_list = list()
for tag_type_id in tag_type_sql_list:
tag_type_list.append(tag_type_id)
......@@ -61,4 +62,4 @@ class TagTransfer(object):
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
\ No newline at end of file
return dict()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment