Skip to content

P
physical
Commit f09a5adf authored by zhanglu's avatar zhanglu
Browse files
Options

帖子同步到es

parent 0d6fdd80
Hide whitespace changes
Inline Side-by-side
Showing with 98 additions and 171 deletions
app_conf.xml
View file @ f09a5adf
......@@ -9,6 +9,7 @@
<config name="initializer_list">
<element value="physical.django_init"/>
<element value="search.views.topic"/>
<element value="search.views.business_topic"/>
<element value="search.views.pick"/>
<element value="search.views.group"/>
<element value="search.views.user"/>
......
search/utils/topic.py
View file @ f09a5adf
......@@ -560,10 +560,30 @@ class TopicUtils(object):
return f
for k, v in filters.items():
if k == "group_id":
if k in ["create_time_gte", "create_time_lte"]:
if k == "create_time_gte":
op = "gte"
elif k == "create_time_lte":
op = "lte"
f.append({
"term": {"group_id": v},
"range": {
"create_time_val": {
op: v,
}
}
})
else:
if isinstance(k, list):
f.append({
"terms": {k: v},
})
else:
f.append({
"term": {k: v},
})
if filter_online:
f.append({"term": {"is_online": True}})
......@@ -598,22 +618,29 @@ class TopicUtils(object):
},
})
elif sorts_by == TOPIC_SEARCH_SORT.VOTE_NUM:
pass
elif sorts_by == '2':
sort_rule.append({
"id":{
"order":"desc"
},
})
return sort_rule
@classmethod
def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
must = cls.process_filters(filters, filter_online=True)
q = {
"query": {
"bool": {
"must": cls.process_filters(filters, filter_online=True),
"must": must,
"must_not": cls.process_nfilters(nfilters),
}
},
"_source": {
"includes":["id"]
},
"sort": [],
}
}
if sorts_by:
......@@ -626,7 +653,13 @@ class TopicUtils(object):
query_body=q, offset=offset, size=size
)
return result_dict["hits"]
return {
"hits": result_dict["hits"],
"total_count": result_dict["total_count"]
}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
return {
"hits": [],
"total_count": 0
}
search/views/business_topic.py
View file @ f09a5adf
......@@ -14,16 +14,17 @@ from libs.es import ESPerform
@bind("physical/search/business/topic")
def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
result_list = TopicUtils.list_topic_ids(
filters=filters, nfilters=nfilters, sorts_by=sorts_by,
offset=offset, size=size, filter_online=False
offset=offset, size=size, filter_online=False,
index_name="business_topic"
)
topic_ids = [item["_source"]["id"] for item in result_list]
return {"topic_ids": topic_ids}
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": []}
return {"topic_ids": [], "total_count": 0}
search/views/topic.py
View file @ f09a5adf
......@@ -285,11 +285,14 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
try:
result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
topic_ids = [item["_source"]["id"] for item in result_list]
return {"topic_ids": topic_ids}
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {
"topic_ids": topic_ids,
"total_count": result_list["total_count"]
}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": []}
return {"topic_ids": [], "total_count": 0}
@bind("physical/search/query_topic_by_user_similarity")
......
trans2es/mapping/business_topic.json deleted 100644 → 0
View file @ 0d6fdd80
// 业务端帖子存储表做宽表查询
{
"dynamic": "strict",
"properties":{
"id":{"type": "long"},
"user_id":{"type": "long"},
"is_shadow": {"type": "boolean"},
"is_online":{"type": "boolean"},
"is_deleted":{"type": "boolean"},
"has_image":{"type": "boolean"}, //是否有图
"has_video":{"type": "boolean"}, //是否是视频
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"vote_num":{"type": "long"},
"reply_num":{"type": "long"},
"share_num": {"type": "long"},
"name":{
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"description":{
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"content":{
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"content_level":{"type": "text"},
"virtual_content_level":{"type": "text"},
"language_type": {"type": "long"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"create_time":{"type": "date", "format": "date_time_no_millis"},
"update_time":{"type": "date", "format": "date_time_no_millis"},
"create_time_val":{"type": "long"},
"update_time_val":{"type": "long"},
"tag_list":{"type": "long"}, // 标签属性
"edit_tag_list":{"type": "long"}, //编辑标签
"tag_name_list":{
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
}
}
}
\ No newline at end of file
trans2es/mapping/topic.json
View file @ f09a5adf
......@@ -25,6 +25,29 @@
"update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"},
"update_time_val":{"type":"long"},
"language_type":{"type":"long"}
"language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
}
}
}
\ No newline at end of file
trans2es/models/__init__.py
View file @ f09a5adf
......@@ -13,4 +13,4 @@ from .tag import AccountUserTag
from .user import User
from .group import Group
from .topic import Topic
from .pictorial import PictorialTopic
# from .pictorial import PictorialTopic
trans2es/models/pictorial.py
View file @ f09a5adf
from django.db import models
# from django.db import models
class PictorialTopic(models.Model):
"""画报帖子关系"""
# class PictorialTopic(models.Model):
# """画报帖子关系"""
class Meta:
verbose_name = u'画报帖子关系'
app_label = 'community'
db_table = 'community_pictorial_topic'
# class Meta:
# verbose_name = u'画报帖子关系'
# app_label = 'community'
# db_table = 'community_pictorial_topic'
id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
user_id = models.BigIntegerField(verbose_name=u'用户ID')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除')
# id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
# pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
# topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
# user_id = models.BigIntegerField(verbose_name=u'用户ID')
# is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
# is_online = models.BooleanField(verbose_name=u'是否上线')
# is_deleted = models.BooleanField(verbose_name=u'是否删除')
trans2es/type_info.py
View file @ f09a5adf
......@@ -265,16 +265,6 @@ def get_type_info_map():
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name='business_topic', # 运营后台
type='business_topic',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,
get_data_func=TopicTransfer.get_business_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2,
),
TypeInfo(
name = "user", #用户
type="user",
......
trans2es/utils/topic_transfer.py
View file @ f09a5adf
......@@ -94,61 +94,4 @@ class TopicTransfer(object):
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
@classmethod
def get_business_topic_data(cls,instance):
try:
res = {
"id": instance.id,
"user_id": instance.user_id,
"is_shadow": instance.is_shadow,
"is_online": instance.is_online,
"is_deleted": instance.is_deleted,
"has_image": instance.topic_has_image(),
"has_video": instance.has_video,
"is_recommend": True if instance.is_recommend else False,
"is_complaint": instance.is_complaint,
"vote_num": instance.vote_num,
"reply_num": instance.reply_num,
"share_num": instance.share_num,
"name": instance.name,
"description": instance.description,
"content": instance.content,
"content_level": instance.content_level,
"virtual_content_level": instance.virtual_content_level,
"language_type": instance.language_type,
"like_num_crawl": instance.like_num_crawl,
"comment_num_crawl": instance.comment_num_crawl,
"is_crawl": instance.is_crawl,
"platform": instance.platform,
"platform_id": instance.platform_id,
"drop_score": instance.drop_score,
"sort_score": instance.sort_score,
}
(topic_tag_id_list, edit_tag_id_list) = instance.get_topic_tag_id_list()
res["tag_list"] = topic_tag_id_list
res["edit_tag_list"] = edit_tag_id_list
res["tag_name_list"] = instance.get_tag_name_list(res["tag_list"]),
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
res["create_time"] = tzlc_create_time
res["create_time_val"] = int(time.mktime(tzlc_create_time.timetuple()))
update_time = instance.update_time
tzlc_update_time = tzlc(update_time)
res["update_time"] = tzlc_update_time
res["update_time_val"] = int(time.mktime(tzlc_update_time.timetuple()))
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
return None
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment