Skip to content

P
physical
Commit e6f89fd1 authored by 李小芳's avatar 李小芳
Browse files
Options

Merge branch '721_goods-sort' into 'test' 

721 goods sort

See merge request alpha/physical!390
parents 415b0086 74a5dd9f
Hide whitespace changes
Inline Side-by-side
Showing with 517 additions and 234 deletions
libs/es.py
View file @ e6f89fd1
...@@ -155,9 +155,9 @@ class ESPerform(object): ...@@ -155,9 +155,9 @@ class ESPerform(object):
bulk_actions = [] bulk_actions = []
if sub_index_name == "topic" or \ if sub_index_name=="topic" or \
sub_index_name == "topic-star-routing" or \ sub_index_name=="topic-star-routing" or \
sub_index_name == "topic-high-star": sub_index_name=="topic-high-star":
for data in data_list: for data in data_list:
if data: if data:
bulk_actions.append({ bulk_actions.append({
...@@ -232,7 +232,7 @@ class ESPerform(object): ...@@ -232,7 +232,7 @@ class ESPerform(object):
return {"total_count": 0, "hits": []} return {"total_count": 0, "hits": []}
@classmethod @classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body): def get_analyze_results(cls,es_cli, sub_index_name, query_body):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
...@@ -242,7 +242,7 @@ class ESPerform(object): ...@@ -242,7 +242,7 @@ class ESPerform(object):
logging.error("index:%s is not existing,get_search_results error!" % official_index_name) logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body) res = es_cli.indices.analyze(index=official_index_name,body=query_body)
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -276,7 +276,7 @@ class ESPerform(object): ...@@ -276,7 +276,7 @@ class ESPerform(object):
if high_num > 3: if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % ( logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list))) str(es_nodes_list), str(es_nodes_info_list)))
return True return True
else: else:
return False return False
...@@ -298,8 +298,8 @@ class ESPerform(object): ...@@ -298,8 +298,8 @@ class ESPerform(object):
functions_list += [ functions_list += [
{ {
"filter": { "filter": {
"constant_score": { "constant_score":{
"filter": { "filter":{
"term": {"content_level": 6}} "term": {"content_level": 6}}
} }
}, },
...@@ -307,8 +307,8 @@ class ESPerform(object): ...@@ -307,8 +307,8 @@ class ESPerform(object):
}, },
{ {
"filter": { "filter": {
"constant_score": { "constant_score":{
"filter": { "filter":{
"term": {"content_level": 5}} "term": {"content_level": 5}}
} }
}, },
...@@ -316,8 +316,8 @@ class ESPerform(object): ...@@ -316,8 +316,8 @@ class ESPerform(object):
}, },
{ {
"filter": { "filter": {
"constant_score": { "constant_score":{
"filter": { "filter":{
"term": {"content_level": 4}} "term": {"content_level": 4}}
} }
}, },
...@@ -411,7 +411,7 @@ class ESPerform(object): ...@@ -411,7 +411,7 @@ class ESPerform(object):
} }
}, },
"_source": { "_source": {
"include": ["id", "user_id"] "include": ["id","user_id"]
}, },
"sort": [ "sort": [
{"_score": {"order": "desc"}}, {"_score": {"order": "desc"}},
...@@ -420,7 +420,7 @@ class ESPerform(object): ...@@ -420,7 +420,7 @@ class ESPerform(object):
], ],
"collapse": { "collapse": {
"field": "user_id" "field": "user_id"
} }
} }
if len(have_read_topic_id_list) > 0: if len(have_read_topic_id_list) > 0:
...@@ -429,8 +429,7 @@ class ESPerform(object): ...@@ -429,8 +429,7 @@ class ESPerform(object):
"id": have_read_topic_id_list "id": have_read_topic_id_list
} }
} }
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q,
query_body=q,
offset=0, size=size, routing="6") offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]] topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
...@@ -442,7 +441,7 @@ class ESPerform(object): ...@@ -442,7 +441,7 @@ class ESPerform(object):
logging.info("topic_id_list:%s" % str(topic_id_dict)) logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict return topic_id_list,topic_id_dict
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list() return list()
physical/DBRouter.py
View file @ e6f89fd1
# !/usr/bin/env python # !/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
DATABASE_APPS_MAPPING = {'face': 'face', 'commodity': 'commodity'}
class DBRouter: class DBRouter:
...@@ -15,8 +16,8 @@ class DBRouter: ...@@ -15,8 +16,8 @@ class DBRouter:
# if model._meta.app_label == 'face': # if model._meta.app_label == 'face':
# return 'face' # return 'face'
if model._meta.app_label == 'commodity': if model._meta.app_label in DATABASE_APPS_MAPPING:
return 'commodity' return DATABASE_APPS_MAPPING[model._meta.app_label]
return None return None
def db_for_write(self, model, **hints): def db_for_write(self, model, **hints):
...@@ -26,29 +27,32 @@ class DBRouter: ...@@ -26,29 +27,32 @@ class DBRouter:
# if model._meta.app_label == 'face': # if model._meta.app_label == 'face':
# return 'face' # return 'face'
if model._meta.app_label == 'commodity': if model._meta.app_label in DATABASE_APPS_MAPPING:
return 'commodity' return DATABASE_APPS_MAPPING[model._meta.app_label]
return None return None
def allow_relation(self, obj1, obj2, **hints): def allow_relation(self, obj1, obj2, **hints):
""" """
Allow relations if a model in the user app is involved. Allow relations if a model in the user app is involved.
# """ # """
# if obj1._meta.app_label == 'face' or \ db_obj1 = DATABASE_APPS_MAPPING.get(obj1._meta.app_label)
# obj2._meta.app_label == 'face': db_obj2 = DATABASE_APPS_MAPPING.get(obj2._meta.app_label)
# return True if db_obj1 and db_obj2:
if obj1._meta.app_label == 'commodity' or \ if db_obj1 == db_obj2:
obj2._meta.app_label == 'commodity': return True
return True else:
return None return False
else:
return None
def allow_migrate(self, db, app_label, model_name=None, **hints): def allow_migrate(self, db, app_label, model_name=None, **hints):
""" """
Make sure the auth app only appears in the 'users_db' Make sure the auth app only appears in the 'users_db'
database. database.
""" """
# if app_label == 'face': if db in DATABASE_APPS_MAPPING.values():
# return db == 'face' return DATABASE_APPS_MAPPING.get(app_label) == db
if app_label == 'commodity': elif app_label in DATABASE_APPS_MAPPING:
return db == 'commodity' return False
return None return None
search/utils/product.py 0 → 100644
View file @ e6f89fd1
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics
class ProductUtils(object):
@classmethod
def get_product_sku(cls, query='', offset=0, size=10, sort_type='', filters={}):
try:
multi_fields = {
'cn_name_pre': 2,
'en_name_pre': 2,
'alias': 2,
'brand_cn_name': 2,
'brand_en_name': 2,
'brand_alias': 2,
'category_cn_name': 2,
"effect_cn_name": 2
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'cross_fields',
'operator': 'and',
'fields': query_fields,
}
sku_must_flter = cls.sku_filter(filters)
logging.info("get sku_must_flter:%s " % sku_must_flter)
logging.info("get sku_must_flter:%s " % type(sku_must_flter))
q = {
"query": {
"bool": {
"must": sku_must_flter
}
}
}
if query != '':
q = {
"query": {
"bool": {
"must": sku_must_flter,
"should": {
"multi_match": multi_match
},
"minimum_should_match": 1
}
}
}
q["sort"] = [{"comment_nums": {"order": "desc"}}]
logging.info("get product query:%s" % q)
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="product", query_body=q,
offset=offset, size=size)
return result_dict
except:
logging.error("catch exception, query_sku:%s" % traceback.format_exc())
return []
@classmethod
def sku_filter(cls, filters):
"""处理过滤器部分。"""
logging.info("get filters:%s" % filters)
f = [
{'term': {"have_image": True}},
{'term': {"is_online": True}},
{'term': {"is_deleted": False}},
]
if not filters:
return f
for k, v in filters.items():
if v in (None, '', []):
continue
if k == "brand":
f.append({"term": {"cn_name_pre": v}},
{"term": {"en_name_pre": v}},
{"term": {"alias": v}}, )
if k == "effect":
f.append({"term": {"effect_cn_name": v}})
if k == "category":
f.append({"term": {"category_cn_name": v}})
return f
search/views/group.py
View file @ e6f89fd1
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging import logging
import traceback import traceback
import json import json
import time
import datetime
from libs.cache import redis_client
from gm_rpcd.all import bind
from libs.cache import redis_client from libs.cache import redis_client
from libs.es import ESPerform from libs.es import ESPerform
from search.utils.group import GroupUtils from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes from search.utils.common import GroupSortTypes
from libs.es import ESPerform from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics from trans2es.models.pictorial import PictorialTopics
from trans2es.models.pictorial import CommunityPictorialActivity
from alpha_types.venus import PICTORIAL_ACTIVITY_SORT
@bind("physical/search/query_pictorial") @bind("physical/search/query_pictorial")
...@@ -333,6 +338,101 @@ def search_physical(query="", offset=0, size=10): ...@@ -333,6 +338,101 @@ def search_physical(query="", offset=0, size=10):
return {"search_pictorial_ids": []} return {"search_pictorial_ids": []}
@bind("physical/search/pictorial_activity")
def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
"""
榜单活动的排名和最新tab下的排序
排名根据该榜单下的帖子的总票数倒排
最新根据榜单的创建时间排序
过滤掉下线的无封面的帖子
:param activity_id:
:param offset:
:param size:
:param sort_type:
:return:
"""
try:
now = datetime.datetime.now()
activity_status = CommunityPictorialActivity.objects.filter(id=activity_id).values_list("end_time",
flat=True).first()
now = int(time.mktime(now.timetuple()))
activity_status = int(time.mktime(activity_status.timetuple()))
pictorial_ids_list = []
if sort_type == 1:
redis_key = "pictorial:activity:vote:id:" + str(activity_id)
else:
redis_key = "pictorial:activity:create:id:" + str(activity_id)
if now < activity_status:
q = {
"query": {
"bool": {
"must": [
{
"term": {
"activity_join": activity_id
}
},
# {
# "term": {
# "is_cover": True
# }
# },
{
"term": {
"is_online": True
}
}
]
}
}
}
q["sort"] = process_sort(sort_type)
es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]:
pictorial_id = item["_source"]["id"]
vote_num = item["_source"]["topic_vote_number"]
pictorial_ids_list.append({"pictorial_id": pictorial_id, "vote_num": vote_num})
redis_client.set(redis_key, json.dumps(pictorial_ids_list))
else:
redis_field_val_list = redis_client.get(redis_key)
pictorial_ids_list = json.loads(str(redis_field_val_list, encoding="utf-8"))
return {"pictorial_activity_sort": pictorial_ids_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"pictorial_activity_sort": []}
def process_sort(sorts_by):
"""处理排序部分。"""
sort_rule = []
if isinstance(sorts_by, int):
if sorts_by == PICTORIAL_ACTIVITY_SORT.VOTE_NUM:
sort_rule.append({
"topic_vote_number": {
"order": "desc"
}
})
if sorts_by == PICTORIAL_ACTIVITY_SORT.CREATE_TIME:
sort_rule.append({
"create_time": {
"order": "desc"
}
})
logging.info("get picotirial:%s" % sort_rule)
return sort_rule
@bind("physical/search/pictorialid_topic") @bind("physical/search/pictorialid_topic")
def pictorial_topic_sort_peoplehot(pictorial_id=-1, user_id=-1, offset=0, size=10): def pictorial_topic_sort_peoplehot(pictorial_id=-1, user_id=-1, offset=0, size=10):
try: try:
......
search/views/product.py 0 → 100644
View file @ e6f89fd1
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.product import ProductUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
@bind("physical/search/query_product_sku")
def product_hot_sort(query='', offset=0, size=10, filters={}):
'''
商品sku排序
:param query:
:param offset:
:param size:
:param sort_type:
:param filters:
:return:
'''
try:
res = ProductUtils.get_product_sku(query=query, offset=offset, size=size, sort_type=sort_type, filters=filters)
product_list = []
res_hit = res["hits"]
for item in res_hit:
product_id = item["_source"]["id"]
product_list.append(product_id)
return {"product_hot_ids": product_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"product_hot_ids": []}
search/views/topic.py
View file @ e6f89fd1
...@@ -14,7 +14,6 @@ from search.utils.common import * ...@@ -14,7 +14,6 @@ from search.utils.common import *
from libs.es import ESPerform from libs.es import ESPerform
from django.conf import settings from django.conf import settings
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE): def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try: try:
if user_id == -1: if user_id == -1:
...@@ -29,11 +28,10 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -29,11 +28,10 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size, recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size,
single_size=size, query_type=query_type,
query_type=query_type, filter_topic_id_list=have_read_topic_id_list,index_type="topic",routing="4,5,6")
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids) have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = { redis_dict = {
...@@ -48,31 +46,27 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -48,31 +46,27 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None, def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1,useful_tag_list =[]):
usefulrecall=-1, useful_tag_list=[]):
try: try:
topic_star_routing = "6" topic_star_routing = "6"
index_type = "topic-high-star" index_type = "topic-high-star"
device_redis_key = "" device_redis_key = ""
if query is None: if query is None:
if user_id > 0: if user_id>0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str( device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
query_type)
else: else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type) redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else: else:
topic_star_routing = "3,4,5,6" topic_star_routing = "3,4,5,6"
index_type = "topic" index_type = "topic"
if user_id > 0: if user_id>0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str( redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
query) + ":query_type:" + str(query_type) device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
else: else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str( redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
...@@ -84,17 +78,18 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -84,17 +78,18 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if query is None: if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else: else:
if offset > 0: # 首次搜索时不需要过滤已读 if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
elif len(device_redis_key) > 0: elif len(device_redis_key)>0:
redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list)
if redis_field_val_list[0]: if redis_field_val_list[0]:
if query is None: if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else: else:
if offset > 0: # 首次搜索时不需要过滤已读 if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list) have_read_topic_id_list.extend(promote_topic_list)
...@@ -103,72 +98,66 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -103,72 +98,66 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
attention_tag_list = list() attention_tag_list = list()
recommend_topic_list = list() recommend_topic_list = list()
if query is None: if query is None:
if user_id != -1: if user_id != -1:
# 有用标签召回 # 有用标签召回
if usefulrecall != -1: if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4, useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6", "topic-high-star", "6",useful_tag_list=useful_tag_list)
useful_tag_list=useful_tag_list) # user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id) # redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key) # user_similar_score_redis_list = json.loads(
# user_similar_score_redis_list = json.loads( # redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else [] size = size - len(useful_topic_id_list)
size = size - len(useful_topic_id_list) have_read_topic_id_list.extend(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
# linucb 推荐帖子 topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key) linucb_recommend_topic_id_list = list()
linucb_recommend_topic_id_list = list() recommend_topic_list = list()
recommend_topic_list = list() if b"data" in recommend_topic_dict:
if b"data" in recommend_topic_dict: linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"]) if linucb_recommend_topic_id_list == None:
if linucb_recommend_topic_id_list == None: linucb_recommend_topic_id_list = list()
linucb_recommend_topic_id_list = list() # 推荐帖子是强插的，要保证推荐帖子不在已读里
# 推荐帖子是强插的，要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
logging.warning( recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
"type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list))) recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list)) # cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index) # newcursor = cursor + 6
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8")) if len(recommend_topic_id_list) > 0:
# newcursor = cursor + 6 recommend_topic_list = recommend_topic_id_list[0:size]
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor) # redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict: if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"]) linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0: if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0:
for i in recommend_topic_list: for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)]) recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None: # if have_read_topic_id_list == None:
# have_read_topic_id_list = list() # have_read_topic_id_list = list()
# 用户关注标签 # 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id) redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else [] attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list) > 0: if len(recommend_topic_list)>0:
size = size - len(recommend_topic_list) size = size-len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list) have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list() # have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list) # have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter( # promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True) # is_online=1).values_list("topic_id",flat=True)
# #
# for topic_id in promote_recommend_topic_id_list: # for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id) # have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list() rank_topic_id_list = list()
if size > 0: if size>0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size, rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size, query=query, single_size=size,query=query, query_type=query_type,
query_type=query_type, filter_topic_id_list=have_read_topic_id_list,
filter_topic_id_list=have_read_topic_id_list, index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
index_type=index_type, routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None: # if len(recommend_topic_list) == 6 and query is None:
# if (size < 11): # if (size < 11):
...@@ -185,7 +174,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -185,7 +174,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(rank_topic_id_list) # topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list) have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000: if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list) - 30000 cut_len = len(have_read_topic_id_list)-30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:] have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = { redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list), "have_read_topic_list": json.dumps(have_read_topic_id_list),
...@@ -194,13 +183,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -194,13 +183,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 每个session key保存60天 # 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60) redis_client.expire(redis_key, 60 * 60 * 24 * 60)
if usefulrecall != -1: if usefulrecall != -1:
return recommend_topic_list, rank_topic_id_list, useful_topic_id_list return recommend_topic_list,rank_topic_id_list,useful_topic_id_list
else: else:
return recommend_topic_list, rank_topic_id_list return recommend_topic_list, rank_topic_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return [], [], [] return [],[],[]
else: else:
return [], [] return [], []
...@@ -215,8 +204,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1): ...@@ -215,8 +204,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend") @bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND, def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1,useful_tag_list=[]):
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[]):
""" """
:remark:首页推荐，目前只推荐日记 :remark:首页推荐，目前只推荐日记
:param session_id: :param session_id:
...@@ -249,34 +237,26 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic ...@@ -249,34 +237,26 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
have_read_topic_id_list = list(json.loads(redis_field_val_list[0])) have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset: if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset + size] recommend_topic_ids = have_read_topic_id_list[offset:offset+size]
else: else:
recommend_topic_ids = have_read_topic_id_list[0:size] recommend_topic_ids = have_read_topic_id_list[0:size]
else: else:
if usefulrecall != -1 and len(useful_tag_list) > 0: if usefulrecall != -1 and len(useful_tag_list)>0:
recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
tag_id=0, offset=0, query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall,useful_tag_list=useful_tag_list)
size=size, return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids}
query_type=query_type, else:
promote_topic_list=promote_topic_list, recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
usefulrecall=usefulrecall, query_type=query_type,promote_topic_list=promote_topic_list)
useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids, return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []} return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]}
else: else:
return {"linucb_topic_ids": [], "rank_topic_ids": []} return {"linucb_topic_ids": [], "rank_topic_ids": []}
@bind("physical/search/discover_page") @bind("physical/search/discover_page")
...@@ -320,12 +300,11 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10) ...@@ -320,12 +300,11 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
if not isinstance(device_id, str): if not isinstance(device_id, str):
device_id = "" device_id = ""
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
offset=offset, size=size, query=query) if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size:
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, offset=offset, size=size,
query=query, disable_collpase=True) query=query,disable_collpase=True)
return {"recommend_topic_ids": rank_topic_id_list} return {"recommend_topic_ids": rank_topic_id_list}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -347,9 +326,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -347,9 +326,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if not isinstance(user_id, int): if not isinstance(user_id, int):
user_id = -1 user_id = -1
if user_id > 0: if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id) redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id)
else: else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id) redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id)
if int(offset) == 0: if int(offset) == 0:
have_read_topic_list = list() have_read_topic_list = list()
# redis_dict = { # redis_dict = {
...@@ -362,7 +341,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -362,7 +341,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
have_read_topic_list = list() have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list'] redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list) have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else [] have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id) have_read_topic_list.append(topic_id)
...@@ -370,33 +349,28 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -370,33 +349,28 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
topic_tag_result = list() topic_tag_result = list()
result = list() result = list()
if len(topic_tag_list) != 0: if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list, topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
size, es_cli_obj, index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
index_type="topic", routing="3,4,5,6", topic_tag_size = len(topic_tag_result)
topic_tag_list=topic_tag_list) have_read_topic_list.extend(topic_tag_result)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else: else:
topic_tag_size = 0 topic_tag_size = 0
if topic_tag_size < size: if topic_tag_size <size:
size = size - topic_tag_size size = size - topic_tag_size
if topic_user_id != -1: if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
have_read_topic_list, size, es_cli_obj,
size, es_cli_obj, index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
index_type="topic", )
routing="3,4,5,6", topic_user_size = len(topic_user_result)
topic_user_id=topic_user_id have_read_topic_list.extend(topic_user_result)
) if topic_user_size < size:
topic_user_size = len(topic_user_result) size = size - topic_user_size
have_read_topic_list.extend(topic_user_result) result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
if topic_user_size < size: have_read_topic_list,
size = size - topic_user_size size, es_cli_obj,
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, index_type="topic", routing="4,5,6")
have_read_topic_list, have_read_topic_list.extend(result)
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key) # have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else [] # have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
...@@ -472,8 +446,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10): ...@@ -472,8 +446,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。""" """帖子搜索。"""
try: try:
(topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters, (topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size) sorts_by=sorts_by, offset=offset, size=size)
return { return {
"topic_ids": topic_id_list, "topic_ids": topic_id_list,
...@@ -496,10 +470,8 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -496,10 +470,8 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try: try:
must_topic_id_list = list(topic_similarity_score_dict.keys()) must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size, topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
single_size=size, must_topic_id_list=must_topic_id_list,index_type="topic",routing="4,5,6")
must_topic_id_list=must_topic_id_list, index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list} return {"recommend_topic_ids": topic_id_list}
except: except:
......
trans2es/mapping/pictorial.json
View file @ e6f89fd1
...@@ -19,6 +19,9 @@ ...@@ -19,6 +19,9 @@
"effective":{"type":"boolean"}, "effective":{"type":"boolean"},
"offline_score":{"type":"long"}, "offline_score":{"type":"long"},
"is_default":{"type":"long"}, "is_default":{"type":"long"},
"is_cover":{"type":"boolean"} "is_cover":{"type":"boolean"},
"topic_vote_number":{"type":"long"},
"activity_join":{"type":"long"}
} }
} }
\ No newline at end of file
trans2es/mapping/product.json
View file @ e6f89fd1
{ {
"dynamic":"strict", "dynamic":"strict",
"_routing": {"required": true}, "_routing": {"required": false},
"properties": { "properties": {
"id":{"type":"long"}, //id "id":{"type":"long"}, //id
"is_online":{"type":"boolean"},//上线 "is_online":{"type":"boolean"},//上线
......
trans2es/mapping/topic-high-star.json
View file @ e6f89fd1
{"dynamic":"strict", {
"dynamic":"strict",
"_routing": {"required": true}, "_routing": {"required": true},
"properties": { "properties": {
"id":{"type":"long"}, "id":{"type":"long"},
......
trans2es/mapping/topic-star-routing.json
View file @ e6f89fd1
...@@ -8,18 +8,18 @@ ...@@ -8,18 +8,18 @@
"vote_num":{"type":"long"}, "vote_num":{"type":"long"},
"total_vote_num":{"type":"long"}, "total_vote_num":{"type":"long"},
"reply_num":{"type":"long"}, "reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名 "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性 "useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"}, "share_num":{"type":"long"},
"pick_id_list":{"type":"long"}, "pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分 "offline_score":{"type":"double"},//离线算分
...@@ -44,14 +44,14 @@ ...@@ -44,14 +44,14 @@
"platform": {"type": "long"}, "platform": {"type": "long"},
"platform_id": {"type": "long"}, "platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分 "drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分 "sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID "pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称 "pictorial_name":{ // 所在组名称
"type": "text", "type": "text",
"analyzer": "keyword", "analyzer": "gm_default_index",
"search_analyzer": "keyword" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
......
trans2es/mapping/topic-star.json
View file @ e6f89fd1
...@@ -44,7 +44,7 @@ ...@@ -44,7 +44,7 @@
"platform": {"type": "long"}, "platform": {"type": "long"},
"platform_id": {"type": "long"}, "platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分 "drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分 "sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID "pictorial_id":{"type": "long"}, //所在组ID
...@@ -60,10 +60,14 @@ ...@@ -60,10 +60,14 @@
"type":"nested", "type":"nested",
"properties":{ "properties":{
"pictorial_id":{"type":"long"}, "pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"}, "real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}, "virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"} "total_vote_cnt":{"type":"long"}
} }
} }
} }
} }
\ No newline at end of file
trans2es/mapping/topic-v1.json
View file @ e6f89fd1
...@@ -60,10 +60,14 @@ ...@@ -60,10 +60,14 @@
"type":"nested", "type":"nested",
"properties":{ "properties":{
"pictorial_id":{"type":"long"}, "pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"}, "real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}, "virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"} "total_vote_cnt":{"type":"long"}
} }
} }
} }
} }
\ No newline at end of file
trans2es/mapping/topic.json
View file @ e6f89fd1
...@@ -8,18 +8,18 @@ ...@@ -8,18 +8,18 @@
"vote_num":{"type":"long"}, "vote_num":{"type":"long"},
"total_vote_num":{"type":"long"}, "total_vote_num":{"type":"long"},
"reply_num":{"type":"long"}, "reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"}, "content_level":{"type":"text"},
"user_id":{"type":"long"}, "user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名 "user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名 "user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID "group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性 "tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性 "useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签 "edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"}, "tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"}, "share_num":{"type":"long"},
"pick_id_list":{"type":"long"}, "pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分 "offline_score":{"type":"double"},//离线算分
...@@ -44,14 +44,14 @@ ...@@ -44,14 +44,14 @@
"platform": {"type": "long"}, "platform": {"type": "long"},
"platform_id": {"type": "long"}, "platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分 "drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分 "sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID "pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称 "pictorial_name":{ // 所在组名称
"type": "text", "type": "text",
"analyzer": "keyword", "analyzer": "gm_default_index",
"search_analyzer": "keyword" "search_analyzer": "gm_default_index"
}, },
"is_excellent":{"type": "long"}, "is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐 "is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
...@@ -60,7 +60,7 @@ ...@@ -60,7 +60,7 @@
"type":"nested", "type":"nested",
"properties":{ "properties":{
"pictorial_id":{"type":"long"}, "pictorial_id":{"type":"long"},
"topic_add_time":{"type":"long"}, "topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"}, "real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"}, "virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"} "total_vote_cnt":{"type":"long"}
......
trans2es/models/pictorial.py
View file @ e6f89fd1
...@@ -4,7 +4,7 @@ import logging ...@@ -4,7 +4,7 @@ import logging
import traceback import traceback
from .tag import Tag from .tag import Tag
from .topic import Topic from .topic import Topic, TopicBillBoard
class PictorialFollow(models.Model): class PictorialFollow(models.Model):
...@@ -78,7 +78,8 @@ class Pictorial(models.Model): ...@@ -78,7 +78,8 @@ class Pictorial(models.Model):
for topic_id in topic_id_list: for topic_id in topic_id_list:
topic_id_object = Topic.objects.filter(id=int(topic_id)).first() topic_id_object = Topic.objects.filter(id=int(topic_id)).first()
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5,6]: if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5,
6]:
effective_num += 1 effective_num += 1
if effective_num >= 5: if effective_num >= 5:
ret = True ret = True
...@@ -145,6 +146,31 @@ class Pictorial(models.Model): ...@@ -145,6 +146,31 @@ class Pictorial(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return False
def get_topic_vote_number(self):
try:
topic_vote = TopicBillBoard.objects.filter(pictorial_id=self.id).values("real_vote_cnt", "virt_vote_cnt")
total_vote_cnt = 0
if topic_vote:
for item in topic_vote:
total_vote_cnt += int(item["virt_vote_cnt"]) + int(item["real_vote_cnt"])
return total_vote_cnt
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
def get_activity_join(self):
try:
activicy_list = list(CommunityPictorialActivityRelation.objects.filter(pictorial_id=self.id, is_online=True,
is_deleted=False).values_list(
"pictorial_activity_id", flat=True))
return activicy_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
class PictorialTag(models.Model): class PictorialTag(models.Model):
"""画报关注标签""" """画报关注标签"""
...@@ -160,3 +186,31 @@ class PictorialTag(models.Model): ...@@ -160,3 +186,31 @@ class PictorialTag(models.Model):
pictorial_id = models.BigIntegerField(verbose_name=u'画报ID', max_length=20) pictorial_id = models.BigIntegerField(verbose_name=u'画报ID', max_length=20)
tag_id = models.BigIntegerField(verbose_name=u'标签ID', max_length=20) tag_id = models.BigIntegerField(verbose_name=u'标签ID', max_length=20)
is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1) is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
class CommunityPictorialActivity(models.Model):
"""榜单活动"""
class Meta:
verbose_name = u"画报标签"
app_label = "community"
db_table = "community_pictorial_activity"
start_time = models.DateTimeField(verbose_name=u'开始时间', default=datetime.datetime.fromtimestamp(0))
end_time = models.DateTimeField(verbose_name=u'结束时间', default=datetime.datetime.fromtimestamp(0))
is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
is_deleted = models.BooleanField(verbose_name=u'是否被删除', max_length=1)
class CommunityPictorialActivityRelation(models.Model):
"""榜单活动关系"""
class Meta:
verbose_name = u"画报标签"
app_label = "community"
db_table = "community_pictorial_activity_relation"
is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
is_deleted = models.BooleanField(verbose_name=u'是否被删除', max_length=1)
pictorial_activity_id = models.BigIntegerField(verbose_name=u'活动ID', max_length=20)
pictorial_id = models.BigIntegerField(verbose_name=u'榜单ID', max_length=20)
trans2es/models/topic.py
View file @ e6f89fd1
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function from __future__ import unicode_literals, absolute_import, print_function
import time
from django.conf import settings from django.conf import settings
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
import traceback import traceback
import logging import logging
import datetime import datetime
import time
from libs.es import ESPerform from libs.es import ESPerform
from django.db import models from django.db import models
from alpha_types.venus import GRAP_PLATFORM from alpha_types.venus import GRAP_PLATFORM
...@@ -281,9 +280,6 @@ class Topic(models.Model): ...@@ -281,9 +280,6 @@ class Topic(models.Model):
def get_related_billboard(self): def get_related_billboard(self):
try: try:
# pictorial_ids = Topic.get_pictorial_id(self)
pictorial_id_list = []
related_billboard_list = list() related_billboard_list = list()
pictorials = PictorialTopic.objects.filter(topic_id=self.id).values("pictorial_id", "create_time") pictorials = PictorialTopic.objects.filter(topic_id=self.id).values("pictorial_id", "create_time")
...@@ -293,10 +289,11 @@ class Topic(models.Model): ...@@ -293,10 +289,11 @@ class Topic(models.Model):
topic_id=self.id).values().first() topic_id=self.id).values().first()
if query_result == None: if query_result == None:
related_billboard_list.append({"pictorial_id": pictorial_id.get('pictorial_id', None), "real_vote_cnt": 0, related_billboard_list.append(
"virt_vote_cnt": 0, "total_vote_cnt": 0, {"pictorial_id": pictorial_id.get('pictorial_id', None), "real_vote_cnt": 0,
"topic_add_createtime": int( "virt_vote_cnt": 0, "total_vote_cnt": 0,
time.mktime(pictorial_id.get("create_time", None).timetuple()))}) "topic_add_createtime": int(
time.mktime(pictorial_id.get("create_time", None).timetuple()))})
else: else:
......
trans2es/utils/pictorial_transfer.py
View file @ e6f89fd1
...@@ -68,6 +68,8 @@ class PictorialTransfer(object): ...@@ -68,6 +68,8 @@ class PictorialTransfer(object):
res["offline_score"] = cls.get_offline_score(instance, res["topic_id_list"]) res["offline_score"] = cls.get_offline_score(instance, res["topic_id_list"])
res["is_default"] = instance.is_default res["is_default"] = instance.is_default
res["is_cover"] = instance.get_is_cover(res["topic_id_list"]) res["is_cover"] = instance.get_is_cover(res["topic_id_list"])
res["topic_vote_number"] = instance.get_topic_vote_number()
res["activity_join"] = instance.get_activity_join()
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
trans2es/utils/tag_transfer.py
View file @ e6f89fd1
...@@ -8,16 +8,15 @@ import logging ...@@ -8,16 +8,15 @@ import logging
import traceback import traceback
from libs.tools import tzlc from libs.tools import tzlc
from trans2es.models.topic import Topic from trans2es.models.topic import Topic
from trans2es.models.tag import TopicTag,CommunityTagType,CommunityTagTypeRelation from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation
import datetime import datetime
from django.conf import settings from django.conf import settings
class TagTransfer(object): class TagTransfer(object):
@classmethod @classmethod
def get_tag_name_data(cls,instance): def get_tag_name_data(cls, instance):
try: try:
res = dict() res = dict()
res["name"] = instance.name res["name"] = instance.name
...@@ -28,7 +27,7 @@ class TagTransfer(object): ...@@ -28,7 +27,7 @@ class TagTransfer(object):
return dict() return dict()
@classmethod @classmethod
def get_tag_data(cls,instance): def get_tag_data(cls, instance):
try: try:
res = dict() res = dict()
...@@ -36,14 +35,14 @@ class TagTransfer(object): ...@@ -36,14 +35,14 @@ class TagTransfer(object):
tag_name_terms_list = list() tag_name_terms_list = list()
for i in range(len(instance.name)): for i in range(len(instance.name)):
for j in range(i,len(instance.name)+1): for j in range(i, len(instance.name) + 1):
name_term = instance.name[i:j].strip() name_term = instance.name[i:j].strip()
if name_term: if name_term:
tag_name_terms_list.append(name_term.lower()) tag_name_terms_list.append(name_term.lower())
res["suggest"] = { res["suggest"] = {
"input":tag_name_terms_list, "input": tag_name_terms_list,
"contexts":{ "contexts": {
"is_online": [instance.is_online], "is_online": [instance.is_online],
"is_deleted": [instance.is_deleted] "is_deleted": [instance.is_deleted]
} }
...@@ -54,24 +53,27 @@ class TagTransfer(object): ...@@ -54,24 +53,27 @@ class TagTransfer(object):
topic_num = 0 topic_num = 0
res["near_new_topic_num"] = topic_num res["near_new_topic_num"] = topic_num
if instance.is_online==True and instance.is_deleted==False: if instance.is_online == True and instance.is_deleted == False:
topic_id_list = list() topic_id_list = list()
sql_result_results = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter( sql_result_results = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(
tag_id=instance.id).values_list("topic_id", "is_online")) tag_id=instance.id).values_list("topic_id", "is_online"))
for topic_id,is_online in sql_result_results: for topic_id, is_online in sql_result_results:
if is_online: if is_online:
topic_id_list.append(topic_id) topic_id_list.append(topic_id)
time_base_val = datetime.datetime.strftime(datetime.datetime.now()+datetime.timedelta(-7), "%Y-%m-%d") time_base_val = datetime.datetime.strftime(datetime.datetime.now() + datetime.timedelta(-7), "%Y-%m-%d")
for topic_begin_index in range(0,len(topic_id_list),100): for topic_begin_index in range(0, len(topic_id_list), 100):
cur_topic_num = Topic.objects.using(settings.SLAVE_DB_NAME).filter(id__in=topic_id_list[topic_begin_index:topic_begin_index+100],create_time__gte=time_base_val).count() cur_topic_num = Topic.objects.using(settings.SLAVE_DB_NAME).filter(
id__in=topic_id_list[topic_begin_index:topic_begin_index + 100],
create_time__gte=time_base_val).count()
topic_num += cur_topic_num topic_num += cur_topic_num
res["near_new_topic_num"] = topic_num res["near_new_topic_num"] = topic_num
tag_type_sql_list = CommunityTagTypeRelation.objects.using(settings.SLAVE_DB_NAME).filter(tag_id=instance.id).values_list("tag_type_id",flat=True) tag_type_sql_list = CommunityTagTypeRelation.objects.using(settings.SLAVE_DB_NAME).filter(
tag_id=instance.id).values_list("tag_type_id", flat=True)
tag_type_list = list() tag_type_list = list()
for tag_type_id in tag_type_sql_list: for tag_type_id in tag_type_sql_list:
tag_type_list.append(tag_type_id) tag_type_list.append(tag_type_id)
...@@ -83,4 +85,4 @@ class TagTransfer(object): ...@@ -83,4 +85,4 @@ class TagTransfer(object):
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict() return dict()
\ No newline at end of file
trans2es/utils/topic_transfer.py
View file @ e6f89fd1
...@@ -8,7 +8,6 @@ from libs.tools import tzlc ...@@ -8,7 +8,6 @@ from libs.tools import tzlc
import time import time
import re import re
import datetime import datetime
from trans2es.models.user import User from trans2es.models.user import User
from trans2es.models.topic import ExcellentTopic, TopicHomeRecommend from trans2es.models.topic import ExcellentTopic, TopicHomeRecommend
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment