Commit e6f89fd1 authored by 李小芳's avatar 李小芳

Merge branch '721_goods-sort' into 'test'

721 goods sort

See merge request alpha/physical!390
parents 415b0086 74a5dd9f
......@@ -155,9 +155,9 @@ class ESPerform(object):
bulk_actions = []
if sub_index_name == "topic" or \
sub_index_name == "topic-star-routing" or \
sub_index_name == "topic-high-star":
if sub_index_name=="topic" or \
sub_index_name=="topic-star-routing" or \
sub_index_name=="topic-high-star":
for data in data_list:
if data:
bulk_actions.append({
......@@ -232,7 +232,7 @@ class ESPerform(object):
return {"total_count": 0, "hits": []}
@classmethod
def get_analyze_results(cls, es_cli, sub_index_name, query_body):
def get_analyze_results(cls,es_cli, sub_index_name, query_body):
try:
assert (es_cli is not None)
......@@ -242,7 +242,7 @@ class ESPerform(object):
logging.error("index:%s is not existing,get_search_results error!" % official_index_name)
return None
res = es_cli.indices.analyze(index=official_index_name, body=query_body)
res = es_cli.indices.analyze(index=official_index_name,body=query_body)
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -276,7 +276,7 @@ class ESPerform(object):
if high_num > 3:
logging.info("check es_nodes_load high,cpu load:%s,ori_cpu_info:%s" % (
str(es_nodes_list), str(es_nodes_info_list)))
str(es_nodes_list), str(es_nodes_info_list)))
return True
else:
return False
......@@ -298,8 +298,8 @@ class ESPerform(object):
functions_list += [
{
"filter": {
"constant_score": {
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 6}}
}
},
......@@ -307,8 +307,8 @@ class ESPerform(object):
},
{
"filter": {
"constant_score": {
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 5}}
}
},
......@@ -316,8 +316,8 @@ class ESPerform(object):
},
{
"filter": {
"constant_score": {
"filter": {
"constant_score":{
"filter":{
"term": {"content_level": 4}}
}
},
......@@ -411,7 +411,7 @@ class ESPerform(object):
}
},
"_source": {
"include": ["id", "user_id"]
"include": ["id","user_id"]
},
"sort": [
{"_score": {"order": "desc"}},
......@@ -420,7 +420,7 @@ class ESPerform(object):
],
"collapse": {
"field": "user_id"
}
}
}
if len(have_read_topic_id_list) > 0:
......@@ -429,8 +429,7 @@ class ESPerform(object):
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star", query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
......@@ -442,7 +441,7 @@ class ESPerform(object):
logging.info("topic_id_list:%s" % str(topic_id_dict))
return topic_id_list, topic_id_dict
return topic_id_list,topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list()
# !/usr/bin/env python
# -*- coding: utf-8 -*-
DATABASE_APPS_MAPPING = {'face': 'face', 'commodity': 'commodity'}
class DBRouter:
......@@ -15,8 +16,8 @@ class DBRouter:
# if model._meta.app_label == 'face':
# return 'face'
if model._meta.app_label == 'commodity':
return 'commodity'
if model._meta.app_label in DATABASE_APPS_MAPPING:
return DATABASE_APPS_MAPPING[model._meta.app_label]
return None
def db_for_write(self, model, **hints):
......@@ -26,29 +27,32 @@ class DBRouter:
# if model._meta.app_label == 'face':
# return 'face'
if model._meta.app_label == 'commodity':
return 'commodity'
if model._meta.app_label in DATABASE_APPS_MAPPING:
return DATABASE_APPS_MAPPING[model._meta.app_label]
return None
def allow_relation(self, obj1, obj2, **hints):
"""
Allow relations if a model in the user app is involved.
# """
# if obj1._meta.app_label == 'face' or \
# obj2._meta.app_label == 'face':
# return True
if obj1._meta.app_label == 'commodity' or \
obj2._meta.app_label == 'commodity':
return True
return None
db_obj1 = DATABASE_APPS_MAPPING.get(obj1._meta.app_label)
db_obj2 = DATABASE_APPS_MAPPING.get(obj2._meta.app_label)
if db_obj1 and db_obj2:
if db_obj1 == db_obj2:
return True
else:
return False
else:
return None
def allow_migrate(self, db, app_label, model_name=None, **hints):
"""
Make sure the auth app only appears in the 'users_db'
database.
"""
# if app_label == 'face':
# return db == 'face'
if app_label == 'commodity':
return db == 'commodity'
if db in DATABASE_APPS_MAPPING.values():
return DATABASE_APPS_MAPPING.get(app_label) == db
elif app_label in DATABASE_APPS_MAPPING:
return False
return None
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics
class ProductUtils(object):
@classmethod
def get_product_sku(cls, query='', offset=0, size=10, sort_type='', filters={}):
try:
multi_fields = {
'cn_name_pre': 2,
'en_name_pre': 2,
'alias': 2,
'brand_cn_name': 2,
'brand_en_name': 2,
'brand_alias': 2,
'category_cn_name': 2,
"effect_cn_name": 2
}
query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
multi_match = {
'query': query,
'type': 'cross_fields',
'operator': 'and',
'fields': query_fields,
}
sku_must_flter = cls.sku_filter(filters)
logging.info("get sku_must_flter:%s " % sku_must_flter)
logging.info("get sku_must_flter:%s " % type(sku_must_flter))
q = {
"query": {
"bool": {
"must": sku_must_flter
}
}
}
if query != '':
q = {
"query": {
"bool": {
"must": sku_must_flter,
"should": {
"multi_match": multi_match
},
"minimum_should_match": 1
}
}
}
q["sort"] = [{"comment_nums": {"order": "desc"}}]
logging.info("get product query:%s" % q)
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="product", query_body=q,
offset=offset, size=size)
return result_dict
except:
logging.error("catch exception, query_sku:%s" % traceback.format_exc())
return []
@classmethod
def sku_filter(cls, filters):
"""处理过滤器部分。"""
logging.info("get filters:%s" % filters)
f = [
{'term': {"have_image": True}},
{'term': {"is_online": True}},
{'term': {"is_deleted": False}},
]
if not filters:
return f
for k, v in filters.items():
if v in (None, '', []):
continue
if k == "brand":
f.append({"term": {"cn_name_pre": v}},
{"term": {"en_name_pre": v}},
{"term": {"alias": v}}, )
if k == "effect":
f.append({"term": {"effect_cn_name": v}})
if k == "category":
f.append({"term": {"category_cn_name": v}})
return f
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
import time
import datetime
from libs.cache import redis_client
from gm_rpcd.all import bind
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics
from trans2es.models.pictorial import CommunityPictorialActivity
from alpha_types.venus import PICTORIAL_ACTIVITY_SORT
@bind("physical/search/query_pictorial")
......@@ -333,6 +338,101 @@ def search_physical(query="", offset=0, size=10):
return {"search_pictorial_ids": []}
@bind("physical/search/pictorial_activity")
def pictorial_activity_sort(activity_id=0, offset=0, size=10, sort_type=1):
"""
榜单活动的排名和最新tab下的排序
排名根据该榜单下的帖子的总票数倒排
最新根据榜单的创建时间排序
过滤掉下线的无封面的帖子
:param activity_id:
:param offset:
:param size:
:param sort_type:
:return:
"""
try:
now = datetime.datetime.now()
activity_status = CommunityPictorialActivity.objects.filter(id=activity_id).values_list("end_time",
flat=True).first()
now = int(time.mktime(now.timetuple()))
activity_status = int(time.mktime(activity_status.timetuple()))
pictorial_ids_list = []
if sort_type == 1:
redis_key = "pictorial:activity:vote:id:" + str(activity_id)
else:
redis_key = "pictorial:activity:create:id:" + str(activity_id)
if now < activity_status:
q = {
"query": {
"bool": {
"must": [
{
"term": {
"activity_join": activity_id
}
},
# {
# "term": {
# "is_cover": True
# }
# },
{
"term": {
"is_online": True
}
}
]
}
}
}
q["sort"] = process_sort(sort_type)
es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]:
pictorial_id = item["_source"]["id"]
vote_num = item["_source"]["topic_vote_number"]
pictorial_ids_list.append({"pictorial_id": pictorial_id, "vote_num": vote_num})
redis_client.set(redis_key, json.dumps(pictorial_ids_list))
else:
redis_field_val_list = redis_client.get(redis_key)
pictorial_ids_list = json.loads(str(redis_field_val_list, encoding="utf-8"))
return {"pictorial_activity_sort": pictorial_ids_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"pictorial_activity_sort": []}
def process_sort(sorts_by):
"""处理排序部分。"""
sort_rule = []
if isinstance(sorts_by, int):
if sorts_by == PICTORIAL_ACTIVITY_SORT.VOTE_NUM:
sort_rule.append({
"topic_vote_number": {
"order": "desc"
}
})
if sorts_by == PICTORIAL_ACTIVITY_SORT.CREATE_TIME:
sort_rule.append({
"create_time": {
"order": "desc"
}
})
logging.info("get picotirial:%s" % sort_rule)
return sort_rule
@bind("physical/search/pictorialid_topic")
def pictorial_topic_sort_peoplehot(pictorial_id=-1, user_id=-1, offset=0, size=10):
try:
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from gm_rpcd.all import bind
import logging
import traceback
import json
from libs.cache import redis_client
from libs.es import ESPerform
from search.utils.product import ProductUtils
from search.utils.common import GroupSortTypes
from libs.es import ESPerform
@bind("physical/search/query_product_sku")
def product_hot_sort(query='', offset=0, size=10, filters={}):
'''
商品sku排序
:param query:
:param offset:
:param size:
:param sort_type:
:param filters:
:return:
'''
try:
res = ProductUtils.get_product_sku(query=query, offset=offset, size=size, sort_type=sort_type, filters=filters)
product_list = []
res_hit = res["hits"]
for item in res_hit:
product_id = item["_source"]["id"]
product_list.append(product_id)
return {"product_hot_ids": product_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"product_hot_ids": []}
......@@ -14,7 +14,6 @@ from search.utils.common import *
from libs.es import ESPerform
from django.conf import settings
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
try:
if user_id == -1:
......@@ -29,11 +28,10 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,
single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
recommend_topic_ids = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size,single_size=size,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,index_type="topic",routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = {
......@@ -48,31 +46,27 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1, useful_tag_list=[]):
query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list = [],disable_collpase=False,usefulrecall = -1,useful_tag_list =[]):
try:
topic_star_routing = "6"
index_type = "topic-high-star"
device_redis_key = ""
if query is None:
if user_id > 0:
if user_id>0:
redis_key = "physical:home_recommend" + ":user_id:" + str(user_id) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(
query_type)
device_redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_recommend" + ":device_id:" + device_id + ":query_type:" + str(query_type)
else:
topic_star_routing = "3,4,5,6"
index_type = "topic"
if user_id > 0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(
query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
if user_id>0:
redis_key = "physical:home_query" + ":user_id:" + str(user_id) + ":query:" + str(query) + ":query_type:" + str(query_type)
device_redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
else:
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(
query) + ":query_type:" + str(query_type)
redis_key = "physical:home_query" + ":device_id:" + device_id + ":query:" + str(query) + ":query_type:" + str(query_type)
redis_field_list = [b'have_read_topic_list']
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
......@@ -84,17 +78,18 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset > 0: # 首次搜索时不需要过滤已读
if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
elif len(device_redis_key) > 0:
elif len(device_redis_key)>0:
redis_field_val_list = redis_client.hmget(device_redis_key, redis_field_list)
if redis_field_val_list[0]:
if query is None:
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
else:
if offset > 0: # 首次搜索时不需要过滤已读
if offset>0: # 首次搜索时不需要过滤已读
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if have_read_topic_id_list == None:
have_read_topic_id_list = list()
have_read_topic_id_list.extend(promote_topic_list)
......@@ -103,72 +98,66 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
attention_tag_list = list()
recommend_topic_list = list()
if query is None:
if user_id != -1:
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6",
useful_tag_list=useful_tag_list)
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list()
recommend_topic_list = list()
if b"data" in recommend_topic_dict:
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
if linucb_recommend_topic_id_list == None:
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的,要保证推荐帖子不在已读里
logging.warning(
"type1:%s,type2:%s" % (type(linucb_recommend_topic_id_list), type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
if user_id != -1:
# 有用标签召回
if usefulrecall != -1:
useful_topic_id_list = TopicUtils.userful_tag_topic_list(user_id, have_read_topic_id_list, 4,
"topic-high-star", "6",useful_tag_list=useful_tag_list)
# user_similar_score_redis_key = "physical:user_similar_score:user_id:" + str(user_id)
# redis_user_similar_score_redis_val = redis_client.get(user_similar_score_redis_key)
# user_similar_score_redis_list = json.loads(
# redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
size = size - len(useful_topic_id_list)
have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list()
recommend_topic_list = list()
if b"data" in recommend_topic_dict:
linucb_recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
if linucb_recommend_topic_id_list == None:
linucb_recommend_topic_id_list = list()
# 推荐帖子是强插的,要保证推荐帖子不在已读里
logging.warning("type1:%s,type2:%s"%(type(linucb_recommend_topic_id_list),type(have_read_topic_id_list)))
recommend_topic_id_list = list(set(linucb_recommend_topic_id_list) - set(have_read_topic_id_list))
recommend_topic_id_list.sort(key=linucb_recommend_topic_id_list.index)
# cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 6
if len(recommend_topic_id_list) > 0:
recommend_topic_list = recommend_topic_id_list[0:size]
# redis_client.hset(topic_recommend_redis_key, "cursor", newcursor)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) > 0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
# have_read_topic_id_list = list()
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list) > 0:
size = size - len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True)
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
if b"datadict" in recommend_topic_dict:
linucb_recommend_topic_id_dict = json.loads(recommend_topic_dict[b"datadict"])
if linucb_recommend_topic_id_dict is not None and len(recommend_topic_list) >0:
for i in recommend_topic_list:
recommend_topic_user_list.append(linucb_recommend_topic_id_dict[str(i)])
# if have_read_topic_id_list == None:
# have_read_topic_id_list = list()
# 用户关注标签
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list)>0:
size = size-len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
# have_read_topic_id_list_add_promote = list()
# have_read_topic_id_list_add_promote.extend(have_read_topic_id_list)
# promote_recommend_topic_id_list = TopicHomeRecommend.objects.using(settings.SLAVE_DB_NAME).filter(
# is_online=1).values_list("topic_id",flat=True)
#
# for topic_id in promote_recommend_topic_id_list:
# have_read_topic_id_list_add_promote.append(topic_id)
rank_topic_id_list = list()
if size > 0:
if size>0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size,
single_size=size, query=query,
query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type, routing=topic_star_routing,
attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
index_type=index_type,routing=topic_star_routing,attention_tag_list=attention_tag_list,linucb_user_id_list=recommend_topic_user_list,disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None:
# if (size < 11):
......@@ -185,7 +174,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# topic_id_list.extend(rank_topic_id_list)
have_read_topic_id_list.extend(rank_topic_id_list)
if len(have_read_topic_id_list) > 30000:
cut_len = len(have_read_topic_id_list) - 30000
cut_len = len(have_read_topic_id_list)-30000
have_read_topic_id_list = have_read_topic_id_list[cut_len:]
redis_dict = {
"have_read_topic_list": json.dumps(have_read_topic_id_list),
......@@ -194,13 +183,13 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# 每个session key保存60天
redis_client.expire(redis_key, 60 * 60 * 24 * 60)
if usefulrecall != -1:
return recommend_topic_list, rank_topic_id_list, useful_topic_id_list
return recommend_topic_list,rank_topic_id_list,useful_topic_id_list
else:
return recommend_topic_list, rank_topic_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return [], [], []
return [],[],[]
else:
return [], []
......@@ -215,8 +204,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[]):
def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=TopicPageType.HOME_RECOMMEND,promote_topic_list=[],usefulrecall = -1,useful_tag_list=[]):
"""
:remark:首页推荐,目前只推荐日记
:param session_id:
......@@ -249,34 +237,26 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
have_read_topic_id_list = list(json.loads(redis_field_val_list[0]))
if len(have_read_topic_id_list) > offset:
recommend_topic_ids = have_read_topic_id_list[offset:offset + size]
recommend_topic_ids = have_read_topic_id_list[offset:offset+size]
else:
recommend_topic_ids = have_read_topic_id_list[0:size]
else:
if usefulrecall != -1 and len(useful_tag_list) > 0:
recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id,
tag_id=0, offset=0,
size=size,
query_type=query_type,
promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids}
else:
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size,
query_type=query_type,
promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids}
if usefulrecall != -1 and len(useful_tag_list)>0:
recommend_topic_ids,rank_topic_ids,useful_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list,usefulrecall=usefulrecall,useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids,"useful_topic_ids":useful_topic_ids}
else:
recommend_topic_ids,rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, offset=0, size=size,
query_type=query_type,promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids,"rank_topic_ids":rank_topic_ids}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1:
return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": []}
return {"linucb_topic_ids": [],"rank_topic_ids":[],"useful_topic_ids":[]}
else:
return {"linucb_topic_ids": [], "rank_topic_ids": []}
return {"linucb_topic_ids": [], "rank_topic_ids": []}
@bind("physical/search/discover_page")
......@@ -320,12 +300,11 @@ def home_query(device_id="", tag_id=-1, user_id=-1, query="", offset=0, size=10)
if not isinstance(device_id, str):
device_id = ""
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size, query=query)
if len(rank_topic_id_list) > 0 and len(rank_topic_id_list) < size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id, offset=offset, size=size, query=query)
if len(rank_topic_id_list)>0 and len(rank_topic_id_list)<size:
recommend_topic_list, rank_topic_id_list = get_home_recommend_topic_ids(user_id, device_id, tag_id,
offset=offset, size=size,
query=query, disable_collpase=True)
query=query,disable_collpase=True)
return {"recommend_topic_ids": rank_topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -347,9 +326,9 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
if not isinstance(user_id, int):
user_id = -1
if user_id > 0:
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:" + str(topic_id)
redis_key = "physical:topic_detail_recommend" + ":user_id:" + str(user_id) + "topic_id:"+str(topic_id)
else:
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:" + str(topic_id)
redis_key = "physical:topic_detail_recommend" + ":device_id:" + device_id + "topic_id:"+str(topic_id)
if int(offset) == 0:
have_read_topic_list = list()
# redis_dict = {
......@@ -362,7 +341,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
have_read_topic_list = list()
redis_field_list = [b'have_read_topic_list']
have_read_topic_redis_data = redis_client.hmget(redis_key, redis_field_list)
have_read_topic_redis_data = redis_client.hmget(redis_key,redis_field_list)
have_read_topic_list = json.loads(have_read_topic_redis_data[0]) if have_read_topic_redis_data[0] else []
es_cli_obj = ESPerform.get_cli()
have_read_topic_list.append(topic_id)
......@@ -370,33 +349,28 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
topic_tag_result = list()
result = list()
if len(topic_tag_list) != 0:
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="3,4,5,6",
topic_tag_list=topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
topic_tag_result = TopicUtils.top_get_topic_detail_recommend_list(user_id,topic_id,have_read_topic_list,size,es_cli_obj,
index_type="topic",routing="3,4,5,6",topic_tag_list = topic_tag_list)
topic_tag_size = len(topic_tag_result)
have_read_topic_list.extend(topic_tag_result)
else:
topic_tag_size = 0
if topic_tag_size < size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic",
routing="3,4,5,6",
topic_user_id=topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
topic_tag_size = 0
if topic_tag_size <size:
size = size - topic_tag_size
if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="3,4,5,6",topic_user_id = topic_user_id
)
topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result)
if topic_user_size < size:
size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list,
size, es_cli_obj,
index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key)
# have_read_topic_list = json.loads(have_read_topic_redis_data) if have_read_topic_redis_data else []
......@@ -472,8 +446,8 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
(topic_id_list, total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
(topic_id_list,total_count) = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size)
return {
"topic_ids": topic_id_list,
......@@ -496,10 +470,8 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try:
must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,
single_size=size,
must_topic_id_list=must_topic_id_list, index_type="topic",
routing="4,5,6")
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
must_topic_id_list=must_topic_id_list,index_type="topic",routing="4,5,6")
return {"recommend_topic_ids": topic_id_list}
except:
......
......@@ -19,6 +19,9 @@
"effective":{"type":"boolean"},
"offline_score":{"type":"long"},
"is_default":{"type":"long"},
"is_cover":{"type":"boolean"}
"is_cover":{"type":"boolean"},
"topic_vote_number":{"type":"long"},
"activity_join":{"type":"long"}
}
}
\ No newline at end of file
}
{
"dynamic":"strict",
"_routing": {"required": true},
"_routing": {"required": false},
"properties": {
"id":{"type":"long"}, //id
"is_online":{"type":"boolean"},//上线
......
{"dynamic":"strict",
{
"dynamic":"strict",
"_routing": {"required": true},
"properties": {
"id":{"type":"long"},
......
......@@ -8,18 +8,18 @@
"vote_num":{"type":"long"},
"total_vote_num":{"type":"long"},
"reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
......@@ -44,14 +44,14 @@
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "keyword",
"search_analyzer": "keyword"
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
......
......@@ -44,7 +44,7 @@
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
......@@ -60,10 +60,14 @@
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"}
}
}
}
}
\ No newline at end of file
}
......@@ -60,10 +60,14 @@
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"}
}
}
}
}
\ No newline at end of file
}
......@@ -8,18 +8,18 @@
"vote_num":{"type":"long"},
"total_vote_num":{"type":"long"},
"reply_num":{"type":"long"},
"name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"description":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"content":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"content_level":{"type":"text"},
"user_id":{"type":"long"},
"user_nick_name":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},//帖子用户名
"user_nick_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},//帖子用户名
"user_nick_name_pre": {"type":"text","analyzer":"keyword"}, //不切词的用户名
"group_id":{"type":"long"}, //所在组ID
"tag_list":{"type":"long"},//标签属性
"useful_tag_list":{"type":"long"},//有用标签属性
"edit_tag_list":{"type":"long"},//编辑标签
"tag_name_list":{"type":"text","analyzer":"keyword","search_analyzer":"keyword"},
"tag_name_list":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"},
"share_num":{"type":"long"},
"pick_id_list":{"type":"long"},
"offline_score":{"type":"double"},//离线算分
......@@ -44,14 +44,14 @@
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "long"}, // 人工降分
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "keyword",
"search_analyzer": "keyword"
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
},
"is_excellent":{"type": "long"},
"is_operation_home_recommend": {"type": "boolean"}, //是否首页运营推荐
......@@ -60,7 +60,7 @@
"type":"nested",
"properties":{
"pictorial_id":{"type":"long"},
"topic_add_time":{"type":"long"},
"topic_add_createtime":{"type":"long"},
"real_vote_cnt":{"type":"long"},
"virt_vote_cnt":{"type":"long"},
"total_vote_cnt":{"type":"long"}
......
......@@ -4,7 +4,7 @@ import logging
import traceback
from .tag import Tag
from .topic import Topic
from .topic import Topic, TopicBillBoard
class PictorialFollow(models.Model):
......@@ -78,7 +78,8 @@ class Pictorial(models.Model):
for topic_id in topic_id_list:
topic_id_object = Topic.objects.filter(id=int(topic_id)).first()
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5,6]:
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5,
6]:
effective_num += 1
if effective_num >= 5:
ret = True
......@@ -145,6 +146,31 @@ class Pictorial(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
def get_topic_vote_number(self):
try:
topic_vote = TopicBillBoard.objects.filter(pictorial_id=self.id).values("real_vote_cnt", "virt_vote_cnt")
total_vote_cnt = 0
if topic_vote:
for item in topic_vote:
total_vote_cnt += int(item["virt_vote_cnt"]) + int(item["real_vote_cnt"])
return total_vote_cnt
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0
def get_activity_join(self):
try:
activicy_list = list(CommunityPictorialActivityRelation.objects.filter(pictorial_id=self.id, is_online=True,
is_deleted=False).values_list(
"pictorial_activity_id", flat=True))
return activicy_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
class PictorialTag(models.Model):
"""画报关注标签"""
......@@ -160,3 +186,31 @@ class PictorialTag(models.Model):
pictorial_id = models.BigIntegerField(verbose_name=u'画报ID', max_length=20)
tag_id = models.BigIntegerField(verbose_name=u'标签ID', max_length=20)
is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
class CommunityPictorialActivity(models.Model):
"""榜单活动"""
class Meta:
verbose_name = u"画报标签"
app_label = "community"
db_table = "community_pictorial_activity"
start_time = models.DateTimeField(verbose_name=u'开始时间', default=datetime.datetime.fromtimestamp(0))
end_time = models.DateTimeField(verbose_name=u'结束时间', default=datetime.datetime.fromtimestamp(0))
is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
is_deleted = models.BooleanField(verbose_name=u'是否被删除', max_length=1)
class CommunityPictorialActivityRelation(models.Model):
"""榜单活动关系"""
class Meta:
verbose_name = u"画报标签"
app_label = "community"
db_table = "community_pictorial_activity_relation"
is_online = models.BooleanField(verbose_name=u'是否上线', max_length=1)
is_deleted = models.BooleanField(verbose_name=u'是否被删除', max_length=1)
pictorial_activity_id = models.BigIntegerField(verbose_name=u'活动ID', max_length=20)
pictorial_id = models.BigIntegerField(verbose_name=u'榜单ID', max_length=20)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function
import time
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
import traceback
import logging
import datetime
import time
from libs.es import ESPerform
from django.db import models
from alpha_types.venus import GRAP_PLATFORM
......@@ -281,9 +280,6 @@ class Topic(models.Model):
def get_related_billboard(self):
try:
# pictorial_ids = Topic.get_pictorial_id(self)
pictorial_id_list = []
related_billboard_list = list()
pictorials = PictorialTopic.objects.filter(topic_id=self.id).values("pictorial_id", "create_time")
......@@ -293,10 +289,11 @@ class Topic(models.Model):
topic_id=self.id).values().first()
if query_result == None:
related_billboard_list.append({"pictorial_id": pictorial_id.get('pictorial_id', None), "real_vote_cnt": 0,
"virt_vote_cnt": 0, "total_vote_cnt": 0,
"topic_add_createtime": int(
time.mktime(pictorial_id.get("create_time", None).timetuple()))})
related_billboard_list.append(
{"pictorial_id": pictorial_id.get('pictorial_id', None), "real_vote_cnt": 0,
"virt_vote_cnt": 0, "total_vote_cnt": 0,
"topic_add_createtime": int(
time.mktime(pictorial_id.get("create_time", None).timetuple()))})
else:
......
......@@ -68,6 +68,8 @@ class PictorialTransfer(object):
res["offline_score"] = cls.get_offline_score(instance, res["topic_id_list"])
res["is_default"] = instance.is_default
res["is_cover"] = instance.get_is_cover(res["topic_id_list"])
res["topic_vote_number"] = instance.get_topic_vote_number()
res["activity_join"] = instance.get_activity_join()
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......
......@@ -8,16 +8,15 @@ import logging
import traceback
from libs.tools import tzlc
from trans2es.models.topic import Topic
from trans2es.models.tag import TopicTag,CommunityTagType,CommunityTagTypeRelation
from trans2es.models.tag import TopicTag, CommunityTagType, CommunityTagTypeRelation
import datetime
from django.conf import settings
class TagTransfer(object):
@classmethod
def get_tag_name_data(cls,instance):
def get_tag_name_data(cls, instance):
try:
res = dict()
res["name"] = instance.name
......@@ -28,7 +27,7 @@ class TagTransfer(object):
return dict()
@classmethod
def get_tag_data(cls,instance):
def get_tag_data(cls, instance):
try:
res = dict()
......@@ -36,14 +35,14 @@ class TagTransfer(object):
tag_name_terms_list = list()
for i in range(len(instance.name)):
for j in range(i,len(instance.name)+1):
for j in range(i, len(instance.name) + 1):
name_term = instance.name[i:j].strip()
if name_term:
tag_name_terms_list.append(name_term.lower())
res["suggest"] = {
"input":tag_name_terms_list,
"contexts":{
"input": tag_name_terms_list,
"contexts": {
"is_online": [instance.is_online],
"is_deleted": [instance.is_deleted]
}
......@@ -54,24 +53,27 @@ class TagTransfer(object):
topic_num = 0
res["near_new_topic_num"] = topic_num
if instance.is_online==True and instance.is_deleted==False:
if instance.is_online == True and instance.is_deleted == False:
topic_id_list = list()
sql_result_results = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(
tag_id=instance.id).values_list("topic_id", "is_online"))
for topic_id,is_online in sql_result_results:
for topic_id, is_online in sql_result_results:
if is_online:
topic_id_list.append(topic_id)
time_base_val = datetime.datetime.strftime(datetime.datetime.now()+datetime.timedelta(-7), "%Y-%m-%d")
time_base_val = datetime.datetime.strftime(datetime.datetime.now() + datetime.timedelta(-7), "%Y-%m-%d")
for topic_begin_index in range(0,len(topic_id_list),100):
cur_topic_num = Topic.objects.using(settings.SLAVE_DB_NAME).filter(id__in=topic_id_list[topic_begin_index:topic_begin_index+100],create_time__gte=time_base_val).count()
for topic_begin_index in range(0, len(topic_id_list), 100):
cur_topic_num = Topic.objects.using(settings.SLAVE_DB_NAME).filter(
id__in=topic_id_list[topic_begin_index:topic_begin_index + 100],
create_time__gte=time_base_val).count()
topic_num += cur_topic_num
res["near_new_topic_num"] = topic_num
tag_type_sql_list = CommunityTagTypeRelation.objects.using(settings.SLAVE_DB_NAME).filter(tag_id=instance.id).values_list("tag_type_id",flat=True)
tag_type_sql_list = CommunityTagTypeRelation.objects.using(settings.SLAVE_DB_NAME).filter(
tag_id=instance.id).values_list("tag_type_id", flat=True)
tag_type_list = list()
for tag_type_id in tag_type_sql_list:
tag_type_list.append(tag_type_id)
......@@ -83,4 +85,4 @@ class TagTransfer(object):
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict()
\ No newline at end of file
return dict()
......@@ -8,7 +8,6 @@ from libs.tools import tzlc
import time
import re
import datetime
from trans2es.models.user import User
from trans2es.models.topic import ExcellentTopic, TopicHomeRecommend
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment