Skip to content

P
physical
Commit 01876b05 authored by Kai's avatar Kai
Browse files
Options

Merge branch 'master' into hk

parents a9791469 537bad5c
Show whitespace changes
Inline Side-by-side
Showing with 176 additions and 100 deletions
.idea/misc.xml
View file @ 01876b05
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (physical1)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser"> <component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" /> <option name="shown" value="true" />
</component> </component>
......
.idea/physical.iml
View file @ 01876b05
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4"> <module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager"> <component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" /> <content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" /> <orderEntry type="jdk" jdkName="Python 3.6 (physical1)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
</component> </component>
<component name="TestRunnerService"> <component name="TestRunnerService">
......
injection/data_sync/tasks.py
View file @ 01876b05
...@@ -11,12 +11,16 @@ import traceback ...@@ -11,12 +11,16 @@ import traceback
from libs.cache import redis_client from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar,UserSimilarScore
import json import json
from linucb.utils.register_user_tag import RegisterUserTag
@shared_task @shared_task
def write_to_es(es_type, pk_list, use_batch_query_set=False): def write_to_es(es_type, pk_list, use_batch_query_set=False):
try: try:
pk_list = list(frozenset(pk_list)) pk_list = list(frozenset(pk_list))
if es_type == "register_user_tag":
RegisterUserTag.get_register_user_tag(pk_list)
else:
type_info_map = get_type_info_map() type_info_map = get_type_info_map()
type_info = type_info_map[es_type] type_info = type_info_map[es_type]
......
libs/es.py
View file @ 01876b05
...@@ -155,7 +155,7 @@ class ESPerform(object): ...@@ -155,7 +155,7 @@ class ESPerform(object):
bulk_actions = [] bulk_actions = []
if sub_index_name=="topic": if sub_index_name=="topic" or sub_index_name=="topic-star-routing":
for data in data_list: for data in data_list:
if data: if data:
bulk_actions.append({ bulk_actions.append({
...@@ -264,29 +264,43 @@ class ESPerform(object): ...@@ -264,29 +264,43 @@ class ESPerform(object):
return True return True
@classmethod @classmethod
def get_tag_topic_list(cls,tag_id): def get_tag_topic_list(cls,tag_id,have_read_topic_id_list):
try: try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
q = { q = {
"query":{ "query":{
"bool":{ "bool":{
"must":[ "must":[
{"range": {"content_level": {"gte": 4, "lte": 5}}}, {"range": {"content_level": {"gte": 3, "lte": 5}}},
{"term":{"is_online": True}}, {"term":{"is_online": True}},
{"term":{"is_deleted": False}}, {"term":{"is_deleted": False}},
{"term":{"tag_list":tag_id}} {"terms":{"tag_list":tag_id}}
] ]
} }
}, },
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list,
"_source":{ "_source":{
"include":["id"] "include":["id"]
}, },
"sort":[ "sort":[
{"_score": {"order": "desc"}},
{"create_time_val":{"order":"desc"}}, {"create_time_val":{"order":"desc"}},
{"language_type":{"order":"asc"}}, {"language_type":{"order":"asc"}},
] ]
} }
if len(have_read_topic_id_list)>0:
q["query"]["bool"]["must_not"] = have_read_topic_id_list
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q, result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=5000,routing="4,5") offset=0, size=100,routing="3,4,5")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]] topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
return topic_id_list return topic_id_list
......
linucb/utils/register_user_tag.py
View file @ 01876b05
...@@ -28,54 +28,62 @@ class RegisterUserTag(object): ...@@ -28,54 +28,62 @@ class RegisterUserTag(object):
linucb_device_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:device_id:" linucb_device_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:device_id:"
linucb_user_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:user_id:" linucb_user_id_register_tag_topic_id_prefix = "physical:linucb:register_tag_topic_recommend:user_id:"
linucb_register_user_tag_key = "physical:linucb:register_user_tag_info"
@classmethod @classmethod
def get_register_user_tag(cls,pk_list): def get_register_user_tag(cls,pk_list):
try: try:
user_id_set = set() # user_id_set = set()
user_id_dict = dict()
query_results = AccountUserTag.objects.filter(pk__in=pk_list) query_results = AccountUserTag.objects.filter(pk__in=pk_list)
for item in query_results: for item in query_results:
tag_id = item.tag_id tag_id = item.tag_id
user_id = item.user user_id = item.user
user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id", flat=True)
user_id_dict[user_id] = user_tag_list
if user_id not in user_id_set: for user_id in user_id_dict:
user_id_set.add(user_id) redis_client.hset(cls.linucb_register_user_tag_key, user_id, json.dumps(list(user_id_dict[user_id])))
user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True)
have_read_topic_id_list = Tools.get_have_read_topic_id_list(-1, user_id,
TopicPageType.HOME_RECOMMEND)
recommend_topic_id_list = list()
cycle_num = int(10000/len(user_tag_list))
for index in range(0,cycle_num):
for tag_id in user_tag_list:
redis_tag_id_key = cls.tag_topic_id_redis_prefix + str(tag_id)
redis_tag_id_data = redis_client.get(redis_tag_id_key)
tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
if not redis_tag_id_data:
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
redis_client.expire(redis_tag_id_key,1*24*60*60)
if len(tag_topic_id_list)>index:
for topic_id in tag_topic_id_list[index:]:
if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list:
recommend_topic_id_list.append(topic_id)
break
redis_register_tag_topic_data = {
"data": json.dumps(recommend_topic_id_list),
"cursor": 0
}
redis_client.hmset(cls.linucb_user_id_register_tag_topic_id_prefix,redis_register_tag_topic_data)
redis_client.expire(cls.linucb_user_id_register_tag_topic_id_prefix,30*24*60*60)
topic_recommend_redis_key = cls.linucb_user_id_recommend_topic_id_prefix + str(user_id)
redis_data_dict = {
"data": json.dumps(recommend_topic_id_list),
"cursor":0
}
redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
redis_client.expire(topic_recommend_redis_key,30*24*60*60)
# if user_id not in user_id_set:
# user_id_set.add(user_id)
#
# user_tag_list = AccountUserTag.objects.filter(user=user_id).values_list("tag_id",flat=True)
#
# have_read_topic_id_list = Tools.get_have_read_topic_id_list(-1, user_id,
# TopicPageType.HOME_RECOMMEND)
# recommend_topic_id_list = list()
# cycle_num = int(10000/len(user_tag_list))
# for index in range(0,cycle_num):
# for tag_id in user_tag_list:
# redis_tag_id_key = cls.tag_topic_id_redis_prefix + str(tag_id)
# redis_tag_id_data = redis_client.get(redis_tag_id_key)
# tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
# if not redis_tag_id_data:
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
# redis_client.expire(redis_tag_id_key,1*24*60*60)
#
# if len(tag_topic_id_list)>index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list.append(topic_id)
# break
#
# redis_register_tag_topic_data = {
# "data": json.dumps(recommend_topic_id_list),
# "cursor": 0
# }
# redis_client.hmset(cls.linucb_user_id_register_tag_topic_id_prefix,redis_register_tag_topic_data)
# redis_client.expire(cls.linucb_user_id_register_tag_topic_id_prefix,30*24*60*60)
#
# topic_recommend_redis_key = cls.linucb_user_id_recommend_topic_id_prefix + str(user_id)
# redis_data_dict = {
# "data": json.dumps(recommend_topic_id_list),
# "cursor":0
# }
# redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
# redis_client.expire(topic_recommend_redis_key,30*24*60*60)
#
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
linucb/views/collect_data.py
View file @ 01876b05
...@@ -7,7 +7,7 @@ from libs.cache import redis_client ...@@ -7,7 +7,7 @@ from libs.cache import redis_client
import logging import logging
from linucb.views.linucb import LinUCB from linucb.views.linucb import LinUCB
import json import json
from trans2es.models.tag import TopicTag from trans2es.models.tag import TopicTag,Tag
import traceback import traceback
from django.conf import settings from django.conf import settings
from libs.es import ESPerform from libs.es import ESPerform
...@@ -50,21 +50,21 @@ class CollectData(object): ...@@ -50,21 +50,21 @@ class CollectData(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return dict() return dict()
def update_recommend_tag_list(self, device_id,user_feature=None): def update_recommend_tag_list(self, device_id,user_feature=None,user_id=None,click_topic_tag_list=None):
try: try:
recommend_tag_set = set() recommend_tag_set = set()
recommend_tag_list = list() recommend_tag_list = list()
recommend_tag_dict = dict() recommend_tag_dict = dict()
redis_linucb_tag_data_dict = self._get_user_linucb_info(device_id) redis_linucb_tag_data_dict = self._get_user_linucb_info(device_id)
if len(redis_linucb_tag_data_dict) == 0: if len(redis_linucb_tag_data_dict) == 0:
recommend_tag_list = LinUCB.get_default_tag_list() recommend_tag_list = LinUCB.get_default_tag_list(user_id)
LinUCB.init_device_id_linucb_info(redis_client, self.linucb_matrix_redis_prefix,device_id,recommend_tag_list) LinUCB.init_device_id_linucb_info(redis_client, self.linucb_matrix_redis_prefix,device_id,recommend_tag_list)
else: else:
user_feature = user_feature if user_feature else self.user_feature user_feature = user_feature if user_feature else self.user_feature
(recommend_tag_dict,recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,redis_linucb_tag_data_dict,user_feature,list(redis_linucb_tag_data_dict.keys())) (recommend_tag_dict,recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,redis_linucb_tag_data_dict,user_feature,list(redis_linucb_tag_data_dict.keys()))
recommend_tag_list = list(recommend_tag_dict.keys())
if len(recommend_tag_dict) > 0: if len(recommend_tag_list) > 0:
recommend_tag_list = list(recommend_tag_set)
tag_recommend_redis_key = self.linucb_recommend_redis_prefix + str(device_id) tag_recommend_redis_key = self.linucb_recommend_redis_prefix + str(device_id)
redis_client.set(tag_recommend_redis_key, json.dumps(recommend_tag_list)) redis_client.set(tag_recommend_redis_key, json.dumps(recommend_tag_list))
# Todo:设置过期时间，调研set是否支持 # Todo:设置过期时间，调研set是否支持
...@@ -74,25 +74,43 @@ class CollectData(object): ...@@ -74,25 +74,43 @@ class CollectData(object):
have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,-1,TopicPageType.HOME_RECOMMEND) have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,-1,TopicPageType.HOME_RECOMMEND)
recommend_topic_id_list = list() recommend_topic_id_list = list()
for index in range(0,100): if click_topic_tag_list:
recommend_topic_id_list_random = list() recommend_topic_id_list_click = ESPerform.get_tag_topic_list(click_topic_tag_list,
for tag_id in recommend_tag_list[0:100]: have_read_topic_id_list)
redis_tag_id_key = self.tag_topic_id_redis_prefix + str(tag_id) if len(recommend_topic_id_list) > 0:
redis_tag_id_data = redis_client.get(redis_tag_id_key) num = min(len(recommend_topic_id_list), 2)
tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else [] recommend_topic_id_list.extend(recommend_topic_id_list_click[0:num])
if not redis_tag_id_data:
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id) tag_id_list = recommend_tag_list[0:100]
redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list)) tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
redis_client.expire(redis_tag_id_key,1*24*60*60) recommend_topic_id_list.extend(tag_topic_id_list)
if len(tag_topic_id_list)>index: # for index in range(0,100):
for topic_id in tag_topic_id_list[index:]: # recommend_topic_id_list_es = list()
if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list: # for tag_id in recommend_tag_list[0:100]:
recommend_topic_id_list_random.append(topic_id) # redis_tag_id_key = self.tag_topic_id_redis_prefix + str(tag_id)
break # redis_tag_id_data = redis_client.get(redis_tag_id_key)
random.shuffle(recommend_topic_id_list_random) # tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
recommend_topic_id_list.extend(recommend_topic_id_list_random) # if not redis_tag_id_data:
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
# redis_client.expire(redis_tag_id_key,1*24*60*60)
#
# if len(tag_topic_id_list)>index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_random and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_random.append(topic_id)
# break
# tag_id = recommend_tag_list[0:100]
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# if len(tag_topic_id_list) > index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_es and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_es.append(topic_id)
# if len(recommend_topic_id_list_es) == 100:
# break
# # random.shuffle(recommend_topic_id_list_random)
# recommend_topic_id_list.extend(recommend_topic_id_list_es)
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id) topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
redis_data_dict = { redis_data_dict = {
"data": json.dumps(recommend_topic_id_list), "data": json.dumps(recommend_topic_id_list),
...@@ -131,14 +149,22 @@ class CollectData(object): ...@@ -131,14 +149,22 @@ class CollectData(object):
if "type" in raw_val_dict and "on_click_feed_topic_card" == raw_val_dict["type"]: if "type" in raw_val_dict and "on_click_feed_topic_card" == raw_val_dict["type"]:
topic_id = raw_val_dict["params"]["business_id"] or raw_val_dict["params"]["topic_id"] topic_id = raw_val_dict["params"]["business_id"] or raw_val_dict["params"]["topic_id"]
device_id = raw_val_dict["device"]["device_id"] device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id))) logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id)))
tag_list = list() tag_list = list()
click_topic_tag_list = list()
click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online") click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online")
for tag_id,is_online in click_sql_query_results: for tag_id,is_online in click_sql_query_results:
if is_online: if is_online:
tag_list.append(tag_id) tag_list.append(tag_id)
tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id=tag_id).values_list("id", "collection", "is_ai")
for id, collection, is_ai in tag_sql_query_results:
if collection == 1 or is_ai == 1:
click_topic_tag_list.append(id)
is_click = 1 is_click = 1
is_vote = 0 is_vote = 0
...@@ -151,7 +177,7 @@ class CollectData(object): ...@@ -151,7 +177,7 @@ class CollectData(object):
self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature) self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature)
# 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后 # 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后
self.update_recommend_tag_list(device_id, user_feature) self.update_recommend_tag_list(device_id, user_feature, user_id,click_topic_tag_list=click_topic_tag_list)
elif "type" in raw_val_dict and "page_precise_exposure" == raw_val_dict["type"]: elif "type" in raw_val_dict and "page_precise_exposure" == raw_val_dict["type"]:
if isinstance(raw_val_dict["params"]["exposure_cards"],str): if isinstance(raw_val_dict["params"]["exposure_cards"],str):
exposure_cards_list = json.loads(raw_val_dict["params"]["exposure_cards"]) exposure_cards_list = json.loads(raw_val_dict["params"]["exposure_cards"])
...@@ -160,6 +186,7 @@ class CollectData(object): ...@@ -160,6 +186,7 @@ class CollectData(object):
else: else:
exposure_cards_list = list() exposure_cards_list = list()
device_id = raw_val_dict["device"]["device_id"] device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
exposure_topic_id_list = list() exposure_topic_id_list = list()
for item in exposure_cards_list: for item in exposure_cards_list:
...@@ -176,6 +203,10 @@ class CollectData(object): ...@@ -176,6 +203,10 @@ class CollectData(object):
exposure_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id__in=exposure_topic_id_list).values_list("topic_id","tag_id","is_online") exposure_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id__in=exposure_topic_id_list).values_list("topic_id","tag_id","is_online")
for topic_id,tag_id,is_online in exposure_sql_query_results: for topic_id,tag_id,is_online in exposure_sql_query_results:
if is_online: if is_online:
# tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
# id=tag_id).values_list("id", "collection", "is_ai")
# for id, collection, is_ai in tag_sql_query_results:
# if collection == 1 or is_ai == 1:
tag_list.append(tag_id) tag_list.append(tag_id)
if topic_id not in topic_tag_id_dict: if topic_id not in topic_tag_id_dict:
...@@ -193,7 +224,7 @@ class CollectData(object): ...@@ -193,7 +224,7 @@ class CollectData(object):
self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature) self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature)
# 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后 # 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后
self.update_recommend_tag_list(device_id, user_feature) self.update_recommend_tag_list(device_id, user_feature, user_id)
else: else:
logging.warning("unknown type msg:%s" % raw_val_dict.get("type", "missing type")) logging.warning("unknown type msg:%s" % raw_val_dict.get("type", "missing type"))
except: except:
......
linucb/views/linucb.py
View file @ 01876b05
...@@ -20,10 +20,15 @@ class LinUCB: ...@@ -20,10 +20,15 @@ class LinUCB:
default_tag_list = list() default_tag_list = list()
@classmethod @classmethod
def get_default_tag_list(cls): def get_default_tag_list(cls,user_id):
try: try:
if len(cls.default_tag_list) == 0: if len(cls.default_tag_list) == 0:
if user_id:
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
cls.default_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(cls.default_tag_list) == 0:
cls.default_tag_list = Tag.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True,collection=1).values_list("id",flat=True)[0:100] cls.default_tag_list = Tag.objects.using(settings.SLAVE_DB_NAME).filter(is_online=True,collection=1).values_list("id",flat=True)[0:100]
return cls.default_tag_list return cls.default_tag_list
......
search/utils/topic.py
View file @ 01876b05
...@@ -122,7 +122,7 @@ class TopicUtils(object): ...@@ -122,7 +122,7 @@ class TopicUtils(object):
@classmethod @classmethod
def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,single_size,query=None,query_type=TopicPageType.FIND_PAGE, def get_recommend_topic_ids(cls,user_id,tag_id,offset,size,single_size,query=None,query_type=TopicPageType.FIND_PAGE,
filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[], filter_topic_id_list=[],test_score=False,must_topic_id_list=[],recommend_tag_list=[],
user_similar_score_list=[],index_type="topic",routing=None): user_similar_score_list=[],index_type="topic",routing=None,attention_tag_list=[]):
""" """
:需增加打散逻辑 :需增加打散逻辑
:remark:获取首页推荐帖子列表 :remark:获取首页推荐帖子列表
...@@ -216,12 +216,12 @@ class TopicUtils(object): ...@@ -216,12 +216,12 @@ class TopicUtils(object):
# ) # )
# query_tag_term_list = cls.___get_should_term_list(user_tag_list) # query_tag_term_list = cls.___get_should_term_list(user_tag_list)
if len(user_tag_list) > 0: if len(attention_tag_list) > 0:
functions_list.append( functions_list.append(
{ {
"filter": {"bool": { "filter": {"bool": {
"should": {"terms": {"tag_list": user_tag_list}}}}, "should": {"terms": {"tag_list": attention_tag_list}}}},
"weight": 1 "weight": 60
} }
) )
# if len(recommend_tag_list)>0: # if len(recommend_tag_list)>0:
......
search/views/topic.py
View file @ 01876b05
...@@ -89,12 +89,16 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -89,12 +89,16 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
user_similar_score_redis_list = json.loads( user_similar_score_redis_list = json.loads(
redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else [] redis_user_similar_score_redis_val) if redis_user_similar_score_redis_val else []
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
size = size-len(recommend_topic_list) size = size-len(recommend_topic_list)
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size, rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
single_size=size,query=query, query_type=query_type, single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list, filter_topic_id_list=have_read_topic_id_list,
recommend_tag_list=recommend_topic_list, recommend_tag_list=recommend_topic_list,
user_similar_score_list=user_similar_score_redis_list,index_type="topic",routing="4,5") user_similar_score_list=user_similar_score_redis_list,index_type="topic",routing="4,5",attention_tag_list=attention_tag_list)
have_read_group_id_set = set() have_read_group_id_set = set()
have_read_user_id_set = set() have_read_user_id_set = set()
unread_topic_id_dict = dict() unread_topic_id_dict = dict()
......
trans2es/mapping/topic-high-star.json
View file @ 01876b05
{ {
"dynamic":"strict", "dynamic":"strict",
"_routing": {"required": true},
"properties": { "properties": {
"id":{"type":"long"}, "id":{"type":"long"},
"is_online":{"type":"boolean"},//上线 "is_online":{"type":"boolean"},//上线
......
trans2es/mapping/topic-star-routing.json
View file @ 01876b05
{ {
"dynamic":"strict", "dynamic":"strict",
"_routing": {"required": true},
"properties": { "properties": {
"id":{"type":"long"}, "id":{"type":"long"},
"is_online":{"type":"boolean"},//上线 "is_online":{"type":"boolean"},//上线
......
trans2es/models/pictorial.py
View file @ 01876b05
...@@ -75,12 +75,19 @@ class Pictorial(models.Model): ...@@ -75,12 +75,19 @@ class Pictorial(models.Model):
def get_effective(self,topic_id_list): def get_effective(self,topic_id_list):
try: try:
topic_id_list = Topic.objects.filter(id__in=topic_id_list,content_level__in=[3,4,5,0],is_online=True).count() effective_num = 0
if topic_id_list >= 5: ret = False
return True
else: for topic_id in topic_id_list:
return False topic_id_object = Topic.objects.filter(id=int(topic_id)).first()
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0,3,4,5]:
effective_num += 1
if effective_num >= 5:
ret = True
break
return ret
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return False
......
trans2es/models/topic.py
View file @ 01876b05
...@@ -107,11 +107,14 @@ class Topic(models.Model): ...@@ -107,11 +107,14 @@ class Topic(models.Model):
@property @property
def is_complaint(self): def is_complaint(self):
"""是否被举报""" """是否被举报"""
try:
if TopicComplaint.objects.filter(topic_id=self.id, is_online=True).exists(): if TopicComplaint.objects.filter(topic_id=self.id, is_online=True).exists():
return True return True
return False return False
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False
def topic_has_image(self): def topic_has_image(self):
try: try:
...@@ -197,14 +200,14 @@ class Topic(models.Model): ...@@ -197,14 +200,14 @@ class Topic(models.Model):
elif self.content_level == '3': elif self.content_level == '3':
offline_score += 2.0 offline_score += 2.0
exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count() # exposure_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=1).count()
click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count() # click_count = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=2).count()
uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count() # uv_num = ActionSumAboutTopic.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=self.id, data_type=3).count()
#
if exposure_count > 0: # if exposure_count > 0:
offline_score += click_count / exposure_count # offline_score += click_count / exposure_count
if uv_num > 0: # if uv_num > 0:
offline_score += (self.vote_num / uv_num + self.reply_num / uv_num) # offline_score += (self.vote_num / uv_num + self.reply_num / uv_num)
""" """
1:马甲账号是否对总分降权？ 1:马甲账号是否对总分降权？
...@@ -247,7 +250,6 @@ class PictorialTopic(models.Model): ...@@ -247,7 +250,6 @@ class PictorialTopic(models.Model):
pictorial_id = models.BigIntegerField(verbose_name=u'画报ID') pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
topic_id = models.BigIntegerField(verbose_name=u'帖子ID') topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True) is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
is_online = models.BooleanField(verbose_name=u'是否上线')
is_deleted = models.BooleanField(verbose_name=u'是否删除') is_deleted = models.BooleanField(verbose_name=u'是否删除')
......
trans2es/type_info.py
View file @ 01876b05
...@@ -143,8 +143,6 @@ class TypeInfo(object): ...@@ -143,8 +143,6 @@ class TypeInfo(object):
)) ))
else: else:
if data: if data:
data_list.append(data)
if self.type == "topic": if self.type == "topic":
ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"]) ori_topic_star = redis_client.hget(self.physical_topic_star, data["id"])
if not ori_topic_star: if not ori_topic_star:
...@@ -155,10 +153,11 @@ class TypeInfo(object): ...@@ -155,10 +153,11 @@ class TypeInfo(object):
old_data = copy.deepcopy(data) old_data = copy.deepcopy(data)
old_data["is_online"] = False old_data["is_online"] = False
old_data["is_deleted"] = True old_data["is_deleted"] = True
old_data["content_level"] = ori_topic_star old_data["content_level"] = int_ori_topic_star
data_list.append(old_data) data_list.append(old_data)
redis_client.hset(self.physical_topic_star, data["id"], data["content_level"]) redis_client.hset(self.physical_topic_star, data["id"], data["content_level"])
data_list.append(data)
# if self.type=="topic" and instance.content_level and int(instance.content_level)>=4: # if self.type=="topic" and instance.content_level and int(instance.content_level)>=4:
# topic_data_high_star_list.append(data) # topic_data_high_star_list.append(data)
......
trans2es/utils/topic_transfer.py
View file @ 01876b05
...@@ -115,8 +115,8 @@ class TopicTransfer(object): ...@@ -115,8 +115,8 @@ class TopicTransfer(object):
if is_excellect: if is_excellect:
res["is_excellent"] = 1 res["is_excellent"] = 1
else: else:
excelllect_object = ExcellentTopic.objects.filter(topic_id=instance.id,is_online=True,is_deleted=False).first() excelllect_object = ExcellentTopic.objects.filter(topic_id=instance.id).first()
if excelllect_object: if excelllect_object and excelllect_object.is_online and not excelllect_object.is_deleted:
res["is_excellent"] = 1 res["is_excellent"] = 1
else: else:
res["is_excellent"] = 0 res["is_excellent"] = 0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment