Skip to content

P
physical
Commit 10b9bba0 authored by 高雅喆's avatar 高雅喆
Browse files
Options

Merge branch 'master' into gyz_fix

parents c96419d6 9baa9923
master deploy/like-prod deploy/like-stage deploy/like-test like-pre/r01 test
1 merge request!532Wsy3
Show whitespace changes
Inline Side-by-side
Showing with 225 additions and 67 deletions
libs/es.py
View file @ 10b9bba0
...@@ -188,10 +188,13 @@ class ESPerform(object): ...@@ -188,10 +188,13 @@ class ESPerform(object):
@classmethod @classmethod
def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10, def get_search_results(cls, es_cli, sub_index_name, query_body, offset=0, size=10,
auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False, auto_create_index=False, doc_type="_doc", aggregations_query=False, is_suggest_request=False,
batch_search=False, routing=None): batch_search=False, routing=None,if_official_index_name=False):
try: try:
assert (es_cli is not None) assert (es_cli is not None)
if if_official_index_name:
official_index_name = sub_index_name
else:
official_index_name = cls.get_official_index_name(sub_index_name, "read") official_index_name = cls.get_official_index_name(sub_index_name, "read")
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
if not index_exists: if not index_exists:
...@@ -401,6 +404,7 @@ class ESPerform(object): ...@@ -401,6 +404,7 @@ class ESPerform(object):
{"term": {"content_level": 6}}, {"term": {"content_level": 6}},
{"term": {"is_online": True}}, {"term": {"is_online": True}},
{"term": {"is_deleted": False}}, {"term": {"is_deleted": False}},
{"term": {"is_new_topic": False}},
{"terms": {"tag_list": tag_id}} {"terms": {"tag_list": tag_id}}
] ]
} }
...@@ -411,9 +415,18 @@ class ESPerform(object): ...@@ -411,9 +415,18 @@ class ESPerform(object):
} }
}, },
"_source": { "_source": {
"include": ["id", "user_id", "latest_reply_time"] "include": ["id", "user_id", "latest_reply_time", "topic_ctr_30", "like_rate_30"]
}, },
"sort": [ "sort": [
{
"_script": {
"order": "desc",
"script": {
"inline": "5*doc['topic_ctr_30'].value+doc['like_rate_30'].value"
},
"type": "number"
}
},
{"latest_reply_time": {"order": "desc"}}, {"latest_reply_time": {"order": "desc"}},
{"create_time_val": {"order": "desc"}}, {"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}}, {"language_type": {"order": "asc"}},
...@@ -446,10 +459,69 @@ class ESPerform(object): ...@@ -446,10 +459,69 @@ class ESPerform(object):
logging.info("topic_id_list:%s" % str(topic_id_dict)) logging.info("topic_id_list:%s" % str(topic_id_dict))
logging.info("linucb_tag_id_list_2_same_tagset_ids:" + str(tag_id)) logging.info("get_tag_topic_list_dict:gyz" + str(q) + str(result_dict))
logging.info("linucb_tag_id_list_2_same_tagset_ids_2_topics_detail:" + str(topic_id_dict_latest_reply_time)) logging.info("linucb_tag_id_list_2_same_tagset_ids_2_topics_detail:" + str(topic_id_dict_latest_reply_time))
return topic_id_list, topic_id_dict return topic_id_list, topic_id_dict
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list(), dict()
@classmethod
def get_tag_new_topic_list(cls, tag_id, have_read_topic_id_list, size=10):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{"term": {"content_level": 6}},
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"term": {"is_new_topic": True}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
}
},
"_source": {
"include": ["id", "user_id"]
},
"sort": [
{"latest_reply_time": {"order": "desc"}},
{"create_time_val": {"order": "desc"}},
{"language_type": {"order": "asc"}},
],
"collapse": {
"field": "user_id"
}
}
if len(have_read_topic_id_list) > 0:
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms": {
"id": have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic-high-star",
query_body=q,
offset=0, size=size, routing="6")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
return topic_id_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return list() return list()
......
linucb/views/collect_data.py
View file @ 10b9bba0
...@@ -263,6 +263,7 @@ class CollectData(object): ...@@ -263,6 +263,7 @@ class CollectData(object):
if "type" in raw_val_dict and \ if "type" in raw_val_dict and \
(raw_val_dict["type"] in ("on_click_feed_topic_card","on_click_button")): (raw_val_dict["type"] in ("on_click_feed_topic_card","on_click_button")):
click_topic_tag_list = list() click_topic_tag_list = list()
device_id = ""
if "on_click_feed_topic_card" == raw_val_dict["type"]: if "on_click_feed_topic_card" == raw_val_dict["type"]:
topic_id = raw_val_dict["params"]["topic_id"] topic_id = raw_val_dict["params"]["topic_id"]
device_id = raw_val_dict["device"]["device_id"] device_id = raw_val_dict["device"]["device_id"]
......
search/views/group.py
View file @ 10b9bba0
...@@ -303,7 +303,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -303,7 +303,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
@bind("physical/search/pictorial_topic_sort") @bind("physical/search/pictorial_topic_sort")
def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL_TOPIC_SORT.HOT): def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL_TOPIC_SORT.HOT, user_id=-1):
""" """
:remark 画报排序 人气 部分 :remark 画报排序 人气 部分
人气按照票数从大到小排序，相同票数按照图片票数更新时间由旧到新排序 人气按照票数从大到小排序，相同票数按照图片票数更新时间由旧到新排序
...@@ -420,7 +420,7 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL ...@@ -420,7 +420,7 @@ def pictorial_topic_sort(pictorial_id=-1, offset=0, size=10, sort_type=PICTORIAL
pict_pictorial_ids_list = [] pict_pictorial_ids_list = []
# 获取es链接对象 # 获取es链接对象
es_cli_obj = ESPerform.get_cli() es_cli_obj = ESPerform.get_cli()
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size) result_dict = ESPerform.get_search_results(es_cli=es_cli_obj, sub_index_name="mv-alpha-topic-prod-190905001", query_body=q, offset=offset, size=size,if_official_index_name=True)
# logging.info("get pictorial_topic_sort res:%s" % result_dict) # logging.info("get pictorial_topic_sort res:%s" % result_dict)
......
search/views/topic.py
View file @ 10b9bba0
...@@ -8,7 +8,6 @@ import traceback ...@@ -8,7 +8,6 @@ import traceback
import json import json
from search.utils.topic import TopicUtils from search.utils.topic import TopicUtils
from trans2es.models.topic import TopicHomeRecommend from trans2es.models.topic import TopicHomeRecommend
from libs.es import ESPerform
from libs.cache import redis_client from libs.cache import redis_client
from search.utils.common import * from search.utils.common import *
from libs.es import ESPerform from libs.es import ESPerform
...@@ -17,7 +16,6 @@ from libs.tools import get_have_read_lin_pictorial_id_list ...@@ -17,7 +16,6 @@ from libs.tools import get_have_read_lin_pictorial_id_list
import datetime import datetime
from trans2es.models.tag import Tag from trans2es.models.tag import Tag
from search.views.tag import get_same_tagset_ids from search.views.tag import get_same_tagset_ids
from linucb.views.collect_data import CollectData
from linucb.views.linucb import LinUCB from linucb.views.linucb import LinUCB
from alpha_types.physical.enum import STRATEGY_TYPE from alpha_types.physical.enum import STRATEGY_TYPE
...@@ -55,9 +53,9 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -55,9 +53,9 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
return [] return []
def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query=None, def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, underexposure_lin_topic_count, size, query=None,
query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False, query_type=TopicPageType.HOME_RECOMMEND, promote_topic_list=[], disable_collpase=False,
usefulrecall=-1, useful_tag_list=[], has_score= False,gray_list=[]): usefulrecall=-1, useful_tag_list=[], has_score=False, gray_list=[]):
try: try:
topic_star_routing = "6" topic_star_routing = "6"
index_type = "topic-high-star" index_type = "topic-high-star"
...@@ -110,6 +108,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -110,6 +108,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
recommend_topic_user_list = list() recommend_topic_user_list = list()
attention_tag_list = list() attention_tag_list = list()
recommend_topic_list = list() recommend_topic_list = list()
underexposure_lin_topic_ids = list()
if query is None: if query is None:
if user_id != -1: if user_id != -1:
# 有用标签召回 # 有用标签召回
...@@ -125,11 +124,29 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -125,11 +124,29 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
have_read_topic_id_list.extend(useful_topic_id_list) have_read_topic_id_list.extend(useful_topic_id_list)
# linucb 推荐帖子 # linucb 推荐帖子
linucb_recommend_tags = list()
if STRATEGY_TYPE.CTR_GRAY in gray_list: if STRATEGY_TYPE.CTR_GRAY in gray_list:
topic_recommend_redis_key = "ctr_physical:linucb:topic_recommend:device_id:" + str(device_id) topic_recommend_redis_key = "ctr_physical:linucb:topic_recommend:device_id:" + str(device_id)
linucb_recommend_tags_key = "ctr_physical:linucb:tag_recommend:device_id:" + str(device_id)
linucb_recommend_tags = redis_client.get(linucb_recommend_tags_key)
if linucb_recommend_tags:
linucb_recommend_tags = json.loads(linucb_recommend_tags)
else: else:
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id) topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
linucb_recommend_tags_key = "physical:linucb:tag_recommend:device_id:" + str(device_id)
linucb_recommend_tags = redis_client.get(linucb_recommend_tags_key)
if linucb_recommend_tags:
linucb_recommend_tags = json.loads(linucb_recommend_tags)
# linucb 推荐新帖子
if linucb_recommend_tags:
linucb_recommend_tags_set_tags = get_same_tagset_ids(linucb_recommend_tags)
underexposure_lin_topic_ids = ESPerform.get_tag_new_topic_list(linucb_recommend_tags_set_tags, have_read_topic_id_list, underexposure_lin_topic_count)
size = size - len(underexposure_lin_topic_ids)
have_read_topic_id_list.extend(underexposure_lin_topic_ids)
redis_client.publish("new_topic_impression", json.dumps(underexposure_lin_topic_ids))
# linucb 推荐老帖子
recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key) recommend_topic_dict = redis_client.hgetall(topic_recommend_redis_key)
linucb_recommend_topic_id_list = list() linucb_recommend_topic_id_list = list()
recommend_topic_list = list() recommend_topic_list = list()
...@@ -236,32 +253,32 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -236,32 +253,32 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
ret_list = rank_topic_id_list if query is None else ret_data_list ret_list = rank_topic_id_list if query is None else ret_data_list
if usefulrecall != -1: if usefulrecall != -1:
if has_score: if has_score:
return recommend_topic_list, ret_list, useful_topic_id_list, rank_topic_score return underexposure_lin_topic_ids, recommend_topic_list, ret_list, useful_topic_id_list, rank_topic_score
else: else:
return recommend_topic_list, ret_list, useful_topic_id_list return underexposure_lin_topic_ids, recommend_topic_list, ret_list, useful_topic_id_list
else: else:
if has_score: if has_score:
return recommend_topic_list, ret_list, rank_topic_score return underexposure_lin_topic_ids, recommend_topic_list, ret_list, rank_topic_score
else: else:
return recommend_topic_list, ret_list return underexposure_lin_topic_ids, recommend_topic_list, ret_list
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
if has_score: if has_score:
return [], [], [], [] return [], [], [], [], []
else: else:
return [], [], [] return [], [], [], []
else: else:
if has_score: if has_score:
return [], [], [] return [], [], [], []
else: else:
return [], [] return [], [], []
def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list=[]): def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list=[]):
try: try:
if STRATEGY_TYPE.CTR_GRAY in gray_list: if STRATEGY_TYPE.CTR_GRAY in gray_list:
pictorial_recommend_redis_key = "ctr_physical:linucb:pictorial_recommend:device_id:"+str(device_id) pictorial_recommend_redis_key = "ctr_physical:linucb:pictorial_recommend:device_id:" + str(device_id)
else: else:
pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id) pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id)
have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id, user_id, have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id, user_id,
...@@ -298,14 +315,16 @@ def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list ...@@ -298,14 +315,16 @@ def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4, gray_list
return [] return []
def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list = [],rank_topic_id_list = [],linucb_pictorial_id_list = [],rank_topic_score = []): def get_topic_and_pictorial_detail_for_debug(device_id="", linucb_topic_id_list=[], rank_topic_id_list=[],
linucb_pictorial_id_list=[], rank_topic_score=[]):
try: try:
linucb_tag_dict = get_device_linucb_tag(device_id,size=20,has_score=True) linucb_tag_dict = get_device_linucb_tag(device_id, size=20, has_score=True)
linucb_topic_results = dict() linucb_topic_results = dict()
linucb_pictorial_results = dict() linucb_pictorial_results = dict()
rank_topic_results = dict() rank_topic_results = dict()
if len(linucb_topic_id_list) >0: if len(linucb_topic_id_list) > 0:
linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list),linucb_topic_list=linucb_topic_id_list) linucb_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(linucb_topic_id_list),
linucb_topic_list=linucb_topic_id_list)
linucb_topic_results = dict() linucb_topic_results = dict()
if len(linucb_topic_dict) > 0: if len(linucb_topic_dict) > 0:
# num = 1 # num = 1
...@@ -321,11 +340,13 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list ...@@ -321,11 +340,13 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list
{"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"], {"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"],
"name": linucb_tag_dict[edit_tag]["name"]}) "name": linucb_tag_dict[edit_tag]["name"]})
else: else:
edit_tag_name_score_list.append({"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]}) edit_tag_name_score_list.append(
linucb_topic_results.update({str(topic_id):{"id":topic_id,"content_level":content_level,"edit_tags":edit_tag_name_score_list}}) {"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
linucb_topic_results.update({str(topic_id): {"id": topic_id, "content_level": content_level,
"edit_tags": edit_tag_name_score_list}})
logging.info("linucb_topic:{}".format(linucb_topic_results)) logging.info("linucb_topic:{}".format(linucb_topic_results))
# num += 1 # num += 1
if len(rank_topic_id_list) >0: if len(rank_topic_id_list) > 0:
rank_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(rank_topic_id_list), rank_topic_dict = TopicUtils.get_linucb_topic_info_for_debug(len(rank_topic_id_list),
linucb_topic_list=rank_topic_id_list) linucb_topic_list=rank_topic_id_list)
rank_topic_results = dict() rank_topic_results = dict()
...@@ -334,10 +355,10 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list ...@@ -334,10 +355,10 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list
for i in range(len(rank_topic_id_list)): for i in range(len(rank_topic_id_list)):
topic_id = rank_topic_id_list[i] topic_id = rank_topic_id_list[i]
score = rank_topic_score[i] score = rank_topic_score[i]
rank_topic_results.update({str(topic_id): {"id": topic_id, "rank":score}}) rank_topic_results.update({str(topic_id): {"id": topic_id, "rank": score}})
# num += 1 # num += 1
logging.info("rank_topic:{}".format(rank_topic_results)) logging.info("rank_topic:{}".format(rank_topic_results))
if len(linucb_pictorial_id_list) >0: if len(linucb_pictorial_id_list) > 0:
if len(linucb_pictorial_id_list) > 0: if len(linucb_pictorial_id_list) > 0:
linucb_pictorial_dict = TopicUtils.get_linucb_pictorial_info_for_debug(len(linucb_pictorial_id_list), linucb_pictorial_dict = TopicUtils.get_linucb_pictorial_info_for_debug(len(linucb_pictorial_id_list),
linucb_pictorial_list=linucb_pictorial_id_list) linucb_pictorial_list=linucb_pictorial_id_list)
...@@ -355,17 +376,19 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list ...@@ -355,17 +376,19 @@ def get_topic_and_pictorial_detail_for_debug(device_id = "",linucb_topic_id_list
{"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"], {"id": edit_tag, "score": linucb_tag_dict[edit_tag]["score"],
"name": linucb_tag_dict[edit_tag]["name"]}) "name": linucb_tag_dict[edit_tag]["name"]})
else: else:
edit_tag_name_score_list.append({"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]}) edit_tag_name_score_list.append(
{"id": edit_tag, "score": 0, "name": edit_tag_List_dict[edit_tag]})
linucb_pictorial_results.update({str(pictorial_id): {"id": pictorial_id, linucb_pictorial_results.update({str(pictorial_id): {"id": pictorial_id,
"edit_tags": edit_tag_name_score_list}}) "edit_tags": edit_tag_name_score_list}})
logging.info("linucb_pictorial:{}".format(linucb_pictorial_results)) logging.info("linucb_pictorial:{}".format(linucb_pictorial_results))
# num += 1 # num += 1
result = { "linucb_topic":linucb_topic_results,"rank_topic":rank_topic_results,"linucb_pictorial":linucb_pictorial_results} result = {"linucb_topic": linucb_topic_results, "rank_topic": rank_topic_results,
"linucb_pictorial": linucb_pictorial_results}
logging.info("get_topic_and_pictorial_detail_for_debug:result:{}:device_id:{}".format(result, device_id)) logging.info("get_topic_and_pictorial_detail_for_debug:result:{}:device_id:{}".format(result, device_id))
return result return result
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return { "linucb_topic":{},"rank_topic":{},"linucb_pictorial":{}} return {"linucb_topic": {}, "rank_topic": {}, "linucb_pictorial": {}}
@bind("physical/search/query_tag_id_by_topic") @bind("physical/search/query_tag_id_by_topic")
...@@ -378,7 +401,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1): ...@@ -378,7 +401,7 @@ def query_tag_id_by_topic(offset=0, size=10, topic_id_list=[], user_id=-1):
@bind("physical/search/home_recommend") @bind("physical/search/home_recommend")
def home_recommend(device_id="", user_id=-1, offset=0, lin_pictorial_count=4, size=10, query_type=TopicPageType.HOME_RECOMMEND, def home_recommend(device_id="", user_id=-1, offset=0, underexposure_lin_topic_count =4, lin_pictorial_count=4, size=10, query_type=TopicPageType.HOME_RECOMMEND,
promote_topic_list=[], usefulrecall=-1, useful_tag_list=[], is_debug=False,gray_list=[]): promote_topic_list=[], usefulrecall=-1, useful_tag_list=[], is_debug=False,gray_list=[]):
""" """
:remark:首页推荐，目前只推荐日记 :remark:首页推荐，目前只推荐日记
...@@ -420,77 +443,88 @@ def home_recommend(device_id="", user_id=-1, offset=0, lin_pictorial_count=4, si ...@@ -420,77 +443,88 @@ def home_recommend(device_id="", user_id=-1, offset=0, lin_pictorial_count=4, si
else: else:
if usefulrecall != -1 and len(useful_tag_list) > 0: if usefulrecall != -1 and len(useful_tag_list) > 0:
recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count, gray_list=gray_list) recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count,
gray_list=gray_list)
size = size - len(recommend_pictorial_ids) size = size - len(recommend_pictorial_ids)
if is_debug: if is_debug:
has_score = True has_score = True
recommend_topic_ids, rank_topic_ids, useful_topic_ids, rank_topic_score = get_home_recommend_topic_ids( underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids, useful_topic_ids, rank_topic_score = get_home_recommend_topic_ids(
user_id, device_id, user_id, device_id,
tag_id=0, offset=0, tag_id=0, offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size, size=size,
query_type=query_type, query_type=query_type,
promote_topic_list=promote_topic_list, promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall, usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list, has_score=has_score,gray_list=gray_list) useful_tag_list=useful_tag_list, has_score=has_score, gray_list=gray_list)
else: else:
has_score = False has_score = False
recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids, useful_topic_ids = get_home_recommend_topic_ids(user_id,
device_id, device_id,
tag_id=0, tag_id=0,
offset=0, offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size, size=size,
query_type=query_type, query_type=query_type,
promote_topic_list=promote_topic_list, promote_topic_list=promote_topic_list,
usefulrecall=usefulrecall, usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list, useful_tag_list=useful_tag_list,
has_score=has_score,gray_list=gray_list) has_score=has_score,
gray_list=gray_list)
if not is_debug: if not is_debug:
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids, return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids} "useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids}
else: else:
results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids, results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
recommend_pictorial_ids, rank_topic_score) recommend_pictorial_ids, rank_topic_score)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids, return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids, "useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids,
"debug_model_data": results} "debug_model_data": results}
else: else:
recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count, gray_list=gray_list) recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, lin_pictorial_count,
gray_list=gray_list)
size = size - len(recommend_pictorial_ids) size = size - len(recommend_pictorial_ids)
if is_debug: if is_debug:
has_score = True has_score = True
recommend_topic_ids, rank_topic_ids, rank_topic_score = get_home_recommend_topic_ids(user_id, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids, rank_topic_score = get_home_recommend_topic_ids(user_id,
device_id, device_id,
tag_id=0, tag_id=0,
offset=0, offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size, size=size,
query_type=query_type, query_type=query_type,
promote_topic_list=promote_topic_list, promote_topic_list=promote_topic_list,
has_score=has_score,gray_list=gray_list) has_score=has_score,
gray_list=gray_list)
else: else:
has_score = False has_score = False
recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0, underexposure_lin_topic_ids, recommend_topic_ids, rank_topic_ids = get_home_recommend_topic_ids(user_id, device_id, tag_id=0,
offset=0, size=size, offset=0,
underexposure_lin_topic_count=underexposure_lin_topic_count,
size=size,
query_type=query_type, query_type=query_type,
promote_topic_list=promote_topic_list, promote_topic_list=promote_topic_list,
has_score=has_score,gray_list=gray_list) has_score=has_score,
gray_list=gray_list)
if not is_debug: if not is_debug:
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids, return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"linucb_pictorial_ids": recommend_pictorial_ids} "linucb_pictorial_ids": recommend_pictorial_ids}
else: else:
results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids, results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
recommend_pictorial_ids, rank_topic_score) recommend_pictorial_ids, rank_topic_score)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids, return {"underexposure_lin_topic_ids": underexposure_lin_topic_ids, "linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"linucb_pictorial_ids": recommend_pictorial_ids, "debug_model_data": results} "linucb_pictorial_ids": recommend_pictorial_ids, "debug_model_data": results}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": [],"linucb_pictorial_ids": [], "debug_model_data": {}} return {"underexposure_lin_topic_ids": [], "linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": [],"linucb_pictorial_ids": [], "debug_model_data": {}}
else: else:
return {"linucb_topic_ids": [], "rank_topic_ids": [], "linucb_pictorial_ids": [],"debug_model_data": {}} return {"underexposure_lin_topic_ids": [], "linucb_topic_ids": [], "rank_topic_ids": [], "linucb_pictorial_ids": [],"debug_model_data": {}}
@bind("physical/search/discover_page") @bind("physical/search/discover_page")
...@@ -724,7 +758,7 @@ def query_topic_by_tag_aggregation(user_id, tag_id, offset, size): ...@@ -724,7 +758,7 @@ def query_topic_by_tag_aggregation(user_id, tag_id, offset, size):
@bind("physical/search/topic") @bind("physical/search/topic")
def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10): def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10, user_id=-1):
"""帖子搜索。""" """帖子搜索。"""
try: try:
...@@ -766,12 +800,12 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -766,12 +800,12 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
@bind("physical/search/clear_history_for_debug") @bind("physical/search/clear_history_for_debug")
def clear_history(device_id=None,user_id=-1): def clear_history(device_id=None, user_id=-1):
try: try:
msg = list() msg = list()
redis_key_list = list() redis_key_list = list()
if device_id: if device_id:
redis_key_list.extend(('physical:linucb:device_id:%s','physical:home_recommend:device_id:%s:query_type:1', redis_key_list.extend(('physical:linucb:device_id:%s', 'physical:home_recommend:device_id:%s:query_type:1',
'physical:linucb:tag_recommend:device_id:%s', 'physical:linucb:tag_recommend:device_id:%s',
'physical:linucb:topic_recommend:device_id:%s', 'physical:linucb:topic_recommend:device_id:%s',
'physical:linucb:pictorial_recommend:device_id:%s', 'physical:linucb:pictorial_recommend:device_id:%s',
...@@ -782,7 +816,8 @@ def clear_history(device_id=None,user_id=-1): ...@@ -782,7 +816,8 @@ def clear_history(device_id=None,user_id=-1):
redis_client.delete(key) redis_client.delete(key)
msg.append('device_id_clear') msg.append('device_id_clear')
if user_id != -1: if user_id != -1:
redis_user_key_list = ["physical:home_recommend:user_id:%s:query_type:1",'physical:home_pictorial_recommend:user_id:%s:query_type:1'] redis_user_key_list = ["physical:home_recommend:user_id:%s:query_type:1",
'physical:home_pictorial_recommend:user_id:%s:query_type:1']
for redis_key in redis_user_key_list: for redis_key in redis_user_key_list:
key = redis_key % user_id key = redis_key % user_id
if redis_client.type(key) != "b'none'": if redis_client.type(key) != "b'none'":
...@@ -795,7 +830,8 @@ def clear_history(device_id=None,user_id=-1): ...@@ -795,7 +830,8 @@ def clear_history(device_id=None,user_id=-1):
@bind("physical/search/home_recommend_debug") @bind("physical/search/home_recommend_debug")
def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank_topic_ids = [],recommend_pictorial_ids =[]): def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids=[], rank_topic_ids=[],
recommend_pictorial_ids=[]):
""" """
:remark:首页推荐，目前只推荐日记 :remark:首页推荐，目前只推荐日记
:param session_id: :param session_id:
...@@ -806,7 +842,8 @@ def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank ...@@ -806,7 +842,8 @@ def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank
""" """
try: try:
if device_id: if device_id:
results = get_topic_and_pictorial_detail_for_debug(device_id,recommend_topic_ids,rank_topic_ids,recommend_pictorial_ids) results = get_topic_and_pictorial_detail_for_debug(device_id, recommend_topic_ids, rank_topic_ids,
recommend_pictorial_ids)
return results return results
else: else:
return {} return {}
...@@ -815,13 +852,13 @@ def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank ...@@ -815,13 +852,13 @@ def home_recommend_debug(device_id="", user_id=-1, recommend_topic_ids = [],rank
return {} return {}
def get_device_linucb_tag(device_id = None,size = 20,has_score = False): def get_device_linucb_tag(device_id=None, size=20, has_score=False):
# { # {
# "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字}, # "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字},
# {"id": 6, "name": 名字}, ] # {"id": 6, "name": 名字}, ]
# } # }
try: try:
user_feature = [1,1] user_feature = [1, 1]
if device_id: if device_id:
linucb_matrix_redis_prefix = "physical:linucb:device_id:" linucb_matrix_redis_prefix = "physical:linucb:device_id:"
redis_key = linucb_matrix_redis_prefix + str(device_id) redis_key = linucb_matrix_redis_prefix + str(device_id)
...@@ -829,7 +866,10 @@ def get_device_linucb_tag(device_id = None,size = 20,has_score = False): ...@@ -829,7 +866,10 @@ def get_device_linucb_tag(device_id = None,size = 20,has_score = False):
if len(redis_linucb_tag_data_dict) == 0: if len(redis_linucb_tag_data_dict) == 0:
return {"linucb": []} return {"linucb": []}
else: else:
(recommend_tag_dict,recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,redis_linucb_tag_data_dict,user_feature,list(redis_linucb_tag_data_dict.keys())) (recommend_tag_dict, recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,
redis_linucb_tag_data_dict,
user_feature, list(
redis_linucb_tag_data_dict.keys()))
recommend_tag_list = list(recommend_tag_dict.keys()) recommend_tag_list = list(recommend_tag_dict.keys())
linucb_tag_dict_list = list() linucb_tag_dict_list = list()
linucb_tag_dict = dict() linucb_tag_dict = dict()
...@@ -846,7 +886,7 @@ def get_device_linucb_tag(device_id = None,size = 20,has_score = False): ...@@ -846,7 +886,7 @@ def get_device_linucb_tag(device_id = None,size = 20,has_score = False):
return linucb_tag_dict_list return linucb_tag_dict_list
else: else:
return {"linucb":[]} return {"linucb": []}
except: except:
return {"linucb": []} return {"linucb": []}
...@@ -864,13 +904,13 @@ def get_edit_tag_name(tag_lst): ...@@ -864,13 +904,13 @@ def get_edit_tag_name(tag_lst):
@bind("physical/search/device_linucb_tag_debug") @bind("physical/search/device_linucb_tag_debug")
def get_device_linucb_tag2(device_id = None,size = 20,has_score = False): def get_device_linucb_tag2(device_id=None, size=20, has_score=False):
# { # {
# "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字}, # "lincub": [{"id": 1, "name": 名字}, {"id": 2, "name": 名字}, {"id": 3, "name": 名字}, {"id": 5, "name": 名字},
# {"id": 6, "name": 名字}, ] # {"id": 6, "name": 名字}, ]
# } # }
try: try:
user_feature = [1,1] user_feature = [1, 1]
if device_id: if device_id:
linucb_matrix_redis_prefix = "physical:linucb:device_id:" linucb_matrix_redis_prefix = "physical:linucb:device_id:"
redis_key = linucb_matrix_redis_prefix + str(device_id) redis_key = linucb_matrix_redis_prefix + str(device_id)
...@@ -878,7 +918,10 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False): ...@@ -878,7 +918,10 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
if len(redis_linucb_tag_data_dict) == 0: if len(redis_linucb_tag_data_dict) == 0:
return {"linucb": []} return {"linucb": []}
else: else:
(recommend_tag_dict,recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,redis_linucb_tag_data_dict,user_feature,list(redis_linucb_tag_data_dict.keys())) (recommend_tag_dict, recommend_tag_set) = LinUCB.linucb_recommend_tag(device_id,
redis_linucb_tag_data_dict,
user_feature, list(
redis_linucb_tag_data_dict.keys()))
recommend_tag_list = list(recommend_tag_dict.keys()) recommend_tag_list = list(recommend_tag_dict.keys())
linucb_tag_dict_list = list() linucb_tag_dict_list = list()
linucb_tag_dict = dict() linucb_tag_dict = dict()
...@@ -889,7 +932,7 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False): ...@@ -889,7 +932,7 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
result_lst = [] result_lst = []
for id, name in tag_query_results: for id, name in tag_query_results:
result_lst.append({"name": name, "score": recommend_tag_dict[str(id)]}) result_lst.append({"name": name, "score": recommend_tag_dict[str(id)]})
linucb_tag_dict = {"linucb":result_lst} linucb_tag_dict = {"linucb": result_lst}
logging.info("physical/search/device_linucb_tag_debug:%s" % str(linucb_tag_dict)) logging.info("physical/search/device_linucb_tag_debug:%s" % str(linucb_tag_dict))
return linucb_tag_dict return linucb_tag_dict
else: else:
...@@ -898,6 +941,6 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False): ...@@ -898,6 +941,6 @@ def get_device_linucb_tag2(device_id = None,size = 20,has_score = False):
return linucb_tag_dict_list return linucb_tag_dict_list
else: else:
return {"linucb":[]} return {"linucb": []}
except: except:
return {"linucb": []} return {"linucb": []}
trans2es/management/commands/redis_new_topic_sub.py
View file @ 10b9bba0
...@@ -4,7 +4,7 @@ import datetime ...@@ -4,7 +4,7 @@ import datetime
from libs.cache import redis_client from libs.cache import redis_client
from libs.error import logging_exception from libs.error import logging_exception
from django.conf import settings from django.conf import settings
from trans2es.models.portrait_stat import LikeDeviceTagStat from trans2es.models.portrait_stat import LikeTopicStat
try: try:
ps = redis_client.pubsub() ps = redis_client.pubsub()
...@@ -28,9 +28,9 @@ try: ...@@ -28,9 +28,9 @@ try:
insert_list = [] insert_list = []
for topic in insert_topic_ids: for topic in insert_topic_ids:
insert_list.append( insert_list.append(
LikeDeviceTagStat(create_time=datetime.datetime.today(), update_time=datetime.datetime.today(), LikeTopicStat(create_time=datetime.datetime.today(), update_time=datetime.datetime.today(),
topic_id=topic, is_new_topic=0, topic_ctr_30=0.0, like_rate_30=0.0)) topic_id=topic, is_new_topic=0, topic_ctr_30=0.0, like_rate_30=0.0))
LikeDeviceTagStat.objects.using(settings.MASTER_DB_NAME).bulk_create(insert_list) LikeTopicStat.objects.using(settings.MASTER_DB_NAME).bulk_create(insert_list)
logging.info("impressions count gt 100 topic ids" + str(insert_topic_ids)) logging.info("impressions count gt 100 topic ids" + str(insert_topic_ids))
json_all_new_topic_impression_count = json.dumps(all_new_topic_impression_count) json_all_new_topic_impression_count = json.dumps(all_new_topic_impression_count)
logging.info("all_new_topic_impression_count" + str(all_new_topic_impression_count)) logging.info("all_new_topic_impression_count" + str(all_new_topic_impression_count))
......
trans2es/management/commands/trans2es_data2es_parallel.py
View file @ 10b9bba0
...@@ -28,10 +28,13 @@ from trans2es.models.topic import Topic,ActionSumAboutTopic ...@@ -28,10 +28,13 @@ from trans2es.models.topic import Topic,ActionSumAboutTopic
from search.utils.common import * from search.utils.common import *
from linucb.views.collect_data import CollectData from linucb.views.collect_data import CollectData
from injection.data_sync.tasks import sync_user_similar_score from injection.data_sync.tasks import sync_user_similar_score
import datetime
from trans2es.models.tag import Tag from trans2es.models.tag import Tag
from libs.cache import redis_client from libs.cache import redis_client
from trans2es.models.tag import TopicTag from trans2es.models.tag import TopicTag
from libs.error import logging_exception
from trans2es.models.portrait_stat import LikeTopicStat
...@@ -215,6 +218,42 @@ class Command(BaseCommand): ...@@ -215,6 +218,42 @@ class Command(BaseCommand):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def sub_redis_new_topic_ids(self):
try:
ps = redis_client.pubsub()
ps.subscribe("new_topic_impression")
all_new_topic_impression_count_key = "all_new_topic_impression_count_key"
for item in ps.listen():
if item['type'] == 'message':
new_topic_ids = json.loads(item["data"])
all_new_topic_impression_count = json.loads(redis_client.get(all_new_topic_impression_count_key))
insert_topic_ids = []
for topic in new_topic_ids:
topic = str(topic)
if topic in all_new_topic_impression_count:
all_new_topic_impression_count[topic] = all_new_topic_impression_count[topic] + 1
if all_new_topic_impression_count[topic] > 100:
insert_topic_ids.append(int(topic))
all_new_topic_impression_count.pop(topic)
else:
all_new_topic_impression_count[topic] = 1
if insert_topic_ids:
insert_list = []
for topic in insert_topic_ids:
insert_list.append(
LikeTopicStat(create_time=datetime.datetime.today(),
update_time=datetime.datetime.today(),
topic_id=topic, is_new_topic=0, topic_ctr_30=0.0, like_rate_30=0.0))
LikeTopicStat.objects.using(settings.MASTER_DB_NAME).bulk_create(insert_list)
logging.info("impressions count gt 100 topic ids" + str(insert_topic_ids))
json_all_new_topic_impression_count = json.dumps(all_new_topic_impression_count)
logging.info("all_new_topic_impression_count" + str(all_new_topic_impression_count))
redis_client.set(all_new_topic_impression_count_key, json_all_new_topic_impression_count)
except:
logging_exception()
logging.error("redis new topic sub error!")
def handle(self, *args, **options): def handle(self, *args, **options):
try: try:
type_name_list = get_type_info_map().keys() type_name_list = get_type_info_map().keys()
...@@ -241,5 +280,8 @@ class Command(BaseCommand): ...@@ -241,5 +280,8 @@ class Command(BaseCommand):
if len(options["sync_type"]) and options["sync_type"]=="tagname": if len(options["sync_type"]) and options["sync_type"]=="tagname":
self.sync_tag_collecction_name_set() self.sync_tag_collecction_name_set()
if len(options["sync_type"]) and options["sync_type"] == "new_topic_sub":
self.sub_redis_new_topic_ids()
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment