Skip to content

P
physical
Commit 7be67b71 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'hk_now' into 'master' 

revise linucb recommend topic

See merge request alpha/physical!233
parents a1699140 e0dc1c54
Hide whitespace changes
Inline Side-by-side
Showing with 57 additions and 27 deletions
libs/es.py
View file @ 7be67b71
......@@ -264,29 +264,43 @@ class ESPerform(object):
return True
@classmethod
def get_tag_topic_list(cls,tag_id):
def get_tag_topic_list(cls,tag_id,have_read_topic_id_list):
try:
functions_list = list()
for id in tag_id:
functions_list.append(
{
"filter": {"term": {"tag_list": id}},
"weight": 1
}
)
q = {
"query":{
"bool":{
"must":[
{"range": {"content_level": {"gte": 4, "lte": 5}}},
{"range": {"content_level": {"gte": 3, "lte": 5}}},
{"term":{"is_online": True}},
{"term":{"is_deleted": False}},
{"term":{"tag_list":tag_id}}
{"terms":{"tag_list":tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list,
"_source":{
"include":["id"]
},
"sort":[
{"_score": {"order": "desc"}},
{"create_time_val":{"order":"desc"}},
{"language_type":{"order":"asc"}},
]
}
if len(have_read_topic_id_list)>0:
q["query"]["bool"]["must_not"] = have_read_topic_id_list
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=5000,routing="4,5")
offset=0, size=100,routing="3,4,5")
topic_id_list = [item["_source"]["id"] for item in result_dict["hits"]]
return topic_id_list
......
linucb/views/collect_data.py
View file @ 7be67b71
......@@ -7,7 +7,7 @@ from libs.cache import redis_client
import logging
from linucb.views.linucb import LinUCB
import json
from trans2es.models.tag import TopicTag
from trans2es.models.tag import TopicTag,Tag
import traceback
from django.conf import settings
from libs.es import ESPerform
......@@ -74,28 +74,37 @@ class CollectData(object):
have_read_topic_id_list = Tools.get_have_read_topic_id_list(device_id,-1,TopicPageType.HOME_RECOMMEND)
recommend_topic_id_list = list()
for index in range(0,100):
recommend_topic_id_list_random = list()
for tag_id in recommend_tag_list[0:100]:
redis_tag_id_key = self.tag_topic_id_redis_prefix + str(tag_id)
redis_tag_id_data = redis_client.get(redis_tag_id_key)
tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
if not redis_tag_id_data:
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
redis_client.expire(redis_tag_id_key,1*24*60*60)
if len(tag_topic_id_list)>index:
for topic_id in tag_topic_id_list[index:]:
if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_random and topic_id not in recommend_topic_id_list:
recommend_topic_id_list_random.append(topic_id)
break
random.shuffle(recommend_topic_id_list_random)
recommend_topic_id_list.extend(recommend_topic_id_list_random)
tag_id_list = recommend_tag_list[0:100]
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
# for index in range(0,100):
# recommend_topic_id_list_es = list()
# for tag_id in recommend_tag_list[0:100]:
# redis_tag_id_key = self.tag_topic_id_redis_prefix + str(tag_id)
# redis_tag_id_data = redis_client.get(redis_tag_id_key)
# tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
# if not redis_tag_id_data:
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
# redis_client.expire(redis_tag_id_key,1*24*60*60)
#
# if len(tag_topic_id_list)>index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_random and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_random.append(topic_id)
# break
# tag_id = recommend_tag_list[0:100]
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# if len(tag_topic_id_list) > index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_es and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_es.append(topic_id)
# if len(recommend_topic_id_list_es) == 100:
# break
# # random.shuffle(recommend_topic_id_list_random)
# recommend_topic_id_list.extend(recommend_topic_id_list_es)
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
redis_data_dict = {
"data": json.dumps(recommend_topic_id_list),
"data": json.dumps(tag_topic_id_list),
"cursor":0
}
redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
......@@ -138,7 +147,10 @@ class CollectData(object):
click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online")
for tag_id,is_online in click_sql_query_results:
if is_online:
tag_list.append(tag_id)
# tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id=tag_id).values_list("id","collection","is_ai")
# for id,collection,is_ai in tag_sql_query_results:
# if collection == 1 or is_ai == 1:
tag_list.append(tag_id)
is_click = 1
is_vote = 0
......@@ -176,7 +188,11 @@ class CollectData(object):
exposure_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id__in=exposure_topic_id_list).values_list("topic_id","tag_id","is_online")
for topic_id,tag_id,is_online in exposure_sql_query_results:
if is_online:
tag_list.append(tag_id)
# tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
# id=tag_id).values_list("id", "collection", "is_ai")
# for id, collection, is_ai in tag_sql_query_results:
# if collection == 1 or is_ai == 1:
tag_list.append(tag_id)
if topic_id not in topic_tag_id_dict:
topic_tag_id_dict[topic_id] = list()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment