Skip to content

P
physical
Commit 8dc8df6f authored by Kai's avatar Kai
Browse files
Options

update

parents 3c6db57f 58d4273c
Show whitespace changes
Inline Side-by-side
Showing with 96 additions and 88 deletions
libs/es.py
View file @ 8dc8df6f
......@@ -276,18 +276,22 @@ class ESPerform(object):
)
q = {
"query":{
"bool":{
"must":[
"function_score":{
"query": {
"bool": {
"must": [
{"range": {"content_level": {"gte": 3, "lte": 5}}},
{"term":{"is_online": True}},
{"term":{"is_deleted": False}},
{"terms":{"tag_list":tag_id}}
{"term": {"is_online": True}},
{"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list,
"functions": functions_list
}
},
"_source":{
"include":["id"]
},
......@@ -298,7 +302,11 @@ class ESPerform(object):
]
}
if len(have_read_topic_id_list)>0:
q["query"]["bool"]["must_not"] = have_read_topic_id_list
q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms":{
"id":have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=100,routing="3,4,5")
......
linucb/views/collect_data.py
View file @ 8dc8df6f
......@@ -35,6 +35,7 @@ class CollectData(object):
self.linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
self.linucb_recommend_topic_id_prefix = "physical:linucb:topic_recommend:device_id:"
self.tag_topic_id_redis_prefix = "physical:tag_id:topic_id_list:"
self.click_recommend_redis_key_prefix = "physical:click_recommend:device_id:"
# 默认
self.user_feature = [0,1]
......@@ -78,49 +79,54 @@ class CollectData(object):
have_read_topic_id_list.extend(promote_recommend_topic_id_list)
recommend_topic_id_list = list()
recommend_topic_id_list_click = list()
if click_topic_tag_list:
if len(click_topic_tag_list)>0:
recommend_topic_id_list_click = ESPerform.get_tag_topic_list(click_topic_tag_list,
have_read_topic_id_list)
if len(recommend_topic_id_list) > 0:
num = min(len(recommend_topic_id_list), 2)
recommend_topic_id_list.extend(recommend_topic_id_list_click[0:num])
if len(recommend_topic_id_list_click) > 0:
num = min(len(recommend_topic_id_list_click), 2)
logging.info("recommend_topic_id_list:%s" % (str(num)))
for i in range(0,num):
recommend_topic_id_list.append(recommend_topic_id_list_click[i])
have_read_topic_id_list.extend(recommend_topic_id_list)
click_recommend_redis_key = self.click_recommend_redis_key_prefix + str(device_id)
click_redis_data_dict = {
"data": json.dumps(recommend_topic_id_list),
"cursor": 0
}
redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
total_topic_list = list()
tag_topic_id_list = list()
tag_id_list = recommend_tag_list[0:100]
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
recommend_topic_id_list.extend(tag_topic_id_list)
# for index in range(0,100):
# recommend_topic_id_list_es = list()
# for tag_id in recommend_tag_list[0:100]:
# redis_tag_id_key = self.tag_topic_id_redis_prefix + str(tag_id)
# redis_tag_id_data = redis_client.get(redis_tag_id_key)
# tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
# if not redis_tag_id_data:
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
# redis_client.expire(redis_tag_id_key,1*24*60*60)
#
# if len(tag_topic_id_list)>index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_random and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_random.append(topic_id)
# break
# tag_id = recommend_tag_list[0:100]
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# if len(tag_topic_id_list) > index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_es and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_es.append(topic_id)
# if len(recommend_topic_id_list_es) == 100:
# break
# # random.shuffle(recommend_topic_id_list_random)
# recommend_topic_id_list.extend(recommend_topic_id_list_es)
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
redis_topic_list = list()
cursor = -1
if b"data" in redis_topic_data_dict:
redis_topic_list = json.loads(redis_topic_data_dict[b"data"]) if redis_topic_data_dict[
b"data"] else []
cursor = int(str(redis_topic_data_dict[b"cursor"], encoding="utf-8"))
if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
have_read_topic_id_list.extend(redis_topic_list[:2])
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
if len(recommend_topic_id_list)>0:
tag_topic_id_list = recommend_topic_id_list + tag_topic_id_list
redis_data_dict = {
"data": json.dumps(recommend_topic_id_list),
"data": json.dumps(tag_topic_id_list),
"cursor":0
}
redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
else:
if cursor<=0 and len(redis_topic_list)>0:
tag_topic_id_list = redis_topic_list[:2] + tag_topic_id_list
tag_topic_id_list = list(set(tag_topic_id_list))
return True
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
......@@ -159,7 +165,13 @@ class CollectData(object):
tag_list = list()
click_topic_tag_list = list()
collection_tag_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online","is_collection")
if len(collection_tag_sql_query_results)>0:
for tag_id,is_online,is_collection in collection_tag_sql_query_results:
if is_online and is_collection == 1:
click_topic_tag_list.append(tag_id)
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online")
for tag_id,is_online in click_sql_query_results:
if is_online:
......@@ -167,8 +179,10 @@ class CollectData(object):
tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id=tag_id).values_list("id", "collection", "is_ai")
for id, collection, is_ai in tag_sql_query_results:
if collection == 1 or is_ai == 1:
if (is_ai == 1) and id not in click_topic_tag_list:
click_topic_tag_list.append(id)
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
is_click = 1
is_vote = 0
......
search/utils/topic.py
View file @ 8dc8df6f
......@@ -621,8 +621,7 @@ class TopicUtils(object):
"term": {k: v},
})
if not v:
if v in (None, '', []):
continue
if k in ["create_time_gte", "create_time_lte"]:
......
search/views/tag.py
View file @ 8dc8df6f
......@@ -24,34 +24,6 @@ def get_highlight(fields=[]):
@bind("physical/search/query_tag")
def query_tag(query,offset,size):
try:
"""
q = {
"query":{
"bool":{
"must":[
{"term":{"is_online":True}},
{"term": {"is_deleted": False}}
],
"should":[
{"multi_match":{
"query": query,
"fields":["name"],
"operator":"and"}}
],
"minimum_should_match":1
}
},
"sort":[
{"near_new_topic_num":{"order":"desc"}},
{'_score': {"order": "desc"}}
],
"_source": {
"includes": ["id", "name"]
}
}
q["highlight"] = get_highlight(["name"])
"""
q = {
"suggest":{
"tips-suggest":{
......@@ -121,3 +93,11 @@ def query_by_tag_type(tag_type_id,offset,size):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []}
@bind("physical/search/choice_push_tag")
def choice_push_tag(device_id,user_id):
try:
pass
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []}
search/views/topic.py
View file @ 8dc8df6f
......@@ -59,6 +59,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
# click_recommend_redis_key = "physical:click_recommend:device_id:" + str(device_id)
# recommend_tag_dict = dict()
# tag_recommend_val = redis_client.get(tag_recommend_redis_key)
......@@ -70,11 +72,25 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if b"data" in recommend_topic_dict:
recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
newcursor = cursor + 6
newcursor = cursor + 5
if len(recommend_topic_id_list) > newcursor:
recommend_topic_list = recommend_topic_id_list[cursor:newcursor]
redis_client.hset(topic_recommend_redis_key,"cursor",newcursor)
# click_recommend_topic_id_list = list()
# click_recommend_topic_list = list()
#
# click_recommend_topic_dict = redis_client.hgetall(click_recommend_redis_key)
# if b"data" in click_recommend_topic_dict:
# click_recommend_topic_id_list = json.loads(click_recommend_topic_dict[b"data"])
# cursor = int(str(click_recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 2
# if newcursor < 4 and len(click_recommend_topic_id_list) ==2:
# for i in range(0,2):
# click_recommend_topic_list.append(click_recommend_topic_id_list[i])
# redis_client.hset(click_recommend_redis_key, "cursor", newcursor)
# combine_recommend_topic_list_tmp = click_recommend_topic_list.extend(recommend_topic_list)
# combine_recommend_topic_list = combine_recommend_topic_list_tmp[0:5]
have_read_topic_id_list = list()
if redis_field_val_list[0]:
......@@ -93,8 +109,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
if len(recommend_topic_list)>0:
size = size-len(recommend_topic_list)
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
have_read_topic_id_list.extend(recommend_topic_list)
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list,
recommend_tag_list=recommend_topic_list,
......@@ -138,21 +156,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# have_read_topic_id_list.append(unread_topic_id)
# else:
# break
topic_id_list =list()
if(len(recommend_topic_list)==6):
if(size <11):
topic_id_list.extend(rank_topic_id_list[0:3])
topic_id_list.extend(recommend_topic_id_list[0:3])
topic_id_list.extend(rank_topic_id_list[3:size])
topic_id_list.extend(recommend_topic_id_list[3:6])
else:
topic_id_list.extend(rank_topic_id_list[0:size-7])
topic_id_list.extend(recommend_topic_id_list[0:3])
topic_id_list.extend(rank_topic_id_list[size-7:size])
topic_id_list.extend(recommend_topic_id_list[3:6])
else:
topic_id_list.extend(rank_topic_id_list)
# topic_id_list.extend(recommend_topic_list)
topic_id_list.extend(recommend_topic_list)
# recommend_topic_list.extend(topic_id_list)
have_read_topic_id_list.extend(topic_id_list)
if len(have_read_topic_id_list) > 30000:
......
trans2es/models/tag.py
View file @ 8dc8df6f
......@@ -19,6 +19,8 @@ class TopicTag(models.Model):
tag_id = models.IntegerField(verbose_name=u"标签ID")
topic_id = models.IntegerField(verbose_name=u"帖子ID")
is_online = models.BooleanField(verbose_name=u"是否在线")
is_collection = models.IntegerField(verbose_name=u"是否编辑标签")
class AccountUserTag(models.Model):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment