Skip to content

P
physical
Commit 8dc8df6f authored by Kai's avatar Kai
Browse files
Options

update

parents 3c6db57f 58d4273c
Hide whitespace changes
Inline Side-by-side
Showing with 107 additions and 99 deletions
libs/es.py
View file @ 8dc8df6f
...@@ -276,18 +276,22 @@ class ESPerform(object): ...@@ -276,18 +276,22 @@ class ESPerform(object):
) )
q = { q = {
"query":{ "query":{
"bool":{ "function_score":{
"must":[ "query": {
{"range": {"content_level": {"gte": 3, "lte": 5}}}, "bool": {
{"term":{"is_online": True}}, "must": [
{"term":{"is_deleted": False}}, {"range": {"content_level": {"gte": 3, "lte": 5}}},
{"terms":{"tag_list":tag_id}} {"term": {"is_online": True}},
] {"term": {"is_deleted": False}},
{"terms": {"tag_list": tag_id}}
]
}
},
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list
} }
}, },
"boost_mode": "sum",
"score_mode": "sum",
"functions": functions_list,
"_source":{ "_source":{
"include":["id"] "include":["id"]
}, },
...@@ -298,7 +302,11 @@ class ESPerform(object): ...@@ -298,7 +302,11 @@ class ESPerform(object):
] ]
} }
if len(have_read_topic_id_list)>0: if len(have_read_topic_id_list)>0:
q["query"]["bool"]["must_not"] = have_read_topic_id_list q["query"]["function_score"]["query"]["bool"]["must_not"] = {
"terms":{
"id":have_read_topic_id_list
}
}
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q, result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", query_body=q,
offset=0, size=100,routing="3,4,5") offset=0, size=100,routing="3,4,5")
......
linucb/views/collect_data.py
View file @ 8dc8df6f
...@@ -35,6 +35,7 @@ class CollectData(object): ...@@ -35,6 +35,7 @@ class CollectData(object):
self.linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:" self.linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
self.linucb_recommend_topic_id_prefix = "physical:linucb:topic_recommend:device_id:" self.linucb_recommend_topic_id_prefix = "physical:linucb:topic_recommend:device_id:"
self.tag_topic_id_redis_prefix = "physical:tag_id:topic_id_list:" self.tag_topic_id_redis_prefix = "physical:tag_id:topic_id_list:"
self.click_recommend_redis_key_prefix = "physical:click_recommend:device_id:"
# 默认 # 默认
self.user_feature = [0,1] self.user_feature = [0,1]
...@@ -78,49 +79,54 @@ class CollectData(object): ...@@ -78,49 +79,54 @@ class CollectData(object):
have_read_topic_id_list.extend(promote_recommend_topic_id_list) have_read_topic_id_list.extend(promote_recommend_topic_id_list)
recommend_topic_id_list = list() recommend_topic_id_list = list()
recommend_topic_id_list_click = list()
if click_topic_tag_list: if click_topic_tag_list:
recommend_topic_id_list_click = ESPerform.get_tag_topic_list(click_topic_tag_list, if len(click_topic_tag_list)>0:
recommend_topic_id_list_click = ESPerform.get_tag_topic_list(click_topic_tag_list,
have_read_topic_id_list) have_read_topic_id_list)
if len(recommend_topic_id_list) > 0: if len(recommend_topic_id_list_click) > 0:
num = min(len(recommend_topic_id_list), 2) num = min(len(recommend_topic_id_list_click), 2)
recommend_topic_id_list.extend(recommend_topic_id_list_click[0:num]) logging.info("recommend_topic_id_list:%s" % (str(num)))
for i in range(0,num):
recommend_topic_id_list.append(recommend_topic_id_list_click[i])
have_read_topic_id_list.extend(recommend_topic_id_list)
click_recommend_redis_key = self.click_recommend_redis_key_prefix + str(device_id)
click_redis_data_dict = {
"data": json.dumps(recommend_topic_id_list),
"cursor": 0
}
redis_client.hmset(click_recommend_redis_key, click_redis_data_dict)
total_topic_list = list()
tag_topic_id_list = list()
tag_id_list = recommend_tag_list[0:100] tag_id_list = recommend_tag_list[0:100]
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
recommend_topic_id_list.extend(tag_topic_id_list)
# for index in range(0,100):
# recommend_topic_id_list_es = list()
# for tag_id in recommend_tag_list[0:100]:
# redis_tag_id_key = self.tag_topic_id_redis_prefix + str(tag_id)
# redis_tag_id_data = redis_client.get(redis_tag_id_key)
# tag_topic_id_list = json.loads(redis_tag_id_data) if redis_tag_id_data else []
# if not redis_tag_id_data:
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# redis_client.set(redis_tag_id_key,json.dumps(tag_topic_id_list))
# redis_client.expire(redis_tag_id_key,1*24*60*60)
#
# if len(tag_topic_id_list)>index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_random and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_random.append(topic_id)
# break
# tag_id = recommend_tag_list[0:100]
# tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id)
# if len(tag_topic_id_list) > index:
# for topic_id in tag_topic_id_list[index:]:
# if topic_id not in have_read_topic_id_list and topic_id not in recommend_topic_id_list_es and topic_id not in recommend_topic_id_list:
# recommend_topic_id_list_es.append(topic_id)
# if len(recommend_topic_id_list_es) == 100:
# break
# # random.shuffle(recommend_topic_id_list_random)
# recommend_topic_id_list.extend(recommend_topic_id_list_es)
topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id) topic_recommend_redis_key = self.linucb_recommend_topic_id_prefix + str(device_id)
redis_data_dict = { redis_topic_data_dict = redis_client.hgetall(topic_recommend_redis_key)
"data": json.dumps(recommend_topic_id_list), redis_topic_list = list()
"cursor":0 cursor = -1
} if b"data" in redis_topic_data_dict:
redis_client.hmset(topic_recommend_redis_key,redis_data_dict) redis_topic_list = json.loads(redis_topic_data_dict[b"data"]) if redis_topic_data_dict[
b"data"] else []
cursor = int(str(redis_topic_data_dict[b"cursor"], encoding="utf-8"))
if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
have_read_topic_id_list.extend(redis_topic_list[:2])
tag_topic_id_list = ESPerform.get_tag_topic_list(tag_id_list,have_read_topic_id_list)
if len(recommend_topic_id_list)>0:
tag_topic_id_list = recommend_topic_id_list + tag_topic_id_list
redis_data_dict = {
"data": json.dumps(tag_topic_id_list),
"cursor":0
}
redis_client.hmset(topic_recommend_redis_key,redis_data_dict)
else:
if cursor<=0 and len(redis_topic_list)>0:
tag_topic_id_list = redis_topic_list[:2] + tag_topic_id_list
tag_topic_id_list = list(set(tag_topic_id_list))
return True return True
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
...@@ -159,7 +165,13 @@ class CollectData(object): ...@@ -159,7 +165,13 @@ class CollectData(object):
tag_list = list() tag_list = list()
click_topic_tag_list = list() click_topic_tag_list = list()
collection_tag_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online","is_collection")
if len(collection_tag_sql_query_results)>0:
for tag_id,is_online,is_collection in collection_tag_sql_query_results:
if is_online and is_collection == 1:
click_topic_tag_list.append(tag_id)
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online") click_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id).values_list("tag_id","is_online")
for tag_id,is_online in click_sql_query_results: for tag_id,is_online in click_sql_query_results:
if is_online: if is_online:
...@@ -167,8 +179,10 @@ class CollectData(object): ...@@ -167,8 +179,10 @@ class CollectData(object):
tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter( tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id=tag_id).values_list("id", "collection", "is_ai") id=tag_id).values_list("id", "collection", "is_ai")
for id, collection, is_ai in tag_sql_query_results: for id, collection, is_ai in tag_sql_query_results:
if collection == 1 or is_ai == 1: if (is_ai == 1) and id not in click_topic_tag_list:
click_topic_tag_list.append(id) click_topic_tag_list.append(id)
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
is_click = 1 is_click = 1
is_vote = 0 is_vote = 0
......
search/utils/topic.py
View file @ 8dc8df6f
...@@ -621,8 +621,7 @@ class TopicUtils(object): ...@@ -621,8 +621,7 @@ class TopicUtils(object):
"term": {k: v}, "term": {k: v},
}) })
if v in (None, '', []):
if not v:
continue continue
if k in ["create_time_gte", "create_time_lte"]: if k in ["create_time_gte", "create_time_lte"]:
......
search/views/tag.py
View file @ 8dc8df6f
...@@ -24,34 +24,6 @@ def get_highlight(fields=[]): ...@@ -24,34 +24,6 @@ def get_highlight(fields=[]):
@bind("physical/search/query_tag") @bind("physical/search/query_tag")
def query_tag(query,offset,size): def query_tag(query,offset,size):
try: try:
"""
q = {
"query":{
"bool":{
"must":[
{"term":{"is_online":True}},
{"term": {"is_deleted": False}}
],
"should":[
{"multi_match":{
"query": query,
"fields":["name"],
"operator":"and"}}
],
"minimum_should_match":1
}
},
"sort":[
{"near_new_topic_num":{"order":"desc"}},
{'_score': {"order": "desc"}}
],
"_source": {
"includes": ["id", "name"]
}
}
q["highlight"] = get_highlight(["name"])
"""
q = { q = {
"suggest":{ "suggest":{
"tips-suggest":{ "tips-suggest":{
...@@ -121,3 +93,11 @@ def query_by_tag_type(tag_type_id,offset,size): ...@@ -121,3 +93,11 @@ def query_by_tag_type(tag_type_id,offset,size):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []} return {"tag_list": []}
@bind("physical/search/choice_push_tag")
def choice_push_tag(device_id,user_id):
try:
pass
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []}
search/views/topic.py
View file @ 8dc8df6f
...@@ -59,6 +59,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -59,6 +59,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
redis_field_val_list = redis_client.hmget(redis_key, redis_field_list) redis_field_val_list = redis_client.hmget(redis_key, redis_field_list)
topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id) topic_recommend_redis_key = "physical:linucb:topic_recommend:device_id:" + str(device_id)
# click_recommend_redis_key = "physical:click_recommend:device_id:" + str(device_id)
# recommend_tag_dict = dict() # recommend_tag_dict = dict()
# tag_recommend_val = redis_client.get(tag_recommend_redis_key) # tag_recommend_val = redis_client.get(tag_recommend_redis_key)
...@@ -70,11 +72,25 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -70,11 +72,25 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
if b"data" in recommend_topic_dict: if b"data" in recommend_topic_dict:
recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"]) recommend_topic_id_list = json.loads(recommend_topic_dict[b"data"])
cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8")) cursor = int(str(recommend_topic_dict[b"cursor"], encoding="utf-8"))
newcursor = cursor + 6 newcursor = cursor + 5
if len(recommend_topic_id_list) > newcursor: if len(recommend_topic_id_list) > newcursor:
recommend_topic_list = recommend_topic_id_list[cursor:newcursor] recommend_topic_list = recommend_topic_id_list[cursor:newcursor]
redis_client.hset(topic_recommend_redis_key,"cursor",newcursor) redis_client.hset(topic_recommend_redis_key,"cursor",newcursor)
# click_recommend_topic_id_list = list()
# click_recommend_topic_list = list()
#
# click_recommend_topic_dict = redis_client.hgetall(click_recommend_redis_key)
# if b"data" in click_recommend_topic_dict:
# click_recommend_topic_id_list = json.loads(click_recommend_topic_dict[b"data"])
# cursor = int(str(click_recommend_topic_dict[b"cursor"], encoding="utf-8"))
# newcursor = cursor + 2
# if newcursor < 4 and len(click_recommend_topic_id_list) ==2:
# for i in range(0,2):
# click_recommend_topic_list.append(click_recommend_topic_id_list[i])
# redis_client.hset(click_recommend_redis_key, "cursor", newcursor)
# combine_recommend_topic_list_tmp = click_recommend_topic_list.extend(recommend_topic_list)
# combine_recommend_topic_list = combine_recommend_topic_list_tmp[0:5]
have_read_topic_id_list = list() have_read_topic_id_list = list()
if redis_field_val_list[0]: if redis_field_val_list[0]:
...@@ -93,8 +109,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -93,8 +109,10 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id) redis_tag_data = redis_client.hget("physical:linucb:register_user_tag_info", user_id)
attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else [] attention_tag_list = json.loads(redis_tag_data) if redis_tag_data else []
size = size-len(recommend_topic_list) if len(recommend_topic_list)>0:
rank_topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size, size = size-len(recommend_topic_list)
have_read_topic_id_list.extend(recommend_topic_list)
topic_id_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=offset, size=size,
single_size=size,query=query, query_type=query_type, single_size=size,query=query, query_type=query_type,
filter_topic_id_list=have_read_topic_id_list, filter_topic_id_list=have_read_topic_id_list,
recommend_tag_list=recommend_topic_list, recommend_tag_list=recommend_topic_list,
...@@ -138,21 +156,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -138,21 +156,8 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
# have_read_topic_id_list.append(unread_topic_id) # have_read_topic_id_list.append(unread_topic_id)
# else: # else:
# break # break
topic_id_list =list()
if(len(recommend_topic_list)==6): topic_id_list.extend(recommend_topic_list)
if(size <11):
topic_id_list.extend(rank_topic_id_list[0:3])
topic_id_list.extend(recommend_topic_id_list[0:3])
topic_id_list.extend(rank_topic_id_list[3:size])
topic_id_list.extend(recommend_topic_id_list[3:6])
else:
topic_id_list.extend(rank_topic_id_list[0:size-7])
topic_id_list.extend(recommend_topic_id_list[0:3])
topic_id_list.extend(rank_topic_id_list[size-7:size])
topic_id_list.extend(recommend_topic_id_list[3:6])
else:
topic_id_list.extend(rank_topic_id_list)
# topic_id_list.extend(recommend_topic_list)
# recommend_topic_list.extend(topic_id_list) # recommend_topic_list.extend(topic_id_list)
have_read_topic_id_list.extend(topic_id_list) have_read_topic_id_list.extend(topic_id_list)
if len(have_read_topic_id_list) > 30000: if len(have_read_topic_id_list) > 30000:
......
trans2es/models/tag.py
View file @ 8dc8df6f
...@@ -19,6 +19,8 @@ class TopicTag(models.Model): ...@@ -19,6 +19,8 @@ class TopicTag(models.Model):
tag_id = models.IntegerField(verbose_name=u"标签ID") tag_id = models.IntegerField(verbose_name=u"标签ID")
topic_id = models.IntegerField(verbose_name=u"帖子ID") topic_id = models.IntegerField(verbose_name=u"帖子ID")
is_online = models.BooleanField(verbose_name=u"是否在线") is_online = models.BooleanField(verbose_name=u"是否在线")
is_collection = models.IntegerField(verbose_name=u"是否编辑标签")
class AccountUserTag(models.Model): class AccountUserTag(models.Model):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment