Skip to content

P
physical
Commit 294351f1 authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'test' into 'day_week_mouth' 

# Conflicts:
#   search/views/group.py
parents 42003a85 15d4baee
Hide whitespace changes
Inline Side-by-side
Showing with 186 additions and 102 deletions
linucb/views/collect_data.py
View file @ 294351f1
...@@ -17,6 +17,8 @@ import libs.tools as Tools ...@@ -17,6 +17,8 @@ import libs.tools as Tools
from trans2es.models.pictorial import CommunityPictorialHomeFeed from trans2es.models.pictorial import CommunityPictorialHomeFeed
from libs.error import logging_exception from libs.error import logging_exception
import os import os
from search.views.tag import get_same_tagset_ids
class KafkaManager(object): class KafkaManager(object):
consumser_obj = None consumser_obj = None
...@@ -91,7 +93,8 @@ class CollectData(object): ...@@ -91,7 +93,8 @@ class CollectData(object):
recommend_lin_pictorial_id_list = list() recommend_lin_pictorial_id_list = list()
if click_topic_tag_list and len(click_topic_tag_list)>0: if click_topic_tag_list and len(click_topic_tag_list)>0:
recommend_topic_id_list_click,recommend_topic_id_list_click_dict = ESPerform.get_tag_topic_list_dict(click_topic_tag_list, click_topic_tag_list_same_tagset_ids = get_same_tagset_ids(click_topic_tag_list)
recommend_topic_id_list_click,recommend_topic_id_list_click_dict = ESPerform.get_tag_topic_list_dict(click_topic_tag_list_same_tagset_ids,
have_read_topic_id_list,size=2) have_read_topic_id_list,size=2)
if len(recommend_topic_id_list_click) > 0: if len(recommend_topic_id_list_click) > 0:
recommend_topic_id_list.extend(recommend_topic_id_list_click) recommend_topic_id_list.extend(recommend_topic_id_list_click)
...@@ -120,12 +123,14 @@ class CollectData(object): ...@@ -120,12 +123,14 @@ class CollectData(object):
# if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0: # if len(recommend_topic_id_list)==0 and cursor==0 and len(redis_topic_list)>0:
# have_read_topic_id_list.extend(redis_topic_list[:2]) # have_read_topic_id_list.extend(redis_topic_list[:2])
if len(new_user_click_tag_list)>0: if len(new_user_click_tag_list)>0:
tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(new_user_click_tag_list, have_read_topic_id_list) new_user_click_tag_list_same_tagset_ids = get_same_tagset_ids(new_user_click_tag_list)
recommend_lin_pictorial_id_list = ESPerform.get_tag_pictorial_id_list(new_user_click_tag_list, tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(new_user_click_tag_list_same_tagset_ids, have_read_topic_id_list)
recommend_lin_pictorial_id_list = ESPerform.get_tag_pictorial_id_list(new_user_click_tag_list_same_tagset_ids,
have_read_lin_pictorial_id_list) have_read_lin_pictorial_id_list)
else: else:
tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(tag_id_list,have_read_topic_id_list) tag_id_list_same_tagset_ids = get_same_tagset_ids(tag_id_list)
recommend_lin_pictorial_id_list = ESPerform.get_tag_pictorial_id_list(tag_id_list, tag_topic_id_list,tag_topic_dict = ESPerform.get_tag_topic_list_dict(tag_id_list_same_tagset_ids,have_read_topic_id_list)
recommend_lin_pictorial_id_list = ESPerform.get_tag_pictorial_id_list(tag_id_list_same_tagset_ids,
have_read_lin_pictorial_id_list) have_read_lin_pictorial_id_list)
if len(recommend_topic_id_list)>0 or len(tag_topic_id_list)>0 or len(new_user_click_tag_list) > 0: if len(recommend_topic_id_list)>0 or len(tag_topic_id_list)>0 or len(new_user_click_tag_list) > 0:
......
search/utils/topic.py
View file @ 294351f1
...@@ -4,16 +4,14 @@ ...@@ -4,16 +4,14 @@
import logging import logging
import traceback import traceback
import json import json
from alpha_types.venus import TOPIC_SEARCH_SORT from alpha_types.venus import TOPIC_SEARCH_SORT
from search.views.tag import get_same_tagset_ids
from libs.es import ESPerform from libs.es import ESPerform
from .common import TopicDocumentField from .common import TopicDocumentField
from search.utils.common import * from search.utils.common import *
from trans2es.models.pictorial import PictorialTopics from trans2es.models.pictorial import PictorialTopics
from libs.cache import redis_client from libs.cache import redis_client
class TopicUtils(object): class TopicUtils(object):
@classmethod @classmethod
...@@ -239,9 +237,10 @@ class TopicUtils(object): ...@@ -239,9 +237,10 @@ class TopicUtils(object):
} }
query_function_score["boost_mode"] = "replace" query_function_score["boost_mode"] = "replace"
tag_ids = get_same_tagset_ids(tag_id)
query_function_score["query"]["bool"]["should"] = [ query_function_score["query"]["bool"]["should"] = [
{'multi_match': multi_match}, {'multi_match': multi_match},
{"term": {"tag_list": tag_id}}, {"terms": {"tag_list": tag_ids}},
{"term": {"user_nick_name_pre": query.lower()}} {"term": {"user_nick_name_pre": query.lower()}}
] ]
query_function_score["query"]["bool"]["minimum_should_match"] = 1 query_function_score["query"]["bool"]["minimum_should_match"] = 1
......
search/views/group.py
View file @ 294351f1
...@@ -17,6 +17,7 @@ from trans2es.models.topic import CommunityCategoryTagRelation ...@@ -17,6 +17,7 @@ from trans2es.models.topic import CommunityCategoryTagRelation
from trans2es.models.pictorial import PictorialTopics from trans2es.models.pictorial import PictorialTopics
from trans2es.models.pictorial import CommunityPictorialActivity from trans2es.models.pictorial import CommunityPictorialActivity
from alpha_types.venus import PICTORIAL_ACTIVITY_SORT, PICTORIAL_TOPIC_SORT from alpha_types.venus import PICTORIAL_ACTIVITY_SORT, PICTORIAL_TOPIC_SORT
from search.views.tag import get_same_tagset_ids
@bind("physical/search/query_pictorial") @bind("physical/search/query_pictorial")
...@@ -148,6 +149,10 @@ def pictorial_topic(topic_id=-1, offset=0, size=10): ...@@ -148,6 +149,10 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
for item in result_dict["hits"]: for item in result_dict["hits"]:
pict_pictorial_ids_list = item["_source"]["pictorial_id"] pict_pictorial_ids_list = item["_source"]["pictorial_id"]
topic_tag_list = item["_source"]["tag_list"] topic_tag_list = item["_source"]["tag_list"]
##拿到这些标签的同义词
topic_tag_list = get_same_tagset_ids(topic_tag_list)
##先拿到当前帖子对应的标签的分类ID ##先拿到当前帖子对应的标签的分类ID
tag_type_ids = CommunityCategoryTagRelation.objects.filter(tag_id__in=topic_tag_list, tag_type_ids = CommunityCategoryTagRelation.objects.filter(tag_id__in=topic_tag_list,
is_online=True, is_deleted=False).values_list( is_online=True, is_deleted=False).values_list(
......
search/views/tag.py
View file @ 294351f1
...@@ -6,13 +6,14 @@ from gm_rpcd.all import bind ...@@ -6,13 +6,14 @@ from gm_rpcd.all import bind
import logging import logging
import traceback import traceback
import json import json
from search.utils.topic import TopicUtils
from libs.es import ESPerform from libs.es import ESPerform
from libs.cache import redis_client from libs.cache import redis_client
from search.utils.common import * from search.utils.common import *
from trans2es.models.tag import TopicTag,AccountUserTag,CommunityTagFollow,Tag from trans2es.models.tag import TopicTag, AccountUserTag, CommunityTagFollow, Tag
import time import time
from trans2es.models.tag import CommunityTagSetRelation
from django.conf import settings from django.conf import settings
from libs.error import logging_exception
def get_highlight(fields=[]): def get_highlight(fields=[]):
...@@ -25,17 +26,17 @@ def get_highlight(fields=[]): ...@@ -25,17 +26,17 @@ def get_highlight(fields=[]):
@bind("physical/search/query_tag") @bind("physical/search/query_tag")
def query_tag(query,offset,size): def query_tag(query, offset, size):
try: try:
if query: if query:
query = query.lower() query = query.lower()
pre_q = { pre_q = {
"query":{ "query": {
"bool": { "bool": {
"must":[ "must": [
{"term":{"name_pre": query}}, {"term": {"name_pre": query}},
{"term":{"is_online": True}} {"term": {"is_online": True}}
] ]
} }
}, },
...@@ -46,26 +47,28 @@ def query_tag(query,offset,size): ...@@ -46,26 +47,28 @@ def query_tag(query,offset,size):
ret_list = list() ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=pre_q,offset=0,size=1) result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="tag", query_body=pre_q,
if len(result_dict["hits"])>0: offset=0, size=1)
if len(result_dict["hits"]) > 0:
hitLight = u'<ems>%s</ems>' % query hitLight = u'<ems>%s</ems>' % query
result_dict["hits"][0]["_source"]["highlight"] = result_dict["hits"][0]["_source"]["name"].replace(query, hitLight) result_dict["hits"][0]["_source"]["highlight"] = result_dict["hits"][0]["_source"]["name"].replace(query,
hitLight)
ret_list.append(result_dict["hits"][0]["_source"]) ret_list.append(result_dict["hits"][0]["_source"])
size -= 1 size -= 1
q = { q = {
"suggest":{ "suggest": {
"tips-suggest":{ "tips-suggest": {
"prefix":query, "prefix": query,
"completion":{ "completion": {
"field":"suggest", "field": "suggest",
"size":size, "size": size,
"contexts":{ "contexts": {
"is_online": [True], "is_online": [True],
"is_deleted": [False] "is_deleted": [False]
} }
}
} }
}
}, },
"sort": [ "sort": [
{"near_new_topic_num": {"order": "desc"}} {"near_new_topic_num": {"order": "desc"}}
...@@ -75,14 +78,14 @@ def query_tag(query,offset,size): ...@@ -75,14 +78,14 @@ def query_tag(query,offset,size):
} }
} }
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="tag", query_body=q,
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=q,offset=offset,size=size,is_suggest_request=True) offset=offset, size=size, is_suggest_request=True)
for tips_item in result_dict["suggest"]["tips-suggest"]: for tips_item in result_dict["suggest"]["tips-suggest"]:
for hit_item in tips_item["options"]: for hit_item in tips_item["options"]:
#if len(hit_item["contexts"])==2: # if len(hit_item["contexts"])==2:
if hit_item["_source"]["is_deleted"]==False and hit_item["_source"]["is_online"]==True: if hit_item["_source"]["is_deleted"] == False and hit_item["_source"]["is_online"] == True:
hitLight = u'<ems>%s</ems>' % query hitLight = u'<ems>%s</ems>' % query
hit_item["_source"]["highlight"] = hit_item["_source"]["name"].replace(query,hitLight) hit_item["_source"]["highlight"] = hit_item["_source"]["name"].replace(query, hitLight)
ret_list.append(hit_item["_source"]) ret_list.append(hit_item["_source"])
return {"tag_list": ret_list} return {"tag_list": ret_list}
...@@ -92,20 +95,20 @@ def query_tag(query,offset,size): ...@@ -92,20 +95,20 @@ def query_tag(query,offset,size):
@bind("physical/search/query_by_tag_type") @bind("physical/search/query_by_tag_type")
def query_by_tag_type(tag_type_id,offset,size): def query_by_tag_type(tag_type_id, offset, size):
try: try:
q = { q = {
"query":{ "query": {
"bool":{ "bool": {
"must":[ "must": [
{"term":{"is_online":True}}, {"term": {"is_online": True}},
{"term": {"is_deleted": False}} {"term": {"is_deleted": False}}
], ],
"filter":{"term":{"tag_type": tag_type_id}} "filter": {"term": {"tag_type": tag_type_id}}
} }
}, },
"sort":[ "sort": [
{"near_new_topic_num":{"order":"desc"}} {"near_new_topic_num": {"order": "desc"}}
], ],
"_source": { "_source": {
"includes": ["id", "name"] "includes": ["id", "name"]
...@@ -113,7 +116,8 @@ def query_by_tag_type(tag_type_id,offset,size): ...@@ -113,7 +116,8 @@ def query_by_tag_type(tag_type_id,offset,size):
} }
ret_list = list() ret_list = list()
result_dict = ESPerform.get_search_results(ESPerform.get_cli(),sub_index_name="tag",query_body=q,offset=offset,size=size) result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="tag", query_body=q,
offset=offset, size=size)
for hit_item in result_dict["hits"]: for hit_item in result_dict["hits"]:
ret_list.append(hit_item["_source"]) ret_list.append(hit_item["_source"])
...@@ -123,6 +127,20 @@ def query_by_tag_type(tag_type_id,offset,size): ...@@ -123,6 +127,20 @@ def query_by_tag_type(tag_type_id,offset,size):
return {"tag_list": []} return {"tag_list": []}
@bind("physical/search/choice_pictorial_push_tag")
def choice_pictorial_push_tag(device_id, user_id):
try:
linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
linucb_recommend_tag_data = redis_client.get(tag_recommend_redis_key)
linucb_recommend_tag_list = json.loads(linucb_recommend_tag_data) if linucb_recommend_tag_data else []
return {"pictorial_tag_list": linucb_recommend_tag_list[:50]}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"pictorial_tag_list": []}
@bind("physical/search/choice_push_tag") @bind("physical/search/choice_push_tag")
def choice_push_tag(device_id, user_id): def choice_push_tag(device_id, user_id):
""" """
...@@ -139,7 +157,7 @@ def choice_push_tag(device_id, user_id): ...@@ -139,7 +157,7 @@ def choice_push_tag(device_id, user_id):
redis_push_tag_dict = json.loads(redis_push_tag_data) if redis_push_tag_data else {} redis_push_tag_dict = json.loads(redis_push_tag_data) if redis_push_tag_data else {}
now_sec = int(time.time()) now_sec = int(time.time())
valid_time = 8*7*24*60*60 valid_time = 8 * 7 * 24 * 60 * 60
ori_key_list = list(redis_push_tag_dict.keys()) ori_key_list = list(redis_push_tag_dict.keys())
for tag_id in ori_key_list: for tag_id in ori_key_list:
...@@ -148,8 +166,11 @@ def choice_push_tag(device_id, user_id): ...@@ -148,8 +166,11 @@ def choice_push_tag(device_id, user_id):
redis_push_tag_list = list(redis_push_tag_dict.keys()) redis_push_tag_list = list(redis_push_tag_dict.keys())
redis_push_tag_list = [int(item) for item in redis_push_tag_list] redis_push_tag_list = [int(item) for item in redis_push_tag_list]
account_user_tag_list = list(AccountUserTag.objects.filter(user=user_id,is_deleted=False).values_list("tag_id",flat=True)) account_user_tag_list = list(
community_tag_follow_list = list(CommunityTagFollow.objects.filter(user_id=user_id,is_online=True,is_deleted=False).values_list("tag_id",flat=True)) AccountUserTag.objects.filter(user=user_id, is_deleted=False).values_list("tag_id", flat=True))
community_tag_follow_list = list(
CommunityTagFollow.objects.filter(user_id=user_id, is_online=True, is_deleted=False).values_list("tag_id",
flat=True))
linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:" linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id) tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
...@@ -160,15 +181,16 @@ def choice_push_tag(device_id, user_id): ...@@ -160,15 +181,16 @@ def choice_push_tag(device_id, user_id):
account_user_tag_list.extend(linucb_recommend_tag_list) account_user_tag_list.extend(linucb_recommend_tag_list)
unread_tag_list = list(set(account_user_tag_list) - set(redis_push_tag_list)) unread_tag_list = list(set(account_user_tag_list) - set(redis_push_tag_list))
unread_tag_list = list(Tag.objects.filter(id__in=unread_tag_list, is_online=True, is_deleted=False).values_list("id",flat=True)) unread_tag_list = list(
Tag.objects.filter(id__in=unread_tag_list, is_online=True, is_deleted=False).values_list("id", flat=True))
ret_tag_set = set() ret_tag_set = set()
if len(unread_tag_list)>0: if len(unread_tag_list) > 0:
for tag_id in unread_tag_list: for tag_id in unread_tag_list:
valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id,is_online=True).count() valid_tag_topic_num = TopicTag.objects.filter(tag_id=tag_id, is_online=True).count()
if valid_tag_topic_num>100: if valid_tag_topic_num > 100:
ret_tag_set.add(tag_id) ret_tag_set.add(tag_id)
redis_push_tag_dict[tag_id] = now_sec redis_push_tag_dict[tag_id] = now_sec
if len(ret_tag_set)>=1: if len(ret_tag_set) >= 1:
break break
redis_client.set(redis_push_tag_key, json.dumps(redis_push_tag_dict)) redis_client.set(redis_push_tag_key, json.dumps(redis_push_tag_dict))
...@@ -178,6 +200,7 @@ def choice_push_tag(device_id, user_id): ...@@ -178,6 +200,7 @@ def choice_push_tag(device_id, user_id):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []} return {"tag_list": []}
@bind("physical/search/identity_tag_name") @bind("physical/search/identity_tag_name")
def identity_tag_name(topic_content): def identity_tag_name(topic_content):
try: try:
...@@ -189,8 +212,9 @@ def identity_tag_name(topic_content): ...@@ -189,8 +212,9 @@ def identity_tag_name(topic_content):
'analyzer': "gm_default_index" 'analyzer': "gm_default_index"
} }
cli_info = settings.TAG_ES_INFO_LIST cli_info = settings.TAG_ES_INFO_LIST
res = ESPerform.get_analyze_results(es_cli=ESPerform.get_cli(cli_info=cli_info), sub_index_name="tag", query_body=body) res = ESPerform.get_analyze_results(es_cli=ESPerform.get_cli(cli_info=cli_info), sub_index_name="tag",
query_body=body)
logging.info("duan add,res:%s" % str(res).encode("utf-8")) logging.info("duan add,res:%s" % str(res).encode("utf-8"))
for item in res["tokens"]: for item in res["tokens"]:
...@@ -204,3 +228,32 @@ def identity_tag_name(topic_content): ...@@ -204,3 +228,32 @@ def identity_tag_name(topic_content):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_name_list": []} return {"tag_name_list": []}
def get_same_tagset_ids(tag_list):
"""
获取同一词集下的标签ID
:param tag_list:
:return:
"""
try:
all_tag = list()
if isinstance(tag_list, int):
all_tag.append(list)
else:
all_tag = tag_list
tag_set_list_id = list(
CommunityTagSetRelation.objects.filter(tag_id__in=all_tag, is_deleted=False).values_list("tag_set_id",
flat=True))
logging.info("get tag_set_list_id:%s" % tag_set_list_id)
tag_ids = list(
CommunityTagSetRelation.objects.filter(tag_set_id__in=tag_set_list_id, is_deleted=False).values_list(
"tag_id",
flat=True))
return tag_ids
except:
logging_exception()
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []}
search/views/topic.py
View file @ 294351f1
...@@ -16,6 +16,7 @@ from django.conf import settings ...@@ -16,6 +16,7 @@ from django.conf import settings
from libs.tools import get_have_read_lin_pictorial_id_list from libs.tools import get_have_read_lin_pictorial_id_list
import datetime import datetime
from trans2es.models.tag import Tag from trans2es.models.tag import Tag
from search.views.tag import get_same_tagset_ids
def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE): def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageType.FIND_PAGE):
...@@ -32,11 +33,12 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy ...@@ -32,11 +33,12 @@ def get_discover_page_topic_ids(user_id, device_id, size, query_type=TopicPageTy
if have_read_topic_id_list == None: if have_read_topic_id_list == None:
have_read_topic_id_list = list() have_read_topic_id_list = list()
recommend_topic_ids,ret_data_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0, size=size, recommend_topic_ids, ret_data_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=0, offset=0,
single_size=size, size=size,
query_type=query_type, single_size=size,
filter_topic_id_list=have_read_topic_id_list, query_type=query_type,
index_type="topic", routing="4,5,6") filter_topic_id_list=have_read_topic_id_list,
index_type="topic", routing="4,5,6")
have_read_topic_id_list.extend(recommend_topic_ids) have_read_topic_id_list.extend(recommend_topic_ids)
redis_dict = { redis_dict = {
...@@ -165,14 +167,16 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -165,14 +167,16 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
rank_topic_id_list = list() rank_topic_id_list = list()
ret_data_list = list() ret_data_list = list()
if size > 0: if size > 0:
rank_topic_id_list,ret_data_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id, offset=0, size=size, rank_topic_id_list, ret_data_list = TopicUtils.get_recommend_topic_ids(user_id=user_id, tag_id=tag_id,
single_size=size, query=query, offset=0, size=size,
query_type=query_type, single_size=size, query=query,
filter_topic_id_list=have_read_topic_id_list, query_type=query_type,
index_type=index_type, routing=topic_star_routing, filter_topic_id_list=have_read_topic_id_list,
attention_tag_list=attention_tag_list, index_type=index_type,
linucb_user_id_list=recommend_topic_user_list, routing=topic_star_routing,
disable_collpase=disable_collpase) attention_tag_list=attention_tag_list,
linucb_user_id_list=recommend_topic_user_list,
disable_collpase=disable_collpase)
# if len(recommend_topic_list) == 6 and query is None: # if len(recommend_topic_list) == 6 and query is None:
# if (size < 11): # if (size < 11):
...@@ -202,7 +206,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -202,7 +206,7 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d") today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d")
tomorrow = today + datetime.timedelta(days=1) tomorrow = today + datetime.timedelta(days=1)
nowTime = datetime.datetime.now() nowTime = datetime.datetime.now()
expire_time = (tomorrow - nowTime).seconds + 3*60*60 expire_time = (tomorrow - nowTime).seconds + 3 * 60 * 60
redis_client.expire(redis_key, expire_time) redis_client.expire(redis_key, expire_time)
ret_list = rank_topic_id_list if query is None else ret_data_list ret_list = rank_topic_id_list if query is None else ret_data_list
...@@ -217,10 +221,12 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query ...@@ -217,10 +221,12 @@ def get_home_recommend_topic_ids(user_id, device_id, tag_id, offset, size, query
else: else:
return [], [] return [], []
def get_home_recommend_pictorial_ids(user_id=-1,device_id="",size=4):
def get_home_recommend_pictorial_ids(user_id=-1, device_id="", size=4):
try: try:
pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id) pictorial_recommend_redis_key = "physical:linucb:pictorial_recommend:device_id:" + str(device_id)
have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id,user_id,TopicPageType.HOME_RECOMMEND) have_read_lin_pictorial_id_list = get_have_read_lin_pictorial_id_list(device_id, user_id,
TopicPageType.HOME_RECOMMEND)
pictorial_recommend_dict = redis_client.hgetall(pictorial_recommend_redis_key) pictorial_recommend_dict = redis_client.hgetall(pictorial_recommend_redis_key)
pictorial_recommend_list = list() pictorial_recommend_list = list()
if b"data" in pictorial_recommend_dict: if b"data" in pictorial_recommend_dict:
...@@ -238,7 +244,7 @@ def get_home_recommend_pictorial_ids(user_id=-1,device_id="",size=4): ...@@ -238,7 +244,7 @@ def get_home_recommend_pictorial_ids(user_id=-1,device_id="",size=4):
redis_key = "physical:home_pictorial_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str( redis_key = "physical:home_pictorial_recommend" + ":device_id:" + str(device_id) + ":query_type:" + str(
TopicPageType.HOME_RECOMMEND) TopicPageType.HOME_RECOMMEND)
redis_field_list = 'have_read_pictorial_list' redis_field_list = 'have_read_pictorial_list'
redis_client.hset(redis_key, redis_field_list,have_read_lin_pictorial_id_list) redis_client.hset(redis_key, redis_field_list, have_read_lin_pictorial_id_list)
today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d") today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d")
tomorrow = today + datetime.timedelta(days=1) tomorrow = today + datetime.timedelta(days=1)
nowTime = datetime.datetime.now() nowTime = datetime.datetime.now()
...@@ -313,7 +319,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic ...@@ -313,7 +319,7 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
usefulrecall=usefulrecall, usefulrecall=usefulrecall,
useful_tag_list=useful_tag_list) useful_tag_list=useful_tag_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids, return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"useful_topic_ids": useful_topic_ids,"linucb_pictorial_ids":recommend_pictorial_ids} "useful_topic_ids": useful_topic_ids, "linucb_pictorial_ids": recommend_pictorial_ids}
else: else:
recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, 4) recommend_pictorial_ids = get_home_recommend_pictorial_ids(user_id, device_id, 4)
size = size - len(recommend_pictorial_ids) size = size - len(recommend_pictorial_ids)
...@@ -322,13 +328,14 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic ...@@ -322,13 +328,14 @@ def home_recommend(device_id="", user_id=-1, offset=0, size=10, query_type=Topic
query_type=query_type, query_type=query_type,
promote_topic_list=promote_topic_list) promote_topic_list=promote_topic_list)
return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,"linucb_pictorial_ids":recommend_pictorial_ids} return {"linucb_topic_ids": recommend_topic_ids, "rank_topic_ids": rank_topic_ids,
"linucb_pictorial_ids": recommend_pictorial_ids}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
if usefulrecall != -1: if usefulrecall != -1:
return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": [],"linucb_pictorial_ids":[]} return {"linucb_topic_ids": [], "rank_topic_ids": [], "useful_topic_ids": [], "linucb_pictorial_ids": []}
else: else:
return {"linucb_topic_ids": [], "rank_topic_ids": [],"linucb_pictorial_ids":[]} return {"linucb_topic_ids": [], "rank_topic_ids": [], "linucb_pictorial_ids": []}
@bind("physical/search/discover_page") @bind("physical/search/discover_page")
...@@ -434,6 +441,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -434,6 +441,7 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
normal_tag_result = list() normal_tag_result = list()
category_tag_reuslt = list() category_tag_reuslt = list()
result = list() result = list()
topic_tag_list = get_same_tagset_ids(topic_tag_list)
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter( tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id__in=topic_tag_list, is_online=True, is_deleted=False).values_list("id", "is_category") id__in=topic_tag_list, is_online=True, is_deleted=False).values_list("id", "is_category")
...@@ -474,20 +482,20 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic ...@@ -474,20 +482,20 @@ def topic_detail_page_recommend(device_id="", user_id=-1, topic_id=-1, topic_pic
size = size - category_tag_size size = size - category_tag_size
if topic_user_id != -1: if topic_user_id != -1:
topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, topic_user_result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list, have_read_topic_list,
size, es_cli_obj, size, es_cli_obj,
index_type="topic", index_type="topic",
routing="3,4,5,6", routing="3,4,5,6",
topic_user_id=topic_user_id topic_user_id=topic_user_id
) )
topic_user_size = len(topic_user_result) topic_user_size = len(topic_user_result)
have_read_topic_list.extend(topic_user_result) have_read_topic_list.extend(topic_user_result)
if topic_user_size < size: if topic_user_size < size:
size = size - topic_user_size size = size - topic_user_size
result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id, result = TopicUtils.top_get_topic_detail_recommend_list(user_id, topic_id,
have_read_topic_list, have_read_topic_list,
size, es_cli_obj, size, es_cli_obj,
index_type="topic", routing="4,5,6") index_type="topic", routing="4,5,6")
have_read_topic_list.extend(result) have_read_topic_list.extend(result)
# have_read_topic_redis_data = redis_client.get(redis_key) # have_read_topic_redis_data = redis_client.get(redis_key)
...@@ -589,10 +597,12 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -589,10 +597,12 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
try: try:
must_topic_id_list = list(topic_similarity_score_dict.keys()) must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list,ret_data_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size, topic_id_list, ret_data_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset,
single_size=size, size=size,
must_topic_id_list=must_topic_id_list, index_type="topic", single_size=size,
routing="4,5,6") must_topic_id_list=must_topic_id_list,
index_type="topic",
routing="4,5,6")
return {"recommend_topic_ids": topic_id_list} return {"recommend_topic_ids": topic_id_list}
except: except:
......
trans2es/models/group.py
View file @ 294351f1
...@@ -9,31 +9,33 @@ from libs.es import ESPerform ...@@ -9,31 +9,33 @@ from libs.es import ESPerform
from django.db import models from django.db import models
import datetime import datetime
class Group(models.Model): class Group(models.Model):
class Meta: class Meta:
verbose_name = u'小组' verbose_name = u'小组'
db_table = 'group' db_table = 'group'
id = models.IntegerField(verbose_name=u"小组ID",primary_key=True) id = models.IntegerField(verbose_name=u"小组ID", primary_key=True)
name = models.CharField(verbose_name=u'名称', max_length=100, default='') name = models.CharField(verbose_name=u'名称', max_length=100, default='')
description = models.CharField(verbose_name=u'描述', max_length=200, default='') description = models.CharField(verbose_name=u'描述', max_length=200, default='')
creator_id = models.IntegerField(verbose_name=u'创建者ID') creator_id = models.IntegerField(verbose_name=u'创建者ID')
#icon = ImgUrlField('图标', max_length=255, img_type=IMG_TYPE.WATERMARK, default='') # icon = ImgUrlField('图标', max_length=255, img_type=IMG_TYPE.WATERMARK, default='')
icon = models.CharField(verbose_name=u'图标', max_length=128, default='') icon = models.CharField(verbose_name=u'图标', max_length=128, default='')
topic_num = models.IntegerField(verbose_name=u'帖子数', default=0) topic_num = models.IntegerField(verbose_name=u'帖子数', default=0)
is_online = models.BooleanField(verbose_name=u"是否有效", default=True) is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
is_deleted = models.BooleanField(verbose_name="是否删除") is_deleted = models.BooleanField(verbose_name="是否删除")
is_recommend = models.BooleanField(verbose_name=u"是否推荐", default=False) is_recommend = models.BooleanField(verbose_name=u"是否推荐", default=False)
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0)) create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0)) update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
#获取前一天4，5星发帖数 # 获取前一天4，5星发帖数
def get_high_quality_topic_num(self): def get_high_quality_topic_num(self):
yesterday = datetime.datetime.now()-datetime.timedelta(days=1) yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day) yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day)
yesterday_end_time = "%s-%s-%s 23:59:59" % (yesterday.year, yesterday.month, yesterday.day) yesterday_end_time = "%s-%s-%s 23:59:59" % (yesterday.year, yesterday.month, yesterday.day)
topic_num = self.group_topics.filter(content_level__in=("4","5"),create_time__gte=yesterday_begin_time,create_time__lte=yesterday_end_time).count() topic_num = self.group_topics.filter(content_level__in=("4", "5"), create_time__gte=yesterday_begin_time,
create_time__lte=yesterday_end_time).count()
return topic_num return topic_num
...@@ -48,4 +50,4 @@ class Group(models.Model): ...@@ -48,4 +50,4 @@ class Group(models.Model):
'create_time': self.create_time.timestamp(), 'create_time': self.create_time.timestamp(),
'update_time': self.update_time.timestamp(), 'update_time': self.update_time.timestamp(),
} }
return result return result
\ No newline at end of file
trans2es/models/pictorial.py
View file @ 294351f1
...@@ -93,7 +93,7 @@ class Pictorial(models.Model): ...@@ -93,7 +93,7 @@ class Pictorial(models.Model):
return False return False
# 获取前一天4，5星发帖数 # 获取前一天4，5星发帖数
def get_high_quality_topic_num(self): def get_high_quality_topic_num_pictorial(self):
try: try:
yesterday = datetime.datetime.now() - datetime.timedelta(days=1) yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day) yesterday_begin_time = "%s-%s-%s 00:00:00" % (yesterday.year, yesterday.month, yesterday.day)
...@@ -108,7 +108,7 @@ class Pictorial(models.Model): ...@@ -108,7 +108,7 @@ class Pictorial(models.Model):
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return False return 0
def get_tag_by_id(self): def get_tag_by_id(self):
try: try:
......
trans2es/models/tag.py
View file @ 294351f1
...@@ -107,4 +107,15 @@ class SettingsConfig(models.Model): ...@@ -107,4 +107,15 @@ class SettingsConfig(models.Model):
key = models.IntegerField(verbose_name=u"值") key = models.IntegerField(verbose_name=u"值")
val = models.IntegerField(verbose_name="标签内容") val = models.IntegerField(verbose_name="标签内容")
class CommunityTagSetRelation(models.Model):
class Meta:
verbose_name = "搜索热词"
db_table = "community_tag_set_relation"
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
tag_id = models.IntegerField(verbose_name=u"标签ID")
tag_set_id = models.IntegerField(verbose_name=u"标签ID")
is_deleted = models.BooleanField(verbose_name=u"是否删除")
trans2es/utils/pictorial_transfer.py
View file @ 294351f1
...@@ -59,7 +59,7 @@ class PictorialTransfer(object): ...@@ -59,7 +59,7 @@ class PictorialTransfer(object):
update_time = instance.update_time update_time = instance.update_time
tzlc_udpate_time = tzlc(update_time) tzlc_udpate_time = tzlc(update_time)
res["update_time"] = tzlc_udpate_time res["update_time"] = tzlc_udpate_time
res["high_quality_topic_num"] = instance.get_high_quality_topic_num() res["high_quality_topic_num"] = instance.get_high_quality_topic_num_pictorial()
tag_id_list = instance.get_tag_by_id() tag_id_list = instance.get_tag_by_id()
res["tag_id"] = tag_id_list res["tag_id"] = tag_id_list
res["tag_name"] = instance.get_tag_by_name(tag_id_list) res["tag_name"] = instance.get_tag_by_name(tag_id_list)
......
vest/request/auto_request.py
View file @ 294351f1
...@@ -315,7 +315,7 @@ def send_email(stat_data): ...@@ -315,7 +315,7 @@ def send_email(stat_data):
msg = MIMEText(stat_data, 'plain', 'utf-8') msg = MIMEText(stat_data, 'plain', 'utf-8')
msg['From'] = formataddr(["李小芳", my_sender]) msg['From'] = formataddr(["李小芳", my_sender])
msg["To"] = formataddr(["李小芳", my_user6]) msg["To"] = formataddr(["李小芳", my_user6])
msg['Subject'] = str(datetime.date.today()) + "like的测试环境的自动点赞出问题了" msg['Subject'] = str(datetime.date.today()) + "like的线上环境的自动点赞出问题了"
server = smtplib.SMTP_SSL("smtp.exmail.qq.com", 465) server = smtplib.SMTP_SSL("smtp.exmail.qq.com", 465)
server.login(my_sender, my_pass) server.login(my_sender, my_pass)
server.sendmail(my_sender, [my_user6], msg.as_string()) server.sendmail(my_sender, [my_user6], msg.as_string())
......
vest/request/get_session.py
View file @ 294351f1
...@@ -5,8 +5,8 @@ import json ...@@ -5,8 +5,8 @@ import json
from libs.cache import redis_client from libs.cache import redis_client
from libs.error import logging_exception from libs.error import logging_exception
# login_url = "http://earth.iyanzhi.com/api/account/login_pwd" login_url = "http://earth.iyanzhi.com/api/account/login_pwd"
login_url = "http://earth.gmapp.env/api/account/login_pwd" # login_url = "http://earth.gmapp.env/api/account/login_pwd"
def index_first(): def index_first():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment