Skip to content

P
physical
Commit b6da0fd6 authored by lixiaofang's avatar lixiaofang
Browse files
Options

Merge branch 'master' into day_week_mouth

parents d7d10542 4dab11a7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 91 additions and 37 deletions
linucb/views/collect_data.py
View file @ b6da0fd6
......@@ -181,7 +181,7 @@ class CollectData(object):
raw_val_dict = json.loads(ori_msg.value)
if "type" in raw_val_dict and \
(raw_val_dict["type"] in ("on_click_feed_topic_card","tag_zone_click_focus")):
(raw_val_dict["type"] in ("on_click_feed_topic_card","on_click_button")):
click_topic_tag_list = list()
if "on_click_feed_topic_card" == raw_val_dict["type"]:
......@@ -210,16 +210,18 @@ class CollectData(object):
logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % (
str(device_id), str(topic_id), str(click_topic_tag_list)))
else:
tag_name = raw_val_dict["params"]["query"]
query_type = raw_val_dict["params"]["type"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
if query_type=="do":
elif raw_val_dict["type"] == "on_click_button" and "page_name" in \
raw_val_dict["params"] and "button_name" in raw_val_dict["params"] \
and "extra_param" in raw_val_dict["params"]:
if raw_val_dict["params"]["page_name"] == "search_detail" and \
raw_val_dict["params"]["button_name"] == "focus_tag":
tag_name = raw_val_dict["params"]["extra_param"]
device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
tag_list = list(Tag.objects.using(settings.SLAVE1_DB_NAME).filter(name=tag_name,is_online=True,is_deleted=False, is_category=False).values_list("id",flat=True))
click_topic_tag_list.extend(tag_list)
logging.info("query tag attention,positive tag_list,device_id:%s,query_name:%s,tag_list:%s" % (
str(device_id), tag_name, str(click_topic_tag_list)))
logging.info("query tag attention,positive tag_list,device_id:%s,query_name:%s,tag_list:%s" % (
str(device_id), tag_name, str(click_topic_tag_list)))
logging.info("click_topic_tag_list:%s"%(str(click_topic_tag_list)))
......
search/utils/topic.py
View file @ b6da0fd6
This diff is collapsed.
search/views/group.py
View file @ b6da0fd6
......@@ -139,7 +139,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
}
}
q["_source"] = {
"includes": ["id", "pictorial_id", "tag_list"]
"includes": ["id", "pictorial_id", "edit_tag_list"]
}
result_dict = ESPerform.get_search_results(es_cli_obj, "topic", q, offset, size)
pict_pictorial_ids_list = []
......@@ -148,7 +148,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]:
pict_pictorial_ids_list = item["_source"]["pictorial_id"]
topic_tag_list = item["_source"]["tag_list"]
topic_tag_list = item["_source"]["edit_tag_list"]
##拿到这些标签的同义词
......@@ -178,6 +178,11 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"is_deleted": False
}
},
{
"term": {
"is_cover": True
}
}
]
}
......@@ -185,12 +190,18 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
q["_source"] = {
"includes": ["id", "update_time"]
}
q["sort"] = {
'update_time': {
q["sort"] = [
{
"real_user_activate_time": {
"order": "desc"
}
},
{'update_time': {
'order': 'desc'
}
}
}
]
result_dict = ESPerform.get_search_results(es_cli_obj, "pictorial", q, offset, size)
if len(result_dict["hits"]) > 0:
for item in result_dict["hits"]:
......@@ -208,7 +219,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"bool": {
"must": [{
"terms": {
"tag_id": all_tag
"edit_tag_id": all_tag
}}, {
"term": {
"is_online": True
......@@ -224,6 +235,10 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
}
}
}, {
"term": {
"is_cover": True
}
}]
}
}
......@@ -233,7 +248,7 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
"constant_score": {
"filter": {
"terms": {
"tag_id": topic_tag_list
"edit_tag_id": topic_tag_list
}
}
}
......@@ -251,13 +266,18 @@ def pictorial_topic(topic_id=-1, offset=0, size=10):
}
}
q["_source"] = {
"includes": ["id", "tag_id"]}
"includes": ["id", "edit_tag_id"]}
q["sort"] = [
{
"_score": {
"order": "desc"
}
},
{
"real_user_activate_time": {
"order": "desc"
}
},
{
'update_time': {
'order': 'desc'
......
search/views/tag.py
View file @ b6da0fd6
......@@ -238,7 +238,7 @@ def get_same_tagset_ids(tag_list):
try:
all_tag = list()
if isinstance(tag_list, int):
all_tag.append(list)
all_tag.append(tag_list)
else:
all_tag = tag_list
......@@ -246,14 +246,16 @@ def get_same_tagset_ids(tag_list):
tag_set_list_id = list(
CommunityTagSetRelation.objects.filter(tag_id__in=all_tag, is_deleted=False).values_list("tag_set_id",
flat=True))
logging.info("get tag_set_list_id:%s" % tag_set_list_id)
tag_ids = list(
CommunityTagSetRelation.objects.filter(tag_set_id__in=tag_set_list_id, is_deleted=False).values_list(
"tag_id",
flat=True))
return tag_ids
all_tag.extend(tag_ids)
logging.info("get_same_tagset_ids:%s" % str(all_tag))
return list(set(all_tag))
except:
logging_exception()
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"tag_list": []}
return []
trans2es/management/commands/trans2es_data2es_parallel.py
View file @ b6da0fd6
......@@ -29,6 +29,12 @@ from search.utils.common import *
from linucb.views.collect_data import CollectData
from injection.data_sync.tasks import sync_user_similar_score
from trans2es.models.tag import Tag
from libs.cache import redis_client
from trans2es.models.tag import TopicTag
class Job(object):
__es = None
......@@ -189,6 +195,26 @@ class Command(BaseCommand):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def sync_tag_collecction_name_set(self):
try:
collection_redis_key_name = "physical:official_tag_name_set"
tag_list = list(TopicTag.objects.filter(is_online=True, is_collection=True).values_list("tag_id", flat=True))
tag_name_list = list(Tag.objects.filter(id__in=tag_list, is_online=True, is_deleted=False,
is_category=False).values_list("name", flat=True))
logging.info("duan add,tag_list_len:%d,tag_name_list_len:%d" % (len(tag_list),len(tag_name_list)))
for tag_name in tag_name_list:
try:
if tag_name and len(tag_name)>0 and tag_name[0]:
redis_client.sadd(collection_redis_key_name,tag_name[0])
except:
pass
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
def handle(self, *args, **options):
try:
type_name_list = get_type_info_map().keys()
......@@ -211,5 +237,9 @@ class Command(BaseCommand):
if len(options["sync_type"]) and options["sync_type"]=="similar":
sync_user_similar_score()
if len(options["sync_type"]) and options["sync_type"]=="tagname":
self.sync_tag_collecction_name_set()
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
trans2es/type_info.py
View file @ b6da0fd6
......@@ -13,7 +13,7 @@ import elasticsearch.helpers
import sys
from libs.cache import redis_client
import copy
from django.conf import settings
from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar, pictorial, product
from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
......@@ -315,7 +315,7 @@ def get_type_info_map():
name='topic-star',
type='topic-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query, # 假的
query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query, # 假的
get_data_func=TopicTransfer.get_topic_data, # 假的
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -325,7 +325,7 @@ def get_type_info_map():
name='topic-star-routing',
type='topic-star-routing',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,
query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -335,7 +335,7 @@ def get_type_info_map():
name='topic-high-star', # >=4星日记
type='topic-high-star',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,
query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -345,7 +345,7 @@ def get_type_info_map():
name='excellect-topic', # 优质帖子
type='excellect-topic',
model=topic.ExcellentTopic,
query_deferred=lambda: topic.ExcellentTopic.objects.all().query,
query_deferred=lambda: topic.ExcellentTopic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=ExcellectTopicTransfer.get_excellect_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -355,7 +355,7 @@ def get_type_info_map():
name='topic', # 日记
type='topic',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,
query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -365,7 +365,7 @@ def get_type_info_map():
name='topic-v1', # 日记
type='topic-v1',
model=topic.Topic,
query_deferred=lambda: topic.Topic.objects.all().query,
query_deferred=lambda: topic.Topic.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TopicTransfer.get_topic_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -377,7 +377,7 @@ def get_type_info_map():
model=user.User,
# query_deferred=lambda:user.User.objects.all().query,
# query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query,
query_deferred=lambda: user.User.objects.all().query,
query_deferred=lambda: user.User.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=UserTransfer.get_user_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -399,7 +399,7 @@ def get_type_info_map():
type="celebrity",
model=celebrity.Celebrity,
# query_deferred=lambda:user.User.objects.all().query,
query_deferred=lambda: celebrity.Celebrity.objects.all().query,
query_deferred=lambda: celebrity.Celebrity.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=CelebrityTransfer.get_celebrity_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -410,7 +410,7 @@ def get_type_info_map():
type="group",
model=group.Group,
# query_deferred=lambda:user.User.objects.all().query,
query_deferred=lambda: group.Group.objects.all().query,
query_deferred=lambda: group.Group.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=GroupTransfer.get_group_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -420,7 +420,7 @@ def get_type_info_map():
name="tag", # 标签
type="tag",
model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().query,
query_deferred=lambda: tag.Tag.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TagTransfer.get_tag_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -430,7 +430,7 @@ def get_type_info_map():
name="tag_v1", # 标签
type="tag_v1",
model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().query,
query_deferred=lambda: tag.Tag.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TagTransfer.get_tag_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -440,7 +440,7 @@ def get_type_info_map():
name="tag-name", # 标签名字
type="tag-name",
model=tag.Tag,
query_deferred=lambda: tag.Tag.objects.all().query,
query_deferred=lambda: tag.Tag.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=TagTransfer.get_tag_name_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -450,7 +450,7 @@ def get_type_info_map():
name='contrast_similar', # facesimilar
type='contrast_similar',
model=contrast_similar.ContrastSimilar,
query_deferred=lambda: contrast_similar.ContrastSimilar.objects.all().query,
query_deferred=lambda: contrast_similar.ContrastSimilar.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......@@ -460,7 +460,7 @@ def get_type_info_map():
name="pictorial", # 画报
type="pictorial",
model=pictorial.Pictorial,
query_deferred=lambda: pictorial.Pictorial.objects.all().query,
query_deferred=lambda: pictorial.Pictorial.objects.using(settings.SLAVE2_DB_NAME).all().query,
get_data_func=PictorialTransfer.get_poctorial_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment