Skip to content

P
physical
Commit 4f420ce0 authored by lixiaofang's avatar lixiaofang
Browse files
Options

Merge branch 'master' into vote_topic

parents 230e12fb 7be80ee9
Hide whitespace changes
Inline Side-by-side
Showing with 145 additions and 85 deletions
injection/data_sync/tasks.py
View file @ 4f420ce0
...@@ -15,7 +15,7 @@ from libs.es import ESPerform ...@@ -15,7 +15,7 @@ from libs.es import ESPerform
from libs.cache import redis_client from libs.cache import redis_client
from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar, UserSimilarScore
from linucb.utils.register_user_tag import RegisterUserTag from linucb.utils.register_user_tag import RegisterUserTag
from trans2es.models.tag import RegisterShowTag, Tag from trans2es.models.tag import SettingsConfig, Tag
@shared_task @shared_task
...@@ -113,16 +113,22 @@ def sync_user_similar_score(): ...@@ -113,16 +113,22 @@ def sync_user_similar_score():
def get_tag_count(): def get_tag_count():
try: try:
# 获取搜索推荐热词 # 获取搜索推荐热词
results_registr_tag = list(set(RegisterShowTag.objects.filter(is_deleted=False, is_online=1).values_list("tag_id", flat=True))) # results_registr_tag = list(set(SettingsConfig.objects.filter(is_deleted=False).values_list("val", flat=True)))
# tag_val_list =set()
# for item in results_registr_tag:
# for word in item.split():
# tag_val_list.add(word)
# 获取符合条件的核心词 # 获取符合条件的核心词
results_tag = list(set(Tag.objects.filter(is_online=True, is_deleted=False, collection=1).values_list("id", flat=True))) results_tag = list(
set(Tag.objects.filter(is_online=True, is_deleted=False, collection=1).values_list("id", flat=True)))
redis_registr_tag = "physical:search_hotword:results_registr_tag" redis_registr_tag = "physical:search_hotword:results_registr_tag"
redis_tag = "physical:search_hotword:results_tag" redis_tag = "physical:search_hotword:results_tag"
redis_client.set(redis_registr_tag, list(results_registr_tag)) # redis_client.set(redis_registr_tag, list(results_registr_tag))
redis_client.set(redis_tag, list(results_tag)) redis_client.set(redis_tag, list(results_tag))
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
linucb/views/collect_data.py
View file @ 4f420ce0
...@@ -164,10 +164,23 @@ class CollectData(object): ...@@ -164,10 +164,23 @@ class CollectData(object):
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id))) logging.info("consume topic_id:%s,device_id:%s" % (str(topic_id), str(device_id)))
topic_tag_list = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id,is_online=True).values_list("tag_id",flat=True)) # topic_tag_list = list(TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(topic_id=topic_id,is_online=True).values_list("tag_id",flat=True))
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=topic_tag_list,is_online=True,is_deleted=False).values_list("id","collection","is_ai") # tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(id__in=topic_tag_list,is_online=True,is_deleted=False).values_list("id","collection","is_ai")
for id,collection,is_ai in tag_query_results: # for id,collection,is_ai in tag_query_results:
if collection and is_ai: # if collection and is_ai:
# click_topic_tag_list.append(id)
topic_tag_list = list()
click_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).filter(
topic_id=topic_id, is_online=True).values_list("tag_id", "is_collection")
for tag_id, is_collection in click_results:
topic_tag_list.append(tag_id)
if is_collection:
click_topic_tag_list.append(tag_id)
tag_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id__in=topic_tag_list, is_online=True, is_deleted=False).values_list("id",
"is_ai")
for id, is_ai in tag_query_results:
if is_ai:
click_topic_tag_list.append(id) click_topic_tag_list.append(id)
logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % ( logging.info("positive tag_list,device_id:%s,topic_id:%s,tag_list:%s" % (
...@@ -197,58 +210,58 @@ class CollectData(object): ...@@ -197,58 +210,58 @@ class CollectData(object):
# 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后 # 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后
if len(click_topic_tag_list)>0: if len(click_topic_tag_list)>0:
self.update_recommend_tag_list(device_id, user_feature, user_id,click_topic_tag_list=click_topic_tag_list) self.update_recommend_tag_list(device_id, user_feature, user_id,click_topic_tag_list=click_topic_tag_list)
elif "type" in raw_val_dict and "page_precise_exposure" == raw_val_dict["type"]: # elif "type" in raw_val_dict and "page_precise_exposure" == raw_val_dict["type"]:
if isinstance(raw_val_dict["params"]["exposure_cards"],str): # if isinstance(raw_val_dict["params"]["exposure_cards"],str):
exposure_cards_list = json.loads(raw_val_dict["params"]["exposure_cards"]) # exposure_cards_list = json.loads(raw_val_dict["params"]["exposure_cards"])
elif isinstance(raw_val_dict["params"]["exposure_cards"],list): # elif isinstance(raw_val_dict["params"]["exposure_cards"],list):
exposure_cards_list = raw_val_dict["params"]["exposure_cards"] # exposure_cards_list = raw_val_dict["params"]["exposure_cards"]
else: # else:
exposure_cards_list = list() # exposure_cards_list = list()
device_id = raw_val_dict["device"]["device_id"] # device_id = raw_val_dict["device"]["device_id"]
user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None # user_id = raw_val_dict["user_id"] if "user_id" in raw_val_dict else None
logging.warning("type msg:%s" % raw_val_dict.get("type")) # logging.warning("type msg:%s" % raw_val_dict.get("type"))
exposure_topic_id_list = list() # exposure_topic_id_list = list()
for item in exposure_cards_list: # for item in exposure_cards_list:
if "card_id" not in item: # if "card_id" not in item:
continue # continue
exposure_topic_id = item["card_id"] # exposure_topic_id = item["card_id"]
logging.info( # logging.info(
"consume exposure topic_id:%s,device_id:%s" % (str(exposure_topic_id), str(device_id))) # "consume exposure topic_id:%s,device_id:%s" % (str(exposure_topic_id), str(device_id)))
if exposure_topic_id: # if exposure_topic_id:
exposure_topic_id_list.append(exposure_topic_id) # exposure_topic_id_list.append(exposure_topic_id)
#
topic_tag_id_dict = dict() # topic_tag_id_dict = dict()
tag_list = list() # tag_list = list()
exposure_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).\ # exposure_sql_query_results = TopicTag.objects.using(settings.SLAVE_DB_NAME).\
filter(topic_id__in=exposure_topic_id_list).\ # filter(topic_id__in=exposure_topic_id_list).\
values_list("topic_id","tag_id","is_online","is_collection") # values_list("topic_id","tag_id","is_online","is_collection")
# if len(exposure_sql_query_results)>0: # # if len(exposure_sql_query_results)>0:
for topic_id,tag_id,is_online,is_collection in exposure_sql_query_results: # for topic_id,tag_id,is_online,is_collection in exposure_sql_query_results:
if is_online and is_collection == 1: # if is_online and is_collection == 1:
tag_list.append(tag_id) # tag_list.append(tag_id)
if is_online: # if is_online:
tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter( # tag_sql_query_results = Tag.objects.using(settings.SLAVE_DB_NAME).filter(
id=tag_id).values_list("id", "collection", "is_ai") # id=tag_id).values_list("id", "collection", "is_ai")
for id, collection, is_ai in tag_sql_query_results: # for id, collection, is_ai in tag_sql_query_results:
if (is_ai == 1) and id not in tag_list: # if (is_ai == 1) and id not in tag_list:
tag_list.append(id) # tag_list.append(id)
#
if topic_id not in topic_tag_id_dict: # if topic_id not in topic_tag_id_dict:
topic_tag_id_dict[topic_id] = list() # topic_tag_id_dict[topic_id] = list()
topic_tag_id_dict[topic_id].append(tag_id) # topic_tag_id_dict[topic_id].append(tag_id)
#
is_click = 0 # is_click = 0
is_vote = 0 # is_vote = 0
#
reward = 1 if is_click or is_vote else 0 # reward = 1 if is_click or is_vote else 0
#
logging.info("negative tag_list,device_id:%s,topic_tag_id_dict:%s" % ( # logging.info("negative tag_list,device_id:%s,topic_tag_id_dict:%s" % (
str(device_id), str(topic_tag_id_dict))) # str(device_id), str(topic_tag_id_dict)))
for tag_id in tag_list: # for tag_id in tag_list:
self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature) # self.update_user_linucb_tag_info(reward, device_id, tag_id, user_feature)
#
# 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后 # # 更新该用户的推荐tag数据，放在 更新完成user tag行为信息之后
self.update_recommend_tag_list(device_id, user_feature, user_id) # self.update_recommend_tag_list(device_id, user_feature, user_id)
elif "type" in raw_val_dict and "interest_choice_click_next" == raw_val_dict["type"]: elif "type" in raw_val_dict and "interest_choice_click_next" == raw_val_dict["type"]:
if isinstance(raw_val_dict["params"]["tagid_list"],str): if isinstance(raw_val_dict["params"]["tagid_list"],str):
tagid_list = json.loads(raw_val_dict["params"]["tagid_list"]) tagid_list = json.loads(raw_val_dict["params"]["tagid_list"])
......
search/views/search_hotword.py
View file @ 4f420ce0
...@@ -12,7 +12,7 @@ from search.utils.group import GroupUtils ...@@ -12,7 +12,7 @@ from search.utils.group import GroupUtils
from search.utils.common import GroupSortTypes from search.utils.common import GroupSortTypes
from libs.es import ESPerform from libs.es import ESPerform
from trans2es.models.pictorial import PictorialTopics from trans2es.models.pictorial import PictorialTopics
from trans2es.models.tag import RegisterShowTag, Tag from trans2es.models.tag import SettingsConfig, Tag
from libs.cache import redis_client from libs.cache import redis_client
...@@ -33,25 +33,20 @@ def search_hotword(device_id=-1): ...@@ -33,25 +33,20 @@ def search_hotword(device_id=-1):
""" """
try: try:
all_tag_name_list = set() all_tag_name_list = set()
results_registr_tag = json.loads(redis_client.get("physical:search_hotword:results_registr_tag")) # results_registr_tag = json.loads(redis_client.get("physical:search_hotword:results_registr_tag"))
results_tag = json.loads(redis_client.get("physical:search_hotword:results_tag")) results_tag = json.loads(redis_client.get("physical:search_hotword:results_tag"))
# 先获取搜索推荐热词 # 先获取搜索推荐热词
for num in range(0, len(results_registr_tag) - 1): results_registr_tag = list(set(SettingsConfig.objects.filter(is_deleted=False,key=1).values_list("val", flat=True)))
tag_id = random.randint(0, len(results_registr_tag) - 1) tag_val_list = set()
results_tag_chose = list( for item in results_registr_tag:
set(RegisterShowTag.objects.filter(tag_id=results_registr_tag[tag_id], is_online=True).values_list( for word in item.split():
"tag_id", flat=True))) tag_val_list.add(word)
if results_tag_chose: tag_id_list = random.sample(range(0, len(tag_val_list)), 6)
results_tag_recommend = list( for tag_id in tag_id_list:
set(Tag.objects.filter(id=results_tag_chose[0], is_online=True).values_list("name", tag_val = list(tag_val_list)[tag_id]
flat=True))) all_tag_name_list.add(tag_val)
if results_tag_recommend:
all_tag_name_list.add(results_tag_recommend[0])
if len(all_tag_name_list) == 6 or num == results_tag:
break
logging.info("get all_tag_name_list:%s" % all_tag_name_list) logging.info("get all_tag_name_list:%s" % all_tag_name_list)
# 获取个性化标签 # 获取个性化标签
linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:" linucb_recommend_redis_prefix = "physical:linucb:tag_recommend:device_id:"
tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id) tag_recommend_redis_key = linucb_recommend_redis_prefix + str(device_id)
...@@ -84,3 +79,4 @@ def search_hotword(device_id=-1): ...@@ -84,3 +79,4 @@ def search_hotword(device_id=-1):
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"recommend_tag_name": []} return {"recommend_tag_name": []}
sqls/deploy/sl_user_login_status.sql 0 → 100644
View file @ 4f420ce0
-- Deploy flipr:sl_user_login_status to mysql
BEGIN;
CREATE TABLE `sl_user_login_status` (
`user_id` varchar(100) NOT NULL COMMENT '用户ID',
`is_shadow` tinyint(1) NOT NULL COMMENT '是否是马甲用户',
`first_visit_day` date COMMENT '首次日期',
`last_visit_day` date COMMENT '最后一次登陆日期',
`day_id` varchar(10) NOT NULL COMMENT '数据账期',
PRIMARY KEY (`user_id`),
INDEX `lv_day` (`last_visit_day`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='数据仓库推送表用户登录状态表';
-- XXX Add DDLs here.
COMMIT;
sqls/revert/sl_user_login_status.sql 0 → 100644
View file @ 4f420ce0
-- Revert flipr:sl_user_login_status from mysql
BEGIN;
DROP table sl_user_login_status;
-- XXX Add DDLs here.
COMMIT;
sqls/sqitch.conf 0 → 100644
View file @ 4f420ce0
[core]
engine = mysql
# plan_file = sqitch.plan
# top_dir = .
# [engine "mysql"]
# target = db:mysql:
# registry = sqitch
# client = /usr/local/mysql/bin/mysql
sqls/sqitch.plan 0 → 100644
View file @ 4f420ce0
%syntax-version=1.0.0
%project=flipr
%uri=https://github.com/sqitchers/sqitch-mysql-intro/
sl_user_login_status 2019-06-25T11:06:15Z Lxrent <lxrent@lxrentdeMacBook-Pro.local> # 数据仓库推送表用户登录状态表
sqls/verify/sl_user_login_status.sql 0 → 100644
View file @ 4f420ce0
-- Verify flipr:sl_user_login_status on mysql
BEGIN;
SELECT user_id FROM sl_user_login_status;
-- XXX Add verifications here.
ROLLBACK;
trans2es/mapping/pictorial.json
View file @ 4f420ce0
...@@ -5,8 +5,8 @@ ...@@ -5,8 +5,8 @@
"is_online":{"type":"boolean"},//上线 "is_online":{"type":"boolean"},//上线
"is_deleted":{"type":"boolean"}, "is_deleted":{"type":"boolean"},
"is_recommend":{"type":"boolean"}, "is_recommend":{"type":"boolean"},
"name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_search"},
"description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "description":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_search"},
"topic_num":{"type":"long"}, "topic_num":{"type":"long"},
"creator_id":{"type":"long"}, "creator_id":{"type":"long"},
"icon":{"type":"text"}, "icon":{"type":"text"},
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
"create_time":{"type":"date", "format":"date_time_no_millis"}, "create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"}, "update_time":{"type":"date", "format":"date_time_no_millis"},
"tag_id":{"type":"long"}, "tag_id":{"type":"long"},
"tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_index"}, "tag_name":{"type":"text","analyzer":"gm_default_index","search_analyzer":"gm_default_search"},
"topic_id_list":{"type":"long"}, "topic_id_list":{"type":"long"},
"effective":{"type":"boolean"}, "effective":{"type":"boolean"},
"offline_score":{"type":"long"}, "offline_score":{"type":"long"},
......
trans2es/models/pictorial.py
View file @ 4f420ce0
...@@ -78,7 +78,7 @@ class Pictorial(models.Model): ...@@ -78,7 +78,7 @@ class Pictorial(models.Model):
for topic_id in topic_id_list: for topic_id in topic_id_list:
topic_id_object = Topic.objects.filter(id=int(topic_id)).first() topic_id_object = Topic.objects.filter(id=int(topic_id)).first()
if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5]: if topic_id_object and topic_id_object.is_online and int(topic_id_object.content_level) in [0, 3, 4, 5,6]:
effective_num += 1 effective_num += 1
if effective_num >= 5: if effective_num >= 5:
ret = True ret = True
......
trans2es/models/tag.py
View file @ 4f420ce0
...@@ -96,13 +96,14 @@ class CommunityTagType(models.Model): ...@@ -96,13 +96,14 @@ class CommunityTagType(models.Model):
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0)) update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
class RegisterShowTag(models.Model): class SettingsConfig(models.Model):
class Meta: class Meta:
verbose_name = "搜索热词" verbose_name = "搜索热词"
db_table = "register_show_tag" db_table = "settingsconfig"
id = models.IntegerField(primary_key=True, verbose_name=u"主键ID") id = models.IntegerField(primary_key=True, verbose_name=u"主键ID")
is_deleted = models.IntegerField(verbose_name=u"是否删除") is_deleted = models.IntegerField(verbose_name=u"是否删除")
is_online = models.IntegerField(verbose_name=u"是否在线") key = models.IntegerField(verbose_name=u"值")
tag_id = models.IntegerField(verbose_name="标签ID") val = models.IntegerField(verbose_name="标签内容")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment