Skip to content

P
physical
Commit 15416a2f authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'dlis' into 'master' 

add

See merge request !117
parents ca25c64a cdf3d891
Hide whitespace changes
Inline Side-by-side
Showing with 236 additions and 43 deletions
app_conf.xml
View file @ 15416a2f
......@@ -14,5 +14,6 @@
<element value="search.views.user"/>
<element value="search.views.tag"/>
<element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/>
</config>
</gm_rpcd_config>
search/views/contrast_similar.py 0 → 100644
View file @ 15416a2f
# -*- coding: UTF-8 -*-
# !/usr/bin/env python
from __future__ import unicode_literals, absolute_import, print_function
from gm_rpcd.all import bind
import logging
import traceback
import json
from search.utils.topic import TopicUtils
from libs.es import ESPerform
from libs.cache import redis_client
from search.utils.common import *
from gm_rpcd.all import bind
import functools
import logging
import traceback
from libs.es import ESPerform
logger = logging.getLogger(__name__)
MAX_LOAD = 100
GROUP_SIZE = 10
@bind('physical/search/query_contrast_similar')
def contrast_similar(user_id, offset=0, size=10):
try:
if isinstance(user_id, int):
contrast_topic_id_list = get_all_topic_id_list(user_id, offset, size)
return {"contrast_similar_topic_id": contrast_topic_id_list}
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"contrast_similar_topic_id": []}
def get_all_topic_id_list(user_id, offset, size):
try:
if user_id:
redis_key = "physical:user_similar:participant_user_id:" + str(user_id)
redis_field_val_list = redis_client.get(redis_key)
have_read_topic_id_list = json.loads(redis_field_val_list) if redis_field_val_list else []
read_topic_id_list = list()
if len(have_read_topic_id_list) > 0:
for i in have_read_topic_id_list:
read_topic_id_list.append(i.get("contrast_user_id"))
q = {
"query": {
"function_score": {
"query": {
"bool": {
"must": [{
"term": {
"is_online": True
}
}, {
"term": {
"is_deleted": False
}
},
{
"terms": {
"content_level": [3, 4, 5]
}
}],
"minimum_should_match": 1,
"should": {
"terms": {
"user_id": read_topic_id_list
}
}
}
}
}
}
}
logging.info('contrast_similar,as q:%s' % str(q).encode('utf-8'))
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
query_body=q,
offset=offset, size=size)
contrast_topic_ids = []
if len(result_dict['hits']) > 0:
res_hit = result_dict['hits']
for item in res_hit:
topic_id = item['_source']['id']
contrast_topic_ids.append(topic_id)
logging.info('contrast_similar_topic_id:%s' % contrast_topic_ids)
return contrast_topic_ids
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"contrast_similar_topic_id": []}
trans2es/management/commands/trans2es_data2es_parallel.py
View file @ 15416a2f
......@@ -73,8 +73,8 @@ class SyncDataToRedis(object):
redis_key = redis_key_prefix + str(participant_user_id)
similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False,
participant_user_id=participant_user_id,
similarity__gt=0.4).order_by(
"-similarity").limit(100)
similarity__gt=0.3).order_by(
"-similarity")
item_list = list()
for item in similar_result_items:
......
trans2es/models/topic.py
View file @ 15416a2f
......@@ -105,18 +105,36 @@ class Topic(models.Model):
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_tag_name_list(self,tag_id_list):
def get_tag_name_list(self, tag_id_list):
try:
tag_name_list = list()
query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name",flat=True)
for item in query_results:
tag_name_list.append(item)
logging.info("get tag_id_list :%s"%tag_id_list)
if len(tag_id_list) > 100:
lens = int(len(tag_id_list) / 100)
for i in range(lens+1):
if i != 0:
tag_id_100 = tag_id_list[i * 100:i * 100 + 100]
if i == 0:
tag_id_100 = tag_id_list[i:i + 100]
if i * 100 == lens * 100 and len(tag_id_list[lens * 100:]) > 0:
tag_id_100 = tag_id_list[lens * 100:]
if tag_id_100:
logging.info("get tag_id_100 :%s" % tag_id_100)
query_results = Tag.objects.filter(id__in=tag_id_100).values_list("name", flat=True)
for item in query_results:
tag_name_list.append(item)
else:
query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name", flat=True)
for item in query_results:
tag_name_list.append(item)
return tag_name_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
def get_topic_offline_score(self):
try:
offline_score = 0.0
......
trans2es/models/user.py
View file @ 15416a2f
......@@ -22,7 +22,7 @@ class User(models.Model):
verbose_name = u'用户'
db_table = 'account_user'
id = models.IntegerField(verbose_name="主键ID",primary_key=True)
id = models.IntegerField(verbose_name="主键ID", primary_key=True)
user_id = models.BigIntegerField(verbose_name=u'用户id', unique=True)
nick_name = models.CharField(verbose_name=u'昵称', max_length=255, default='')
profile_pic = models.CharField(verbose_name=u'头像', max_length=300)
......@@ -31,56 +31,75 @@ class User(models.Model):
country_id = models.CharField(verbose_name=u'国家id', max_length=40)
is_online = models.BooleanField(verbose_name="是否上线")
is_deleted = models.BooleanField(verbose_name='是否删除')
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
def get_is_recommend_flag(self):
is_shadow = False
is_recommend = False
query_sql = UserExtra.objects.filter(user_id=self.user_id,is_deleted=False,is_online=True)
query_sql = UserExtra.objects.filter(user_id=self.user_id, is_deleted=False, is_online=True)
for record in query_sql:
is_recommend = record.is_recommend
is_shadow = record.is_shadow
return (is_recommend,is_shadow)
return (is_recommend, is_shadow)
def get_latest_topic_time_val(self):
latest_topic_time_val = -1
# 获取该用户最新发帖时间
topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",flat=True).first()
topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",
flat=True).first()
if topic_records:
tzlc_topic_update_time = tzlc(topic_records)
latest_topic_time_val = int(time.mktime(tzlc_topic_update_time.timetuple()))
return latest_topic_time_val
def get_follow_user_id_list(self):
follow_user_id_list = list()
user_follows = self.userfollow.filter(is_online=True)
for user_follow in user_follows:
follow_user_id_list.append(user_follow.follow_id)
follow_user_detail_list = list()
sql_data_list = User.objects.filter(user_id__in=follow_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id":detail_data.user_id,
"country_id":detail_data.country_id
}
follow_user_detail_list.append(item)
logging.info("follow_user_id_list:%s" % follow_user_id_list)
if len(follow_user_id_list) > 100:
lens = int(len(follow_user_id_list) / 100)
for i in range(lens + 1):
if i != 0:
follow_user_id_list_100 = follow_user_id_list[i * 100:i * 100 + 100]
if i == 0:
follow_user_id_list_100 = follow_user_id_list[i:i + 100]
if i * 100 == lens * 100 and len(follow_user_id_list[lens * 100:]) > 0:
follow_user_id_list_100 = follow_user_id_list[lens * 100:]
if follow_user_id_list_100:
logging.info("follow_user_id_list_100:%s" % follow_user_id_list_100)
sql_data_list = User.objects.filter(user_id__in=follow_user_id_list_100)
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
follow_user_detail_list.append(item)
else:
sql_data_list = User.objects.filter(user_id__in=follow_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
follow_user_detail_list.append(item)
return follow_user_detail_list
def get_attention_group_id_list(self):
try:
attention_group_id_list = list()
query_results = GroupUserRole.objects.filter(is_online=True,user_id=self.user_id)
query_results = GroupUserRole.objects.filter(is_online=True, user_id=self.user_id)
for item in query_results:
item_dict = {
"group_id": item.group_id,
"update_time_val":time.mktime(tzlc(item.update_time).timetuple())
"update_time_val": time.mktime(tzlc(item.update_time).timetuple())
}
attention_group_id_list.append(item_dict)
......@@ -91,24 +110,58 @@ class User(models.Model):
def get_pick_user_id_list(self):
pick_topic_id_list = list()
user_picks = self.user_pick.filter(is_deleted=False,is_pick=True)
user_picks = self.user_pick.filter(is_deleted=False, is_pick=True)
for user_pick in user_picks:
pick_topic_id_list.append(user_pick.picktopic_id)
pick_user_id_list = []
topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list)
for topic_data in topic_sql_list:
pick_user_id_list.append(topic_data.user_id)
if len(pick_topic_id_list) > 100:
lens = int(len(pick_topic_id_list) / 100)
for i in range(lens + 1):
if i != 0:
pick_topic_id_list_100 = pick_topic_id_list[i * 100:i * 100 + 100]
if i == 0:
pick_topic_id_list_100 = pick_topic_id_list[i:i + 100]
if i * 100 == lens * 100 and len(pick_topic_id_list[lens * 100:]) > 0:
pick_topic_id_list_100 = pick_topic_id_list[lens * 100:]
if pick_topic_id_list_100:
topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list_100)
for topic_data in topic_sql_list:
pick_user_id_list.append(topic_data.user_id)
else:
topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list)
for topic_data in topic_sql_list:
pick_user_id_list.append(topic_data.user_id)
pick_user_id_list = tuple(pick_user_id_list)
pick_user_detail_list = list()
sql_data_list = User.objects.filter(user_id__in=pick_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id":detail_data.user_id,
"country_id":detail_data.country_id
}
pick_user_detail_list.append(item)
if len(pick_user_id_list) > 100:
lens = int(len(pick_user_id_list) / 100)
for i in range(lens + 1):
if i != 0:
pick_user_id_list_100 = pick_user_id_list[i * 100:i * 100 + 100]
if i == 0:
pick_user_id_list_100 = pick_user_id_list[i:i + 100]
if i * 100 == lens * 100 and len(pick_user_id_list[lens * 100:]) > 0:
pick_user_id_list_100 = pick_user_id_list[lens * 100:]
if pick_topic_id_list_100:
sql_data_list = User.objects.filter(user_id__in=pick_user_id_list_100)
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
pick_user_detail_list.append(item)
else:
sql_data_list = User.objects.filter(user_id__in=pick_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
pick_user_detail_list.append(item)
return pick_user_detail_list
......@@ -122,22 +175,39 @@ class User(models.Model):
same_group_user_id_list.append(user_items_list.user_id)
same_group_detail_list = list()
sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id":detail_data.user_id,
"country_id":detail_data.country_id
}
same_group_detail_list.append(item)
if len(same_group_user_id_list) > 100:
lens = int(len(same_group_user_id_list) / 100)
for i in range(lens + 1):
if i != 0:
same_group_user_id_list_100 = same_group_user_id_list[i * 100:i * 100 + 100]
if i == 0:
same_group_user_id_list_100 = same_group_user_id_list[i:i + 100]
if i * 100 == (lens - 1) * 100:
same_group_user_id_list_100 = same_group_user_id_list[lens * 100:]
if same_group_user_id_list_100:
sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list_100)
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
same_group_detail_list.append(item)
else:
sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list)
for detail_data in sql_data_list:
item = {
"user_id": detail_data.user_id,
"country_id": detail_data.country_id
}
same_group_detail_list.append(item)
return same_group_detail_list
def get_user_tag_id_list(self):
try:
user_tag_id_list = list()
query_results = AccountUserTag.objects.filter(user=self.user_id,is_deleted=False)
query_results = AccountUserTag.objects.filter(user=self.user_id, is_deleted=False)
for item in query_results:
user_tag_id_list.append(item.tag_id)
......@@ -145,4 +215,3 @@ class User(models.Model):
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment