Merge branch 'dlis' into 'master'

add

See merge request !117

app_conf.xml

@@ -14,5 +14,6 @@
         <element value="search.views.user"/>
         <element value="search.views.tag"/>
         <element value="injection.data_sync.tasks"/>
+        <element value="search.views.contrast_similar"/>
     </config>
 </gm_rpcd_config>

search/views/contrast_similar.py

+# -*- coding: UTF-8 -*-
+# !/usr/bin/env python
+
+from __future__ import unicode_literals, absolute_import, print_function
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from search.utils.topic import TopicUtils
+from libs.es import ESPerform
+from libs.cache import redis_client
+from search.utils.common import *
+
+from gm_rpcd.all import bind
+import functools
+import logging
+import traceback
+from libs.es import ESPerform
+
+logger = logging.getLogger(__name__)
+
+MAX_LOAD = 100
+GROUP_SIZE = 10
+
+
+@bind('physical/search/query_contrast_similar')
+def contrast_similar(user_id, offset=0, size=10):
+    try:
+        if isinstance(user_id, int):
+            contrast_topic_id_list = get_all_topic_id_list(user_id, offset, size)
+            return {"contrast_similar_topic_id": contrast_topic_id_list}
+
+    except:
+
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"contrast_similar_topic_id": []}
+
+
+def get_all_topic_id_list(user_id, offset, size):
+    try:
+        if user_id:
+            redis_key = "physical:user_similar:participant_user_id:" + str(user_id)
+            redis_field_val_list = redis_client.get(redis_key)
+            have_read_topic_id_list = json.loads(redis_field_val_list) if redis_field_val_list else []
+            read_topic_id_list = list()
+            if len(have_read_topic_id_list) > 0:
+                for i in have_read_topic_id_list:
+                    read_topic_id_list.append(i.get("contrast_user_id"))
+                q = {
+                    "query": {
+                        "function_score": {
+                            "query": {
+                                "bool": {
+                                    "must": [{
+                                        "term": {
+                                            "is_online": True
+                                        }
+                                    }, {
+                                        "term": {
+                                            "is_deleted": False
+                                        }
+                                    },
+                                        {
+                                            "terms": {
+                                                "content_level": [3, 4, 5]
+                                            }
+                                        }],
+                                    "minimum_should_match": 1,
+                                    "should": {
+                                        "terms": {
+                                            "user_id": read_topic_id_list
+                                        }
+                                    }
+                                }
+                            }
+
+                        }
+                    }
+
+                }
+
+                logging.info('contrast_similar,as q:%s' % str(q).encode('utf-8'))
+
+                result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
+                                                           query_body=q,
+                                                           offset=offset, size=size)
+                contrast_topic_ids = []
+
+                if len(result_dict['hits']) > 0:
+
+                    res_hit = result_dict['hits']
+
+                    for item in res_hit:
+                        topic_id = item['_source']['id']
+                        contrast_topic_ids.append(topic_id)
+
+                    logging.info('contrast_similar_topic_id:%s' % contrast_topic_ids)
+
+                return contrast_topic_ids
+
+
+    except:
+
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"contrast_similar_topic_id": []}

trans2es/management/commands/trans2es_data2es_parallel.py

@@ -73,8 +73,8 @@ class SyncDataToRedis(object):
                 redis_key = redis_key_prefix + str(participant_user_id)
                 similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False,
                                                                               participant_user_id=participant_user_id,
-                                                                              similarity__gt=0.4).order_by(
-                    "-similarity").limit(100)
+                                                                              similarity__gt=0.3).order_by(
+                    "-similarity")
 
                 item_list = list()
                 for item in similar_result_items:

trans2es/models/topic.py

@@ -105,18 +105,36 @@ class Topic(models.Model):
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return []
 
-    def get_tag_name_list(self,tag_id_list):
+    def get_tag_name_list(self, tag_id_list):
         try:
             tag_name_list = list()
-            query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name",flat=True)
-            for item in query_results:
-                tag_name_list.append(item)
+            logging.info("get tag_id_list :%s"%tag_id_list)
+            if len(tag_id_list) > 100:
+                lens = int(len(tag_id_list) / 100)
+                for i in range(lens+1):
+                    if i != 0:
+                        tag_id_100 = tag_id_list[i * 100:i * 100 + 100]
+                    if i == 0:
+                        tag_id_100 = tag_id_list[i:i + 100]
+                    if i * 100 == lens * 100 and len(tag_id_list[lens * 100:]) > 0:
+                        tag_id_100 = tag_id_list[lens * 100:]
+                    if tag_id_100:
+                        logging.info("get tag_id_100 :%s" % tag_id_100)
+
+                        query_results = Tag.objects.filter(id__in=tag_id_100).values_list("name", flat=True)
+                        for item in query_results:
+                            tag_name_list.append(item)
+            else:
+                query_results = Tag.objects.filter(id__in=tag_id_list).values_list("name", flat=True)
+                for item in query_results:
+                    tag_name_list.append(item)
 
             return tag_name_list
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return []
 
+
     def get_topic_offline_score(self):
         try:
             offline_score = 0.0

trans2es/models/user.py

@@ -22,7 +22,7 @@ class User(models.Model):
         verbose_name = u'用户'
         db_table = 'account_user'
 
-    id = models.IntegerField(verbose_name="主键ID",primary_key=True)
+    id = models.IntegerField(verbose_name="主键ID", primary_key=True)
     user_id = models.BigIntegerField(verbose_name=u'用户id', unique=True)
     nick_name = models.CharField(verbose_name=u'昵称', max_length=255, default='')
     profile_pic = models.CharField(verbose_name=u'头像', max_length=300)
@@ -31,56 +31,75 @@ class User(models.Model):
     country_id = models.CharField(verbose_name=u'国家id', max_length=40)
     is_online = models.BooleanField(verbose_name="是否上线")
     is_deleted = models.BooleanField(verbose_name='是否删除')
-    create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
+    create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
 
     def get_is_recommend_flag(self):
         is_shadow = False
         is_recommend = False
-        query_sql = UserExtra.objects.filter(user_id=self.user_id,is_deleted=False,is_online=True)
+        query_sql = UserExtra.objects.filter(user_id=self.user_id, is_deleted=False, is_online=True)
         for record in query_sql:
             is_recommend = record.is_recommend
             is_shadow = record.is_shadow
 
-        return (is_recommend,is_shadow)
+        return (is_recommend, is_shadow)
 
     def get_latest_topic_time_val(self):
         latest_topic_time_val = -1
 
         # 获取该用户最新发帖时间
-        topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",flat=True).first()
+        topic_records = Topic.objects.filter(user_id=self.user_id).order_by("-update_time").values_list("update_time",
+                                                                                                        flat=True).first()
         if topic_records:
             tzlc_topic_update_time = tzlc(topic_records)
             latest_topic_time_val = int(time.mktime(tzlc_topic_update_time.timetuple()))
 
         return latest_topic_time_val
 
-
     def get_follow_user_id_list(self):
         follow_user_id_list = list()
         user_follows = self.userfollow.filter(is_online=True)
         for user_follow in user_follows:
             follow_user_id_list.append(user_follow.follow_id)
-
         follow_user_detail_list = list()
-        sql_data_list = User.objects.filter(user_id__in=follow_user_id_list)
-        for detail_data in sql_data_list:
-            item = {
-                "user_id":detail_data.user_id,
-                "country_id":detail_data.country_id
-            }
-            follow_user_detail_list.append(item)
+        logging.info("follow_user_id_list:%s" % follow_user_id_list)
+        if len(follow_user_id_list) > 100:
+            lens = int(len(follow_user_id_list) / 100)
+            for i in range(lens + 1):
+                if i != 0:
+                    follow_user_id_list_100 = follow_user_id_list[i * 100:i * 100 + 100]
+                if i == 0:
+                    follow_user_id_list_100 = follow_user_id_list[i:i + 100]
+                if i * 100 == lens * 100 and len(follow_user_id_list[lens * 100:]) > 0:
+                    follow_user_id_list_100 = follow_user_id_list[lens * 100:]
+                if follow_user_id_list_100:
+                    logging.info("follow_user_id_list_100:%s" % follow_user_id_list_100)
+                    sql_data_list = User.objects.filter(user_id__in=follow_user_id_list_100)
+                    for detail_data in sql_data_list:
+                        item = {
+                            "user_id": detail_data.user_id,
+                            "country_id": detail_data.country_id
+                        }
+                        follow_user_detail_list.append(item)
+        else:
+            sql_data_list = User.objects.filter(user_id__in=follow_user_id_list)
+            for detail_data in sql_data_list:
+                item = {
+                    "user_id": detail_data.user_id,
+                    "country_id": detail_data.country_id
+                }
+                follow_user_detail_list.append(item)
 
         return follow_user_detail_list
 
     def get_attention_group_id_list(self):
         try:
             attention_group_id_list = list()
-            query_results = GroupUserRole.objects.filter(is_online=True,user_id=self.user_id)
+            query_results = GroupUserRole.objects.filter(is_online=True, user_id=self.user_id)
             for item in query_results:
                 item_dict = {
                     "group_id": item.group_id,
-                    "update_time_val":time.mktime(tzlc(item.update_time).timetuple())
+                    "update_time_val": time.mktime(tzlc(item.update_time).timetuple())
                 }
                 attention_group_id_list.append(item_dict)
 
@@ -91,24 +110,58 @@ class User(models.Model):
 
     def get_pick_user_id_list(self):
         pick_topic_id_list = list()
-        user_picks = self.user_pick.filter(is_deleted=False,is_pick=True)
+        user_picks = self.user_pick.filter(is_deleted=False, is_pick=True)
         for user_pick in user_picks:
             pick_topic_id_list.append(user_pick.picktopic_id)
 
         pick_user_id_list = []
-        topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list)
-        for topic_data in topic_sql_list:
-            pick_user_id_list.append(topic_data.user_id)
+
+        if len(pick_topic_id_list) > 100:
+            lens = int(len(pick_topic_id_list) / 100)
+            for i in range(lens + 1):
+                if i != 0:
+                    pick_topic_id_list_100 = pick_topic_id_list[i * 100:i * 100 + 100]
+                if i == 0:
+                    pick_topic_id_list_100 = pick_topic_id_list[i:i + 100]
+                if i * 100 == lens * 100 and len(pick_topic_id_list[lens * 100:]) > 0:
+                    pick_topic_id_list_100 = pick_topic_id_list[lens * 100:]
+                if pick_topic_id_list_100:
+                    topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list_100)
+                    for topic_data in topic_sql_list:
+                        pick_user_id_list.append(topic_data.user_id)
+        else:
+            topic_sql_list = Topic.objects.filter(id__in=pick_topic_id_list)
+            for topic_data in topic_sql_list:
+                pick_user_id_list.append(topic_data.user_id)
 
         pick_user_id_list = tuple(pick_user_id_list)
         pick_user_detail_list = list()
-        sql_data_list = User.objects.filter(user_id__in=pick_user_id_list)
-        for detail_data in sql_data_list:
-            item = {
-                "user_id":detail_data.user_id,
-                "country_id":detail_data.country_id
-            }
-            pick_user_detail_list.append(item)
+
+        if len(pick_user_id_list) > 100:
+            lens = int(len(pick_user_id_list) / 100)
+            for i in range(lens + 1):
+                if i != 0:
+                    pick_user_id_list_100 = pick_user_id_list[i * 100:i * 100 + 100]
+                if i == 0:
+                    pick_user_id_list_100 = pick_user_id_list[i:i + 100]
+                if i * 100 == lens * 100 and len(pick_user_id_list[lens * 100:]) > 0:
+                    pick_user_id_list_100 = pick_user_id_list[lens * 100:]
+                if pick_topic_id_list_100:
+                    sql_data_list = User.objects.filter(user_id__in=pick_user_id_list_100)
+                    for detail_data in sql_data_list:
+                        item = {
+                            "user_id": detail_data.user_id,
+                            "country_id": detail_data.country_id
+                        }
+                        pick_user_detail_list.append(item)
+        else:
+            sql_data_list = User.objects.filter(user_id__in=pick_user_id_list)
+            for detail_data in sql_data_list:
+                item = {
+                    "user_id": detail_data.user_id,
+                    "country_id": detail_data.country_id
+                }
+                pick_user_detail_list.append(item)
 
         return pick_user_detail_list
 
@@ -122,22 +175,39 @@ class User(models.Model):
                 same_group_user_id_list.append(user_items_list.user_id)
 
         same_group_detail_list = list()
-        sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list)
-        for detail_data in sql_data_list:
-            item = {
-                "user_id":detail_data.user_id,
-                "country_id":detail_data.country_id
-            }
-            same_group_detail_list.append(item)
+        if len(same_group_user_id_list) > 100:
+            lens = int(len(same_group_user_id_list) / 100)
+            for i in range(lens + 1):
+                if i != 0:
+                    same_group_user_id_list_100 = same_group_user_id_list[i * 100:i * 100 + 100]
+                if i == 0:
+                    same_group_user_id_list_100 = same_group_user_id_list[i:i + 100]
+                if i * 100 == (lens - 1) * 100:
+                    same_group_user_id_list_100 = same_group_user_id_list[lens * 100:]
+                if same_group_user_id_list_100:
+                    sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list_100)
+                    for detail_data in sql_data_list:
+                        item = {
+                            "user_id": detail_data.user_id,
+                            "country_id": detail_data.country_id
+                        }
+                        same_group_detail_list.append(item)
+        else:
+            sql_data_list = User.objects.filter(user_id__in=same_group_user_id_list)
+            for detail_data in sql_data_list:
+                item = {
+                    "user_id": detail_data.user_id,
+                    "country_id": detail_data.country_id
+                }
+                same_group_detail_list.append(item)
 
         return same_group_detail_list
 
-
     def get_user_tag_id_list(self):
         try:
             user_tag_id_list = list()
 
-            query_results = AccountUserTag.objects.filter(user=self.user_id,is_deleted=False)
+            query_results = AccountUserTag.objects.filter(user=self.user_id, is_deleted=False)
             for item in query_results:
                 user_tag_id_list.append(item.tag_id)
 
@@ -145,4 +215,3 @@ class User(models.Model):
         except:
             logging.error("catch exception,err_msg:%s" % traceback.format_exc())
             return []
-