Merge branch 'dev' into 'test'

Dev

See merge request !98

app_conf.xml

@@ -13,6 +13,7 @@
         <element value="search.views.group"/>
         <element value="search.views.user"/>
         <element value="search.views.tag"/>
+        <element value="search.views.contrast_similar"/>
         <element value="injection.data_sync.tasks"/>
     </config>
 </gm_rpcd_config>

physical/DBRouter.py

+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+class DBRouter:
+    """
+    A router to control all database operations on models in the
+    user application.
+    """
+
+    def db_for_read(self, model, **hints):
+        """
+        Attempts to read user models go to users_db.
+        """
+        if model._meta.app_label == 'face':
+            return 'face'
+        return None
+
+    def db_for_write(self, model, **hints):
+        """
+        Attempts to write user models go to users_db.
+        """
+        if model._meta.app_label == 'face':
+            return 'face'
+        return None
+
+    def allow_relation(self, obj1, obj2, **hints):
+        """
+        Allow relations if a model in the user app is involved.
+        """
+        if obj1._meta.app_label == 'face' or \
+                obj2._meta.app_label == 'face':
+            return True
+        return None
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        """
+        Make sure the auth app only appears in the 'users_db'
+        database.
+        """
+        if app_label == 'face':
+            return db == 'face'
+        return None

physical/settings.py

@@ -18,7 +18,6 @@ from celery.schedules import crontab
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
 
@@ -30,9 +29,8 @@ DEBUG = False
 
 ALLOWED_HOSTS = []
 
-
 # Application definition
-SENTRY_CELERY_ENDPOINT="http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
+SENTRY_CELERY_ENDPOINT = "http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
 BROKER_URL = "redis://127.0.0.1:6379/8"
 
 # CELERY_SEND_EVENTS = True
@@ -64,8 +62,8 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-	'trans2es',
-	'search',
+    'trans2es',
+    'search',
     'injection.data_sync',
 )
 
@@ -125,13 +123,17 @@ REDIS_URL = "redis://127.0.0.1:6379"
 # Database
 # https://docs.djangoproject.com/en/1.10/ref/settings/#databases
 
+DATABASE_ROUTERS = ['physical.DBRouter.DBRouter']
+
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
         'NAME': 'alpha',
         'USER': 'work',
         'PASSWORD': 'Gengmei123',
-        'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
+        # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        "HOST": 'rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com',
+        #'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
         'PORT': '3306',
         'OPTIONS': {
             "init_command": "SET foreign_key_checks = 0;",
@@ -143,9 +145,11 @@ DATABASES = {
         'NAME': 'face',
         'USER': 'work',
         'PASSWORD': 'Gengmei123',
-        'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
+        # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        "HOST": "rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com",
+        # 'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
         'PORT': '3306',
-        #'CONN_MAX_AGE': None,
+        # 'CONN_MAX_AGE': None,
         'OPTIONS': {
             "init_command": "SET foreign_key_checks = 0;",
             "charset": "utf8mb4",
@@ -154,13 +158,13 @@ DATABASES = {
 
 }
 ES_INFO_LIST = [
-        {
-            "host": "10.29.130.141",
-            "port": 9200
-        }
+    {
+        "host": "10.29.130.141",
+        "port": 9200
+    }
 ]
 
-ES_INDEX_PREFIX="gm-dbmw"
+ES_INDEX_PREFIX = "gm-dbmw"
 
 MIDDLEWARE_CLASSES = (
     'gm_tracer.middleware.TracerMiddleware',

search/utils/topic.py

@@ -92,7 +92,7 @@ class TopicUtils(object):
             return []
 
     @classmethod
-    def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[]):
+    def get_recommend_topic_ids(cls,user_id,offset,size,query=None,query_type=TopicPageType.FIND_PAGE,filter_topic_id_list=[],must_topic_id_list=[]):
         """
         :需增加打散逻辑
         :remark:获取首页推荐帖子列表
@@ -202,6 +202,12 @@ class TopicUtils(object):
                 "boost_mode": "sum",
                 "functions": functions_list
             }
+            if len(must_topic_id_list) > 0:
+                query_function_score["query"]["bool"]["must"] = {
+                    "terms":{
+                        "id": must_topic_id_list
+                    }
+                }
             if query_type==TopicPageType.FIND_PAGE and len(filter_topic_id_list)>0:
                 query_function_score["query"]["bool"]["must_not"] = {
                     "terms":{

search/views/contrast_similar.py

+# -*- coding: UTF-8 -*-
+# !/usr/bin/env python
+
+from __future__ import unicode_literals, absolute_import, print_function
+from gm_rpcd.all import bind
+import logging
+import traceback
+import json
+from search.utils.topic import TopicUtils
+from libs.es import ESPerform
+from libs.cache import redis_client
+from search.utils.common import *
+
+from gm_rpcd.all import bind
+import functools
+import logging
+import traceback
+from libs.es import ESPerform
+
+logger = logging.getLogger(__name__)
+
+MAX_LOAD = 100
+GROUP_SIZE = 10
+
+
+@bind('physical/search/query_contrast_similar')
+def contrast_similar(user_id, offset=0, size=10):
+    try:
+        if isinstance(user_id, int):
+            contrast_topic_id_list = get_all_topic_id_list(user_id, offset, size)
+            return {"contrast_similar_topic_id": contrast_topic_id_list}
+
+    except:
+
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"contrast_similar_topic_id": []}
+
+
+def get_all_topic_id_list(user_id, offset, size):
+    try:
+        if user_id:
+            redis_key = "physical:user_similar:participant_user_id:" + str(user_id)
+            redis_field_val_list = redis_client.get(redis_key)
+            have_read_topic_id_list = json.loads(redis_field_val_list) if redis_field_val_list else []
+            read_topic_id_list = list()
+            if len(have_read_topic_id_list) > 0:
+                for i in have_read_topic_id_list:
+                    read_topic_id_list.append(i.get("contrast_user_id"))
+                q = {
+                    "query": {
+                        "function_score": {
+                            "query": {
+                                "bool": {
+                                    "must": [{
+                                        "term": {
+                                            "is_online": True
+                                        }
+                                    }, {
+                                        "term": {
+                                            "is_deleted": False
+                                        }
+                                    },
+                                        {
+                                            "terms": {
+                                                "content_level": [3, 4, 5]
+                                            }
+                                        }],
+                                    "minimum_should_match": 1,
+                                    "should": {
+                                        "terms": {
+                                            "user_id": read_topic_id_list
+                                        }
+                                    }
+                                }
+                            }
+
+                        }
+                    }
+
+                }
+
+                logging.info('contrast_similar,as q:%s' % str(q).encode('utf-8'))
+
+                result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic",
+                                                           query_body=q,
+                                                           offset=offset, size=size)
+                contrast_topic_ids = []
+
+                if len(result_dict['hits']) > 0:
+
+                    res_hit = result_dict['hits']
+
+                    for item in res_hit:
+                        topic_id = item['_source']['id']
+                        contrast_topic_ids.append(topic_id)
+
+                    logging.info('contrast_similar_topic_id:%s' % contrast_topic_ids)
+
+                return contrast_topic_ids
+
+
+    except:
+
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"contrast_similar_topic_id": []}

search/views/topic.py

@@ -266,3 +266,23 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
     except:
         logging.error("catch exception, err_msg:%s" % traceback.format_exc())
         return {"topic_ids": []}
+
+
+@bind("physical/search/query_topic_by_user_similarity")
+def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=10):
+    """
+    :remark 按帖子所属用户相似度召回帖子
+    :param tag_id:
+    :param offset:
+    :param size:
+    :return:
+    """
+    try:
+        must_topic_id_list = topic_similarity_score_dict.keys()
+        topic_id_dict = TopicUtils.get_recommend_topic_ids(user_id=-1, offset=offset, size=size,must_topic_id_list=must_topic_id_list)
+
+        recommend_topic_ids = list(topic_id_dict.keys())
+        return {"recommend_topic_ids":recommend_topic_ids}
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+        return {"recommend_topic_id": []}

trans2es/management/commands/trans2es_data2es_parallel.py

@@ -65,8 +65,8 @@ class SyncDataToRedis(object):
                 redis_key = redis_key_prefix + str(participant_user_id)
                 similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False,
                                                                               participant_user_id=participant_user_id,
-                                                                              similarity__gt=0.4).order_by(
-                    "-similarity").limit(100)
+                                                                              similarity__gt=0.3).order_by(
+                    "-similarity")
 
                 item_list = list()
                 for item in similar_result_items:

trans2es/models/face_user_contrast_similar.py

@@ -13,6 +13,7 @@ manager = lambda: models.Manager().db_manager(using='face')
 
 class FaceUserContrastSimilar(models.Model):
     class Meta:
+        app_label = "face"
         verbose_name = u"用户脸相似表"
         db_table = "face_user_contrast_similar"
 
@@ -25,3 +26,4 @@ class FaceUserContrastSimilar(models.Model):
     similarity = models.FloatField(verbose_name=u'相似度', default=0)
     create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+    contrast_user_id = models.IntegerField(verbose_name=u'对照者用户ID')