Merge branch 'dev' into 'master'

Dev

See merge request !13

injection/data_sync/tasks.py

@@ -8,6 +8,9 @@ from trans2es.type_info import get_type_info_map
 from libs.es import ESPerform
 import logging
 import traceback
+from libs.cache import redis_client
+from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar
+import json
 
 @shared_task
 def write_to_es(es_type, pk_list, use_batch_query_set=False):
@@ -26,3 +29,31 @@ def write_to_es(es_type, pk_list, use_batch_query_set=False):
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
 
+
+@shared_task
+def sync_face_similar_data_to_redis():
+    try:
+        result_items = FaceUserContrastSimilar.objects.filter(is_online=True,is_deleted=False).distinct().values("participant_user_id").values_list("participant_user_id",flat=True)
+
+        logging.info("duan add,begin sync_face_similar_data_to_redis!")
+
+        redis_key_prefix = "physical:user_similar:participant_user_id:"
+        for participant_user_id in result_items:
+
+            redis_key = redis_key_prefix + str(participant_user_id)
+            similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True,is_deleted=False,participant_user_id=participant_user_id,similarity__gt=0.4).order_by("-similarity").limit(100)
+
+            item_list = list()
+            for item in similar_result_items:
+                item_list.append(
+                    {
+                        "contrast_user_id":item.contrast_user_id,
+                        "similarity":item.similarity
+                    }
+                )
+            redis_client.set(redis_key,json.dumps(item_list))
+
+            logging.info("duan add,participant_user_id:%d set data done!" % participant_user_id)
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
+

physical/settings.py

@@ -12,6 +12,8 @@ https://docs.djangoproject.com/en/1.10/ref/settings/
 
 import os
 from .log_settings import *
+from datetime import timedelta
+from celery.schedules import crontab
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -67,6 +69,13 @@ INSTALLED_APPS = (
     'injection.data_sync',
 )
 
+CELERYBEAT_SCHEDULE = {
+    'sync_face_similar_data_to_redis': {
+        'task': 'injection.data_sync.tasks.sync_face_similar_data_to_redis',
+        'schedule': timedelta(seconds=120),
+        'args': ()
+    },
+}
 """
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
@@ -124,12 +133,25 @@ DATABASES = {
         'PASSWORD': 'Gengmei123',
         'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
         'PORT': '3306',
-        'CONN_MAX_AGE': None,
         'OPTIONS': {
             "init_command": "SET foreign_key_checks = 0;",
-            "charset": "utf8mb4",  # 为?~F?~T??~L~Aemoji表?~C~E
+            "charset": "utf8mb4",
+        },
+    },
+    'face': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'face',
+        'USER': 'work',
+        'PASSWORD': 'Gengmei123',
+        'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        'PORT': '3306',
+        #'CONN_MAX_AGE': None,
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",
         },
     }
+
 }
 ES_INFO_LIST = [
         {

search/utils/pick.py

@@ -11,7 +11,7 @@ from libs.cache import redis_client
 class PickUtils(object):
 
     @classmethod
-    def refresh_redis_data(cls,have_read_celebrity_list,redis_cli,redis_key):
+    def refresh_redis_data(cls,have_read_celebrity_list,redis_cli,redis_key,if_expire=True):
         try:
             now = datetime.datetime.now()
 
@@ -20,7 +20,10 @@ class PickUtils(object):
 
             valid_seconds = int((end_time - now).seconds)
 
-            redis_cli.setex(redis_key,valid_seconds,json.dumps(have_read_celebrity_list))
+            if if_expire:
+                redis_cli.setex(redis_key,valid_seconds,json.dumps(have_read_celebrity_list))
+            else:
+                redis_cli.set(redis_key,json.dumps(have_read_celebrity_list))
 
             return True
         except:

search/utils/topic.py

@@ -175,15 +175,15 @@ class TopicUtils(object):
                         }
                     }
                 },
-                "score_mode": "sum",
-                "boost_mode": "sum",
+                "score_mode": "max",
+                "boost_mode": "replace",
                 "functions": functions_list
             }
 
             if query is not None:#搜索帖子
                 multi_fields = {
-                    'description': 2,
-                    'name': 4,
+                    'description': 200,
+                    'name': 400,
                 }
                 query_fields = ['^'.join((k, str(v))) for (k, v) in multi_fields.items()]
                 multi_match = {

search/views/pick.py

@@ -116,7 +116,7 @@ def pick_topic(user_id,pick_id,offset=0,size=10):
         if len(recommend_pick_topic_list)>0 and user_id!=-1:
             total_read_topic_ids = have_read_pick_topic_ids + recommend_pick_topic_list
             PickUtils.refresh_redis_data(have_read_celebrity_list=total_read_topic_ids,
-                                         redis_cli=redis_client,redis_key=redis_key)
+                                         redis_cli=redis_client,redis_key=redis_key,if_expire=False)
 
         return {"recommend_pick_topic_ids": recommend_pick_topic_list}
 

trans2es/models/face_user_contrast_similar.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+import traceback
+import logging
+from django.db import models
+import datetime
+
+manager = lambda: models.Manager().db_manager(using='face')
+
+class FaceUserContrastSimilar(models.Model):
+    class Meta:
+        verbose_name=u"用户脸相似表"
+        db_table="face_user_contrast_similar"
+
+    objects = manager()
+
+    id = models.IntegerField(verbose_name="主键ID",primary_key=True)
+    is_online = models.BooleanField(verbose_name="是否上线")
+    is_deleted = models.BooleanField(verbose_name="是否删除")
+    participant_user_id = models.IntegerField(verbose_name=u'参与者用户ID')
+    similarity = models.FloatField(verbose_name=u'相似度',default=0)
+    create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
+    update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))