Merge branch 'similar' into 'master'

Similar

See merge request alpha/physical!120

injection/data_sync/tasks.py

@@ -4,7 +4,7 @@ from celery import shared_task
 from django.conf import settings
 from django.core import serializers
 from trans2es.type_info import get_type_info_map
-#from rpc.all import get_rpc_remote_invoker
+# from rpc.all import get_rpc_remote_invoker
 from libs.es import ESPerform
 import logging
 import traceback
@@ -12,6 +12,7 @@ from libs.cache import redis_client
 from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar
 import json
 
+
 @shared_task
 def write_to_es(es_type, pk_list, use_batch_query_set=False):
     try:
@@ -33,7 +34,8 @@ def write_to_es(es_type, pk_list, use_batch_query_set=False):
 @shared_task
 def sync_face_similar_data_to_redis():
     try:
-        result_items = FaceUserContrastSimilar.objects.filter(is_online=True,is_deleted=False).distinct().values("participant_user_id").values_list("participant_user_id",flat=True)
+        result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False).distinct().values(
+            "participant_user_id").values_list("participant_user_id", flat=True)
 
         logging.info("duan add,begin sync_face_similar_data_to_redis!")
 
@@ -41,19 +43,21 @@ def sync_face_similar_data_to_redis():
         for participant_user_id in result_items:
 
             redis_key = redis_key_prefix + str(participant_user_id)
-            similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True,is_deleted=False,participant_user_id=participant_user_id,similarity__gt=0.4).order_by("-similarity").limit(100)
+            similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False,
+                                                                          participant_user_id=participant_user_id,
+                                                                          similarity__gt=0.3).order_by(
+                "-similarity")
 
             item_list = list()
             for item in similar_result_items:
                 item_list.append(
                     {
-                        "contrast_user_id":item.contrast_user_id,
-                        "similarity":item.similarity
+                        "contrast_user_id": item.contrast_user_id,
+                        "similarity": item.similarity
                     }
                 )
-            redis_client.set(redis_key,json.dumps(item_list))
+            redis_client.set(redis_key, json.dumps(item_list))
 
             logging.info("duan add,participant_user_id:%d set data done!" % participant_user_id)
     except:
         logging.error("catch exception,err_msg:%s" % traceback.format_exc())
-

physical/celery_task_router.py

 # coding=utf-8                                                                           
-from __future__ import unicode_literals, print_function, absolute_import                 
-                                                                                         
-import itertools                                                                         
-                                                                                         
-from django.conf import settings                                                         
+from __future__ import unicode_literals, print_function, absolute_import
+
+import itertools
+
+from django.conf import settings
 import logging
 
+
 class CeleryTaskRouter(object):
     queue_task_map = {
-        "tapir-alpha":[
+        "tapir-alpha": [
             'injection.data_sync.tasks.write_to_es',
         ]
     }
@@ -30,4 +31,4 @@ class CeleryTaskRouter(object):
 
         logging.info("duan add,task is:%s" % str(task))
         queue_name_or_none = self.task_queue_map.get(task)
-        return queue_name_or_none
\ No newline at end of file
+        return queue_name_or_none

physical/settings.py

@@ -18,7 +18,6 @@ from celery.schedules import crontab
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
 
@@ -30,9 +29,8 @@ DEBUG = False
 
 ALLOWED_HOSTS = []
 
-
 # Application definition
-SENTRY_CELERY_ENDPOINT="http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
+SENTRY_CELERY_ENDPOINT = "http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
 BROKER_URL = "redis://127.0.0.1:6379/8"
 
 # CELERY_SEND_EVENTS = True
@@ -64,15 +62,15 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-	'trans2es',
-	'search',
+    'trans2es',
+    'search',
     'injection.data_sync',
 )
 
 CELERYBEAT_SCHEDULE = {
     'sync_face_similar_data_to_redis': {
         'task': 'injection.data_sync.tasks.sync_face_similar_data_to_redis',
-        'schedule': timedelta(seconds=120),
+        'schedule': 120.0,
         'args': ()
     },
 }
@@ -131,7 +129,9 @@ DATABASES = {
         'NAME': 'alpha',
         'USER': 'work',
         'PASSWORD': 'Gengmei123',
-        'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
+
         'PORT': '3306',
         'OPTIONS': {
             "init_command": "SET foreign_key_checks = 0;",
@@ -143,9 +143,10 @@ DATABASES = {
         'NAME': 'face',
         'USER': 'work',
         'PASSWORD': 'Gengmei123',
-        'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
+        'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
         'PORT': '3306',
-        #'CONN_MAX_AGE': None,
+        # 'CONN_MAX_AGE': None,
         'OPTIONS': {
             "init_command": "SET foreign_key_checks = 0;",
             "charset": "utf8mb4",
@@ -154,13 +155,13 @@ DATABASES = {
 
 }
 ES_INFO_LIST = [
-        {
-            "host": "10.29.130.141",
-            "port": 9200
-        }
+    {
+        "host": "10.29.130.141",
+        "port": 9200
+    }
 ]
 
-ES_INDEX_PREFIX="gm-dbmw"
+ES_INDEX_PREFIX = "gm-dbmw"
 
 MIDDLEWARE_CLASSES = (
     'gm_tracer.middleware.TracerMiddleware',

trans2es/models/face_user_contrast_similar.py

@@ -24,3 +24,4 @@ class FaceUserContrastSimilar(models.Model):
     similarity = models.FloatField(verbose_name=u'相似度',default=0)
     create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0))
     update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
+    contrast_user_id = models.IntegerField(verbose_name=u'对照者用户ID')