diff --git a/injection/data_sync/tasks.py b/injection/data_sync/tasks.py index 2e0d416910ea2149ed8d73db042452dea01d9e4b..b29488e9c12febfcec621a9e60a20f1991c75901 100644 --- a/injection/data_sync/tasks.py +++ b/injection/data_sync/tasks.py @@ -4,7 +4,7 @@ from celery import shared_task from django.conf import settings from django.core import serializers from trans2es.type_info import get_type_info_map -#from rpc.all import get_rpc_remote_invoker +# from rpc.all import get_rpc_remote_invoker from libs.es import ESPerform import logging import traceback @@ -12,6 +12,7 @@ from libs.cache import redis_client from trans2es.models.face_user_contrast_similar import FaceUserContrastSimilar import json + @shared_task def write_to_es(es_type, pk_list, use_batch_query_set=False): try: @@ -33,7 +34,8 @@ def write_to_es(es_type, pk_list, use_batch_query_set=False): @shared_task def sync_face_similar_data_to_redis(): try: - result_items = FaceUserContrastSimilar.objects.filter(is_online=True,is_deleted=False).distinct().values("participant_user_id").values_list("participant_user_id",flat=True) + result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False).distinct().values( + "participant_user_id").values_list("participant_user_id", flat=True) logging.info("duan add,begin sync_face_similar_data_to_redis!") @@ -41,19 +43,21 @@ def sync_face_similar_data_to_redis(): for participant_user_id in result_items: redis_key = redis_key_prefix + str(participant_user_id) - similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True,is_deleted=False,participant_user_id=participant_user_id,similarity__gt=0.4).order_by("-similarity").limit(100) + similar_result_items = FaceUserContrastSimilar.objects.filter(is_online=True, is_deleted=False, + participant_user_id=participant_user_id, + similarity__gt=0.3).order_by( + "-similarity") item_list = list() for item in similar_result_items: item_list.append( { - "contrast_user_id":item.contrast_user_id, - "similarity":item.similarity + "contrast_user_id": item.contrast_user_id, + "similarity": item.similarity } ) - redis_client.set(redis_key,json.dumps(item_list)) + redis_client.set(redis_key, json.dumps(item_list)) logging.info("duan add,participant_user_id:%d set data done!" % participant_user_id) except: logging.error("catch exception,err_msg:%s" % traceback.format_exc()) - diff --git a/physical/celery_task_router.py b/physical/celery_task_router.py index 5549e74c714f4e4118f363421314b6d17759ce4e..bb99ece174c65720573838aae0fcd34c5d08ab91 100644 --- a/physical/celery_task_router.py +++ b/physical/celery_task_router.py @@ -1,14 +1,15 @@ # coding=utf-8 -from __future__ import unicode_literals, print_function, absolute_import - -import itertools - -from django.conf import settings +from __future__ import unicode_literals, print_function, absolute_import + +import itertools + +from django.conf import settings import logging + class CeleryTaskRouter(object): queue_task_map = { - "tapir-alpha":[ + "tapir-alpha": [ 'injection.data_sync.tasks.write_to_es', ] } @@ -30,4 +31,4 @@ class CeleryTaskRouter(object): logging.info("duan add,task is:%s" % str(task)) queue_name_or_none = self.task_queue_map.get(task) - return queue_name_or_none \ No newline at end of file + return queue_name_or_none diff --git a/physical/settings.py b/physical/settings.py index 3933016e9c10b95b78ebb45bd00d7f7d85f31c7d..c895e497b83d4297712f43c7e402648d5944232d 100644 --- a/physical/settings.py +++ b/physical/settings.py @@ -18,7 +18,6 @@ from celery.schedules import crontab # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ @@ -30,9 +29,8 @@ DEBUG = False ALLOWED_HOSTS = [] - # Application definition -SENTRY_CELERY_ENDPOINT="http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140" +SENTRY_CELERY_ENDPOINT = "http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140" BROKER_URL = "redis://127.0.0.1:6379/8" # CELERY_SEND_EVENTS = True @@ -64,15 +62,15 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'trans2es', - 'search', + 'trans2es', + 'search', 'injection.data_sync', ) CELERYBEAT_SCHEDULE = { 'sync_face_similar_data_to_redis': { 'task': 'injection.data_sync.tasks.sync_face_similar_data_to_redis', - 'schedule': timedelta(seconds=120), + 'schedule': 120.0, 'args': () }, } @@ -131,7 +129,9 @@ DATABASES = { 'NAME': 'alpha', 'USER': 'work', 'PASSWORD': 'Gengmei123', - 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com', + # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com', + 'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com', + 'PORT': '3306', 'OPTIONS': { "init_command": "SET foreign_key_checks = 0;", @@ -143,9 +143,10 @@ DATABASES = { 'NAME': 'face', 'USER': 'work', 'PASSWORD': 'Gengmei123', - 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com', + # 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com', + 'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com', 'PORT': '3306', - #'CONN_MAX_AGE': None, + # 'CONN_MAX_AGE': None, 'OPTIONS': { "init_command": "SET foreign_key_checks = 0;", "charset": "utf8mb4", @@ -154,13 +155,13 @@ DATABASES = { } ES_INFO_LIST = [ - { - "host": "10.29.130.141", - "port": 9200 - } + { + "host": "10.29.130.141", + "port": 9200 + } ] -ES_INDEX_PREFIX="gm-dbmw" +ES_INDEX_PREFIX = "gm-dbmw" MIDDLEWARE_CLASSES = ( 'gm_tracer.middleware.TracerMiddleware', diff --git a/trans2es/models/face_user_contrast_similar.py b/trans2es/models/face_user_contrast_similar.py index da58e04f816294481b1dbd9bfa175c94c8a6af83..faf0114f09c0a2142fd9d562da348ea48957ad30 100644 --- a/trans2es/models/face_user_contrast_similar.py +++ b/trans2es/models/face_user_contrast_similar.py @@ -24,3 +24,4 @@ class FaceUserContrastSimilar(models.Model): similarity = models.FloatField(verbose_name=u'相似度',default=0) create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0)) update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0)) + contrast_user_id = models.IntegerField(verbose_name=u'对照者用户ID')