Merge branch 'master' into 'test'

# Conflicts:
#   trans2es/utils/hospital_transfer.py

_celery.py

+# !/usr/bin/env python
+# encoding=utf-8
+from __future__ import absolute_import
+
+import os
+# set the default Django settings module for the 'celery' program.
+# os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'search_tips.settings')
+
+import raven
+from raven.contrib.celery import register_signal, register_logger_signal
+from celery import Celery
+from django.conf import settings
+
+
+class Celery(Celery):
+    """wrap for celery.Celery."""
+
+    def on_configure(self):
+        # check if sentry settings provided
+        if not settings.SENTRY_CELERY_ENDPOINT:
+            return
+
+        client = raven.Client(settings.SENTRY_CELERY_ENDPOINT)
+
+        # register a custom filter to filter out duplicate logs
+        register_logger_signal(client)
+
+        # hook into the Celery error handler
+        register_signal(client)
+
+
+app = Celery('search_tips_tasks')
+
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+

injection/admin.py

+from django.contrib import admin
+
+# Register your models here.

injection/data_sync/__init__.py

+default_app_config = 'injection.data_sync.apps.DataSyncApp'

injection/data_sync/apps.py

+# coding=utf-8
+from __future__ import unicode_literals, print_function, absolute_import
+
+
+from django.apps import AppConfig
+
+
+class DataSyncApp(AppConfig):
+
+    name = 'injection.data_sync'
+    label = 'injected_data_sync'
+

injection/data_sync/tasks.py

+# -*- coding: UTF-8 -*-
+import logging
+import traceback
+import json
+import pymysql
+import threading
+import random
+import datetime
+from celery import shared_task
+from django.conf import settings
+from django.core import serializers
+from trans2es.type_info import get_type_info_map,TypeInfo
+# from rpc.all import get_rpc_remote_invoker
+from libs.es import ESPerform
+from libs.cache import redis_client
+
+
+@shared_task
+def write_to_es(es_type, pk_list, use_batch_query_set=False):
+    try:
+        pk_list = list(frozenset(pk_list))
+        type_info_map = get_type_info_map()
+        type_info = type_info_map[es_type]
+
+        logging.info("consume es_type:%s" % str(es_type))
+        type_info.insert_table_by_pk_list(
+            sub_index_name=type_info.name,
+            pk_list=pk_list,
+            use_batch_query_set=use_batch_query_set,
+            es=ESPerform.get_cli()
+        )
+
+    except:
+        logging.error("catch exception,err_msg:%s" % traceback.format_exc())
\ No newline at end of file

injection/models.py

+from django.db import models
+
+# Create your models here.

injection/tests.py

+from django.test import TestCase
+
+# Create your tests here.

injection/views.py

+from django.shortcuts import render
+
+# Create your views here.

libs/es.py

@@ -141,7 +141,7 @@ class ESPerform(object):
         try:
             assert (es_cli is not None)
 
-            official_index_name = cls.get_official_index_name(sub_index_name)
+            official_index_name = cls.get_official_index_name(sub_index_name,"write")
             index_exists = es_cli.indices.exists(official_index_name)
             if not index_exists:
                 if not auto_create_index:

libs/tools.py

@@ -71,8 +71,8 @@ class HostpitalPosInfo(object):
     def get_hostpital_info(cls):
         try:
             if len(cls.hospital_pos_dict) == 0:
-                query = doctor.Hospital.objects.all().query
-                query_results = django.db.models.QuerySet(model=doctor.Hospital, query=query)
+                query_results = doctor.Hospital.objects.filter(is_online=True)
+                #query_results = django.db.models.QuerySet(model=doctor.Hospital, query=query)
                 for item in query_results:
                     name = item.name
                     lng = float(item.baidu_loc_lng)

search_tips/__init__.py

 from __future__ import unicode_literals, absolute_import, print_function
 
 import pymysql
+from _celery import app as celery_app
 
 pymysql.install_as_MySQLdb()
\ No newline at end of file

search_tips/celery.py

+from __future__ import absolute_import, unicode_literals
+import os
+from celery import Celery
+from django.conf import settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'search_tips.settings')
+
+app = Celery('search_tips')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+app.conf.broker_url = settings.CELERY_BROKER_URL
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))

search_tips/celery_task_router.py

+# coding=utf-8                                                                           
+from __future__ import unicode_literals, print_function, absolute_import
+from django.conf import settings
+import itertools
+import logging
+
+
+class CeleryTaskRouter(object):
+    queue_task_map = {
+        "tapir-search_tips": [
+            'injection.data_sync.tasks.write_to_es',
+        ]
+    }
+
+    # Map[TaskName, QueueName]
+    task_queue_map = dict(itertools.chain.from_iterable(
+        [(task, queue) for task in task_list]
+        for (queue, task_list) in queue_task_map.items()
+    ))
+
+    def route_for_task(self, task, args=None, kwargs=None):
+        """
+        if settings.DEBUG:
+            return None
+
+        if task.startswith("statistic") or task.startswith("api.tasks.export_excel_task"):
+            return "slow"
+        """
+
+        queue_name_or_none = self.task_queue_map.get(task)
+        return queue_name_or_none

search_tips/log_settings.py_bak

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+
+import os
+import logging
+
+LOG_DIR = '/data/log/search_tips/app/'
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d  %(message)s',
+    filename=os.path.join(LOG_DIR, 'filelog.log'),
+)
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+
+    'filters': {
+        'require_debug_true': {
+            '()': 'django.utils.log.RequireDebugTrue',
+        },
+    },
+
+    'formatters': {
+        'verbose': {
+            'format': '%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d  %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'profile': {
+            'format': '%(asctime)s %(message)s'
+        },
+        'raw': {
+            'format': '%(message)s'
+        }
+    },
+
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple'
+        },
+
+        # 默认的服务器Log(保存到log/filelog.log中, 通过linux的logrotate来处理日志的分割
+        'default': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'filelog.log'),
+            'formatter': 'verbose',
+        },
+
+        # 默认的服务器ERROR log
+        'default_err': {
+            'level': 'ERROR',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'error_logger.log'),
+            'formatter': 'verbose',
+        },
+        'exception_logger': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'exception_logger.log'),
+            'formatter': 'verbose',
+        },
+
+        'tracer_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'tracer.log'),
+            'formatter': 'raw'
+        },
+    },
+
+    'loggers': {
+        'django': {
+            'handlers': ['default'],
+            'propagate': True,
+            'level': 'INFO',
+        },
+        'django.request': {
+            'handlers': ['default_err'],
+            'level': 'ERROR',
+            'propagate': False,
+        },
+        'exception_logger': {
+            'handlers': ['exception_logger'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+
+        'gm_tracer.subscribe': {
+            'handlers': ['tracer_handler'],
+            'propagate': False,
+            'level': 'INFO'
+        },
+
+    },
+}
+
+

search_tips/settings.py

@@ -48,7 +48,7 @@ DEBUG = True
 # }
 
 CELERY_TIMEZONE = 'Asia/Shanghai'
-CELERY_ROUTES = ['physical.celery_task_router.CeleryTaskRouter']
+CELERY_ROUTES = ['search_tips.celery_task_router.CeleryTaskRouter']
 
 # Application definition
 
@@ -60,7 +60,8 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'trans2es',
-    'search'
+    'search',
+    'injection.data_sync'
 ]
 
 MIDDLEWARE = [

search_tips/settings_local.py_bak

+# -*- coding:utf-8 -*-
+
+
+ALLOWED_HOSTS = ["192.168.78.2"]
+
+SECRET_KEY = 'e%$v6snev0807=t0@gk_n2#r5m6r1h(eil6cp^y3ub@ja@gk_t'
+SENTRY_CELERY_ENDPOINT="http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
+
+BROKER_URL = "redis://127.0.0.1:6379/8"
+
+#REDIS_URL = "redis://127.0.0.1:6379"
+REDIS_URL = "redis://127.0.0.1:6379/1"
+
+CELERY_BROKER_URL = "redis://127.0.0.1:6379/8"
+
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'zhengxing_test',
+        'USER': 'work',
+        'PASSWORD': 'workwork',
+        'HOST': 'rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com',
+        'PORT': '',
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",
+        }
+    },
+    'bran': {
+        'ENGINE': 'django.db.backends.mysql',  # 设置为mysql数据库
+        'NAME': 'bran_test',
+        'USER': 'work',
+        'PASSWORD': 'workwork',
+        'HOST': 'rdsmaqevmuzj6jy.mysql.rds.aliyuncs.com',
+        'PORT': '3306',
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",  # 为了支持emoji表情
+        },
+    }
+}
+
+ES_INFO_LIST = [
+        {
+            "host": "10.29.130.141",
+            "port": 9200
+        }
+]
+
+GM_ORI_ES_INFO_LIST = [
+        {
+            "host": "10.29.130.141",
+            "port": 9200
+        }
+]
+
+ES_INDEX_PREFIX="gm-dbmw"
+
+

trans2es/models/doctor.py

@@ -23,6 +23,7 @@ class Hospital(models.Model):
 
     id = models.CharField(max_length=100, primary_key=True, default=uuid4)
     name = models.CharField(max_length=100, unique=True, help_text=u"医院名称")
+    is_online = models.BooleanField(default=False, help_text=u"是否上线", verbose_name=u"上线")
 
     google_loc_lng = models.FloatField(default=0, help_text=u"经度(-180~180)")
     google_loc_lat = models.FloatField(default=0, help_text=u"纬度")