add security monitor

adapter/log/__init__.py

+# -*- coding: utf-8 -*-
+import logging
+
+from gm_logging.utils import get_exception_logging_func
+from raven.contrib.django.raven_compat.models import client as _sentry_client
+
+
+info_logger = logging.getLogger('info_logger')
+exception_logger = logging.getLogger('exception_logger')
+
+logging_exception = get_exception_logging_func(exception_logger, _sentry_client)
\ No newline at end of file

adapter/old_system.py

+# -*- coding: utf-8 -*-
+from gm_rpcd.all import bind, context
+
+ROUTER_PREFIX = 'courier/'
+bind_prefix = lambda endpoint, **options: bind(ROUTER_PREFIX+endpoint, **options)

api/models/message.py

@@ -645,4 +645,26 @@ class Message(models.Model):
             text = u'[更美红包券 优惠多多]'
         else:
             text = u''
-        return text
\ No newline at end of file
+        return text
+
+
+class MessageBlackUser(models.Model):
+    class Meta:
+        verbose_name = u'消息用户黑名单'
+        verbose_name_plural = u'消息用户黑名单'
+        app_label = 'api'
+
+    user_id = models.IntegerField(verbose_name=u'用户')
+    content = models.TextField(verbose_name=u'那条消息发生的错误', max_length=1024, default='', blank=True)
+    created_time = models.DateTimeField(verbose_name=u'创建时间', default=timezone.now)
+
+    def __unicode__(self):
+        return u'用户【{}】于 {} 发送【{}】被限制'.format(self.user_id, self.created_time, self.content)
+
+    @property
+    def black_detail(self):
+        return {
+            'created_time': self.created_time.strftime('%Y-%m-%d'),
+            'content': self.content,
+            'user': self.user.last_name,
+        }
\ No newline at end of file

common/__init__.py

+# -*- coding: utf-8 -*-

common/types.py

+# -*- coding: utf-8 -*-
+from gm_types.hestia import MESSAGE_TYPE
+
+
+SPECIAL_MSG_TYPE = frozenset([MESSAGE_TYPE.SERVICE, MESSAGE_TYPE.DOCTOR_TOPIC, MESSAGE_TYPE.TEXT_WITH_URL,
+                              MESSAGE_TYPE.DIARY, MESSAGE_TYPE.GIFT])

communication/__init__.py

 # -*- coding: utf-8 -*-
+

courier/settings.py

@@ -79,11 +79,23 @@ WSGI_APPLICATION = 'courier.wsgi.application'
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
-        'NAME': 'zhengxing_test',
+        'NAME': 'whisper_test',
         'USER': 'work',
         'PASSWORD': 'Gengmei1',
-        'HOST': 'mysql-server',
-        'PORT': '3306',
+        'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
+        'PORT': '62120',
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",
+        },
+    },
+    'slave': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'whisper_test',
+        'USER': 'work',
+        'PASSWORD': 'Gengmei1',
+        'HOST': 'bj-cdb-6slgqwlc.sql.tencentcdb.com',
+        'PORT': '62120',
         'OPTIONS': {
             "init_command": "SET foreign_key_checks = 0;",
             "charset": "utf8mb4",
@@ -137,13 +149,13 @@ STATIC_URL = '/static/'
 #######################################################
 # https://docs.djangoproject.com/en/2.2/topics/logging/
 # django.utils.log:18,   django.conf.global_settings:559
-DJANGO_LOG_PATH = '/data/log/courier/app'
-if not os.path.exists(DJANGO_LOG_PATH):
+PROJECT_LOG_PATH = '/data/log/courier/app'
+if not os.path.exists(PROJECT_LOG_PATH):
     try:
-        os.makedirs(DJANGO_LOG_PATH, exist_ok=True)
+        os.makedirs(PROJECT_LOG_PATH, exist_ok=True)
     except PermissionError:
-        DJANGO_LOG_PATH = os.path.join(BASE_DIR, 'log')
-        os.makedirs(DJANGO_LOG_PATH, exist_ok=True)
+        PROJECT_LOG_PATH = os.path.join(BASE_DIR, 'log')
+        os.makedirs(PROJECT_LOG_PATH, exist_ok=True)
 
 LOGGING = {
     'version': 1,
@@ -157,11 +169,15 @@ LOGGING = {
         },
     },
     'formatters': {
-        "verbose": {
+        'verbose': {
             "format": "[%(asctime)-23s]:%(levelname)s:@%(pathname)s:%(lineno)d, "
                       "%(funcName)s: %(message)s",
             # "datefmt": "%Y-%m-%d %H:%M:%S.%f%z"
-        }
+        },
+        'raw': {
+            'format': '%(message)s',
+        },
+
     },
     'handlers': {
         'console': {
@@ -173,7 +189,7 @@ LOGGING = {
         'django': {
             'level': 'INFO',
             'class': 'logging.handlers.TimedRotatingFileHandler',
-            'filename': os.path.join(DJANGO_LOG_PATH, 'django.log'),
+            'filename': os.path.join(PROJECT_LOG_PATH, 'django.log'),
             'when': 'midnight',
             'interval': 1,
             'backupCount': 365,
@@ -182,12 +198,24 @@ LOGGING = {
         'default': {
             'level': 'DEBUG',
             'class': 'logging.handlers.TimedRotatingFileHandler',
-            'filename': os.path.join(DJANGO_LOG_PATH, 'default.log'),
+            'filename': os.path.join(PROJECT_LOG_PATH, 'default.log'),
             'when': 'midnight',
             'interval': 1,
             'backupCount': 365,
             'formatter': 'verbose'
-        }
+        },
+        'exception_logger_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(PROJECT_LOG_PATH, 'exception_logger.log'),
+            'formatter': 'raw',
+        },
+        'info_logger_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(PROJECT_LOG_PATH, 'info_logger.log'),
+            'formatter': 'verbose',
+        },
     },
     'loggers': {
         '': {
@@ -204,6 +232,17 @@ LOGGING = {
             'level': 'DEBUG' if DEBUG else 'WARNING',
             'propagate': False
         },
+        'exception_logger': {
+            'handlers': ['exception_logger_handler'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'info_logger': {
+            'handlers': ['info_logger_handler'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+
         # debug
         'kafka': {
             'level': 'FATAL'
@@ -234,5 +273,5 @@ except ModuleNotFoundError:
 COUNT_LIMIT = 100
 ES_SEARCH_TIMEOUT = '10s'
 DATABASE_ROUTERS = ['courier.db_routers.MessageRouter']
-MESSAGE_DB_NAME = 'message'
+MESSAGE_DB_NAME = 'default'
 

deploy/gunicorn.py

 import multiprocessing
 
 workers = multiprocessing.cpu_count() + 1
-bind = '0.0.0.0:8005'
+bind = '0.0.0.0:8000'
 proc_name = 'courier'
 timeout = 3600
 preload_app = True

extension/__init__.py

+# -*- coding: utf-8 -*-

extension/types.py

+# -*- coding: utf-8 -*-
+from gm_types.utils.enum import Enum, unique
+
+
+class CourierError(int, Enum):
+    pass
+

message/__init__.py

 # -*- coding: utf-8 -*-
-
 from . import signal_handlers # fire sigal regist

message/views.py

@@ -4,19 +4,16 @@ from typing import List, Dict, Optional
 from six import string_types
 import dateutil
 from django.db.models import Q
-from gm_rpcd.all import bind, context
 from gm_types.gaia import MESSAGE_ORDER_TYPE
 from gm_types.msg import CONVERSATION_TYPE, CONVERSATION_ORDER
 
+from adapter.old_system import bind_prefix
 from adapter.rpcd.exceptions import RPCPermanentError
 from api.models.message import ConversationUserStatus
 from rpc import gaia_client
 from search.utils import search_conversation_from_es
 from services.unread.stat import UserUnread
 
-ROUTER_PREFIX = 'courier/'
-
-bind_prefix = lambda endpoint, **options: bind(ROUTER_PREFIX+endpoint, **options)
 
 @bind_prefix('message/conversation/list_v2')
 def batch_get_conversations_v2(user_ids: List[int],
@@ -139,6 +136,7 @@ def batch_get_conversations_v2(user_ids: List[int],
         'total_count': total_count,
     }
 
+
 @bind_prefix('message/conversation/list_v3')
 def message_conversation_list_v3(user_ids: List[int],
                                  offset: int,
@@ -150,7 +148,7 @@ def message_conversation_list_v3(user_ids: List[int],
                                  is_star: Optional[bool]=None,
                                  user_id: Optional[bool]=None,
                                  user_last_name: Optional[bool]=None,
-                                 comment: Optional[bool]=None) -> dict:
+                                 comment: Optional[bool]=None) -> Dict:
     """
     获取会话列表， 与v2不同的是全部走es获取conversation_ids
     :param user_ids: LIST[USER_ID]
@@ -204,8 +202,6 @@ def message_conversation_list_v3(user_ids: List[int],
     }
     es_result_not_reply = search_conversation_from_es(offset=offset, size=size, filters=es_filters,
                                       query=es_query, sort_type=es_sort_type)
-    # es_result = search_conversation_from_es(offset=offset, size=size, filters=es_filters,
-    #                                         query=es_query, sort_type=es_sort_type)
     if reply_status == None:
         es_result = es_result_total
     else:
@@ -234,6 +230,5 @@ def message_conversation_list_v3(user_ids: List[int],
         'reply_total': es_result_reply['total_count'],
         'not_reply_total': es_result_not_reply['total_count']
     }
-    pass
 
 

monitor/__init__.py

+# -*- coding: utf-8 -*-

monitor/views.py

+# -*- coding: utf-8 -*-
+from gm_types.gaia import DOCTOR_TYPE
+
+from adapter.old_system import bind_prefix
+from api.models.message import Message, ConversationUserStatus
+from common.types import SPECIAL_MSG_TYPE
+from rpc.gaia_client import batch_get_doctor_info_by_user_ids
+
+
+@bind_prefix('message/message/message_monitor')
+def message_monitor(ctx, msg_ids):
+    """ 异常私信监控对backend暴露接口 """
+    ret = []
+    user_id_list = []
+
+    messages = Message.objects.filter(id__in=msg_ids)
+    for message in messages:
+        if message.type in SPECIAL_MSG_TYPE:
+           content = Message.get_special_msg_content(message.type, message.body)
+        else:
+           content = Message.get_general_msg_content(message.type, message.content)
+
+        cs = ConversationUserStatus.objects.filter(
+            conversation_id=message.conversation_id
+        ).exclude(user_id=message.user_id).first()
+        if not cs:
+            continue
+        target_uid = cs.user_id
+
+        user_id_list.append(target_uid)
+
+        ret.append({
+            "send_uid": message.user_id,
+            "content": content,
+            "send_time": message.send_time.strftime('%Y-%m-%d %H:%M:%S'),
+            "conversation_id": message.conversation_id,
+            'target_uid': target_uid,
+        })
+
+    user_id_doctor_info_dict = batch_get_doctor_info_by_user_ids(user_ids=user_id_list)
+    for i in ret:
+        doctor_info = user_id_doctor_info_dict.get(str(i.get('target_uid')))
+        if not doctor_info:
+            i.update(doctor_name='',
+                     doctor_id='',
+                     hospital_name='')
+        else:
+            doctor_name = doctor_info.get('meta', {}).get('name', '')
+            if doctor_name:
+                doctor_type = doctor_info.get('meta', {}).get('doctor_type')
+                if doctor_type != DOCTOR_TYPE.OFFICER:
+                    doctor_name = '{} 医生'.format(doctor_name)
+
+            i.update(
+                doctor_name=doctor_name,
+                doctor_id=doctor_info.get('meta', {}).get('id', ''),
+                hospital_name=doctor_info.get('meta', {}).get('hospital_name', '')
+            )
+
+    return ret
\ No newline at end of file

rpc/__init__.py

 # -*- coding: utf-8 -*-
+import functools
+from typing import Union, Dict, List
+
+import helios
+from gm_types.error import ERROR
+
+from adapter.log import logging_exception
+
+
+def rpc_invoke_error_handle_decorator(invoke_func) -> (bool, Union[Dict, List]):
+    @functools.wraps(invoke_func)
+    def wrapper(*args, **kwargs):
+        try:
+            result = invoke_func(*args, **kwargs)
+        except helios.rpc.internal.exceptions.RPCFaultException as e:
+            # errors like lack of parameter, need logined .etc
+            # url = e.parse_url(e.rpcd_server.v1_batch_url, e.request.method)
+            # formated_message = e.kwargs.get('message', '')
+            # url, e.request.params, e.request.session_key,
+            logging_exception()
+            return False, {'code':e.error, 'msg': e.message}
+        except helios.rpc.internal.exceptions.RPCSystemException as e:
+            # errors like attribute errror, unpack dict error .etc
+            # url = e.parse_url(e.rpcd_server.v1_batch_url, e.request.method)
+            # formated_message = e.kwargs.get('message', '')
+            # print(formated_message) # Server Error: argument of type 'NoneType' is not iterable
+            # print(url, e.request.params, e.request.session_key)
+            logging_exception()
+            return False, {'code': ERROR.UNIVERSAL, 'msg': ERROR.getDesc(ERROR.UNIVERSAL)}
+        else:
+            return True, result
+
+    return wrapper
\ No newline at end of file

rpc/context.py

 import contextlib
-import logging
 import threading
 
 from django.contrib.auth.models import AnonymousUser
 from django.conf import settings
 from cached_property import cached_property
-from gm_logging.utils import get_exception_logging_func
 from helios.rpc import create_default_invoker
 
+from adapter.log import info_logger, logging_exception
 from adapter.rpcd.exceptions import RPCLoginRequiredException
 from . import auth
-# from .tool.log_tool import logging_exception, info_logger
-
-info_logger = logging.getLogger(__name__)
-exception_logger = logging.getLogger('exception_logger')
-from raven.contrib.django.raven_compat.models import client as _sentry_client
-logging_exception = get_exception_logging_func(exception_logger, _sentry_client)
 
 
 class Session(object):
@@ -45,11 +38,11 @@ class Session(object):
 
     @property
     def has_login(self):
-
         user = self.user
         info_logger.info(user)
         res = user.is_authenticated()
         return res
+
     def login_required(self):
         if not self.has_login:
             raise RPCLoginRequiredException

rpc/gaia_client.py

 # -*- coding: utf-8 -*-
-from typing import List, Dict
+from typing import List, Dict, Optional, Set
 
+from rpc import rpc_invoke_error_handle_decorator
 from rpc.context import get_rpc_remote_invoker
 
 
-def get_doctor_id_by_user_id(user_id: int) -> int:
-    result = get_rpc_remote_invoker()['gaia/get_doctor_id_by_user_id'](
+@rpc_invoke_error_handle_decorator
+def get_doctor_info(user_id: str) -> Dict:
+    return get_rpc_remote_invoker()['api/doctor'](
         user_id=user_id,
     ).unwrap()
-    return result.get('user_id')
+
+
+@rpc_invoke_error_handle_decorator
+def batch_get_doctor_info_by_doctor_ids(doctor_ids, with_fields=None) -> List[Dict]:
+    return get_rpc_remote_invoker()['api/batch_get_doctor_info_by_doctor_ids'](
+        doctor_ids=doctor_ids,
+        with_fields=with_fields
+    ).unwrap()
 
 
 def get_message_mark_user(user_id_list: List[int],
@@ -17,4 +26,26 @@ def get_message_mark_user(user_id_list: List[int],
     return get_rpc_remote_invoker()['gaia/courier/mark_user'](
         user_id_list=user_id_list,
         target_user_id_list=target_user_id_list
-    ).unwrap()
\ No newline at end of file
+    ).unwrap()
+
+
+@rpc_invoke_error_handle_decorator
+def get_toc_hospital_basic_info(hospital_id: str,
+                                with_fields: Optional[List[str]]=['meta']):
+    return get_rpc_remote_invoker()['api/get_toc_hospital_basic_info'](
+        hospital_id=hospital_id,
+        with_fields=with_fields
+    ).unwrap()
+
+
+def get_doctor_basic_info(doctor_ids: List[str]) -> List[Dict]:
+    return get_rpc_remote_invoker()['api/doctor/basic_info'](
+        doctor_ids=doctor_ids
+    ).unwrap()
+
+
+def batch_get_doctor_info_by_user_ids(user_ids: List[int]) -> Dict:
+    return get_rpc_remote_invoker()['api/batch_get_doctor_info_by_user_ids'](
+        user_ids=user_ids,
+        with_fields=['meta']
+    ).unwrap()

search/utils.py

@@ -8,12 +8,12 @@ from gm_types.msg import CONVERSATION_TYPE, CONVERSATION_ORDER
 limited_size = functools.partial(min, settings.COUNT_LIMIT)
 
 
-def search_conversation_from_es(offset=0,
-                                size=50,
-                                filters={},
-                                query={},
-                                sort_type=CONVERSATION_ORDER.LAST_REPLY_TIME)\
-        -> dict:
+def search_conversation_from_es(offset: int=0,
+                                size: int=50,
+                                filters: Dict={},
+                                query: Dict={},
+                                sort_type: CONVERSATION_ORDER=CONVERSATION_ORDER.LAST_REPLY_TIME)\
+        -> Dict:
     res = search_conversation_es(offset, size, filters, query, sort_type)
     conversation_ids = [int(s['_id']) for s in res['hits']['hits']]
     total_count = res['hits']['total']
@@ -22,11 +22,12 @@ def search_conversation_from_es(offset=0,
         'conversation_ids': conversation_ids,
     }
 
+
 def search_conversation_es(offset: int=0,
                            size: int=50,
                            filters: Dict={},
                            query: Dict={},
-                           sort_type:CONVERSATION_ORDER=CONVERSATION_ORDER.LAST_REPLY_TIME):
+                           sort_type: CONVERSATION_ORDER=CONVERSATION_ORDER.LAST_REPLY_TIME):
     size = limited_size(size)
 
     filter_element_list = []

security/__init__.py

+# -*- coding: utf-8 -*-

security/views.py

+# -*- coding: utf-8 -*-
+from adapter.old_system import bind_prefix
+from api.models.message import MessageBlackUser
+
+
+@bind_prefix('message/message/block_message')
+def block_message(ctx, user_id):
+    """ 手动拉黑私信功能 (内部使用) """
+    MessageBlackUser.objects.create(user_id=user_id)
+    return {"status": True}
\ No newline at end of file

services/unread/stat.py

@@ -5,7 +5,7 @@ import datetime
 from django.conf import settings
 
 # from api.models import Doctor, Order, RefundOrder, Reservation
-
+from django.core.cache import cache
 from rpc.cache import unread_cache
 # from rpc.tool.log_tool import doctor_unread_logger