rpcd tracer adapter

.gitignore

@@ -257,3 +257,4 @@ fabric.properties
 # End of https://www.gitignore.io/api/python,django,pycharm
 
 .idea/*
+courier
\ No newline at end of file

Dockerfile

@@ -11,8 +11,6 @@ RUN pip install -r requirements.txt
 
 COPY . .
 
-ENV GM_RPCD_MODE="deploy"
+ENV GM_RPCD_MODE "deploy"
 ENV GM_RPCD_DEPLOY_CONF_PATH "/srv/apps/courier/deploy.conf"
 CMD gunicorn courier.wsgi:application -c deploy/gunicorn.py
-
-

README.md

+#### Install
 
+```sh
+# ready file: /etc/gm-config/storage/static.dir/helios.dir/static_route_table.json
+mkdir -p /data/log/courier/app
+# hosts文件或者添加courier/settings.py的同级文件settings_local.py覆盖DATABASE和CACHES
+vim /etc/hosts
+    xxxx mysql-server
+    xxxx redis-server
+pip install -r requirements.txt
+``` 
 
-###### python manage.py runserver
-export GM_RPCD_MODE=deploy;export GM_RPCD_DEPLOY_CONF_PATH=`pwd`/deploy.conf; python manage.py runserver
+#### Run
 
+##### python manage.py runserver
+```sh
+GM_RPCD_MODE=deploy GM_RPCD_DEPLOY_CONF_PATH=`pwd`/deploy.conf python manage.py runserver
+```
 
-###### gunicorn
-export GM_RPCD_MODE=deploy;export GM_RPCD_DEPLOY_CONF_PATH=`pwd`/deploy.conf; gunicorn courier.wsgi:application -c deploy/gunicorn.py
+##### gunicorn
+```sh
+GM_RPCD_MODE=deploy GM_RPCD_DEPLOY_CONF_PATH=`pwd`/deploy.conf gunicorn courier.wsgi:application -c deploy/gunicorn.py
+```
 

adapter/rpcd.py → adapter/rpcd/__init__.py

 # -*- coding: utf-8 -*-
-import json
+import logging
 import os
 import uuid
 
 from django.conf import settings
-from django.views import View
-from django.http.response import HttpResponse
 
 from gm_rpcd.internals.configuration.model import Config, environ, from_property
-from gm_rpcd.internals.configuration.model_base import literal, xml_text, xml_text_list
-from gm_rpcd.internals.protocol.request import make_request_from_v1
-from gm_rpcd.internals.protocol.response import response_to_v1_json_value
-from gm_rpcd.internals.utils import once
+from gm_rpcd.internals.configuration.model_base import literal, xml_text, xml_text_list, \
+    DefaultConfigPropertyWrapper, EnvironmentConfigProperty
+
+
+GM_RPCD_APP_CONF_PATH_KEY = 'GM_RPCD_APP_CONF_PATH'
+GM_RPCD_DEVELOP_CONF_PATH_KEY = 'GM_RPCD_DEVELOP_CONF_PATH'
+GM_RPCD_DEPLOY_CONF_PATH_KEY = 'GM_RPCD_DEPLOY_CONF_PATH'
+GM_RPCD_MODE_KEY = 'GM_RPCD_MODE'
 
-try:
-    from django.db import close_old_connections
-except ImportError:
-    close_old_connections = None
 
 class DjangoRpcdConfig(Config):
+    '''因为 最终会从实例的wrapped属性解析出配置， 而wrapped实际上读取的是Config实例
+    的`.__class__.__dict__.`,因此父类中的类成员无法继承, 所以父类中的类成员都要重写'''
     process_unique_id = literal(uuid.uuid4().hex)
-    app_conf_path = environ('GM_RPCD_APP_CONF_PATH').default(os.path.abspath('app_conf.xml'))
-    deploy_conf_path = environ('GM_RPCD_DEPLOY_CONF_PATH')
 
-    mode = environ('GM_RPCD_MODE')
-    is_develop_mode = mode.eq('develop')
-    is_deploy_mode = mode.eq('deploy')
+    app_conf_path = DefaultConfigPropertyWrapper(gm_rpcd_app_path) \
+        if (gm_rpcd_app_path := getattr(settings, GM_RPCD_APP_CONF_PATH_KEY, None)) \
+        else environ('GM_RPCD_APP_CONF_PATH').default(os.path.abspath('app_conf.xml'))
+
+    develop_conf_path = DefaultConfigPropertyWrapper(gm_develop_conf_path) \
+            if (gm_develop_conf_path := getattr(settings, GM_RPCD_DEVELOP_CONF_PATH_KEY, None)) \
+            else environ('GM_RPCD_DEVELOP_CONF_PATH').default( os.path.abspath('.gm_rpcd.develop_conf.xml'))
+
+    deploy_conf_path = EnvironmentConfigProperty(gm_deploy_conf_path) \
+            if (gm_deploy_conf_path := getattr(settings, GM_RPCD_DEPLOY_CONF_PATH_KEY, None)) \
+            else environ('GM_RPCD_DEPLOY_CONF_PATH')
+
+    # mode is instance of  `ConfigProperty('develop')` or `ConfigProperty('deploy')`
+    mode = EnvironmentConfigProperty(gm_rpcd_mode) \
+            if (gm_rpcd_mode := getattr(settings, GM_RPCD_MODE_KEY, None)) \
+            else environ('GM_RPCD_MODE')
+    is_develop_mode, is_deploy_mode  = mode.eq('develop'), mode.eq('deploy')
 
     application_name = xml_text('app', 'application_name')
     service_list = xml_text_list('app', 'service_list').default_func(list)
@@ -36,8 +48,8 @@ class DjangoRpcdConfig(Config):
     # set, use default one GMRpcdRequestInfoExtractor
     request_info_extractor_cls = xml_text('app', 'request_info_extractor').default(None)
 
-    log_dir_for_develop = xml_text('develop', 'log_dir').default(
-        os.path.abspath('.gm_rpcd.log'))
+    log_dir_for_develop = xml_text('develop', 'log_dir').\
+        default(os.path.abspath('.gm_rpcd.log'))
     log_dir_for_deploy = xml_text('deploy', 'log_dir')
 
     sentry_dsn_for_deploy = xml_text('deploy', 'sentry_dsn')
@@ -63,34 +75,37 @@ class DjangoRpcdConfig(Config):
     def local_request_error_log_path(self):
         return os.path.join(self.log_dir, 'gm_rpcd_request_error.log')
 
-######################## PATCH ##########################
-from gm_rpcd.internals.configuration import model
-model.__dict__['config'] = DjangoRpcdConfig().wrapped
 
-from gm_rpcd.internals import configuration
-configuration.__dict__['config'] = model.__dict__['config']
-# ###################### end PATCH ##########################
+def setup_rpcd():
+    logger = logging.getLogger('django.start')
+    logger.info('****** setup rpcd ******')
+    # call `initialize_app` of rpcd
+    config_wrapper = DjangoRpcdConfig().wrapped
+    config_wrapper.is_deploy_mode = True
+    config_wrapper.freeze()
 
-class V1Batch(View):
+    ######################## PATCH #############################
+    from gm_rpcd.internals.configuration import model
+    model.__dict__['config'] = config_wrapper
+
+    from gm_rpcd.internals import configuration
+    configuration.__dict__['config'] = config_wrapper
+    ####################### end PATCH ##########################
 
-    def post(self, request, *args, **kwargs):
     from gm_rpcd.internals.initializations import initialize
+    # expose to module scope to make dispatcher singleton
+    global DISPATCHER
     DISPATCHER = initialize().dispatcher
 
-        requests_json = request.POST.get('requests')
-        requests_value = json.loads(requests_json)
-
-        response_v1_list = []
-        for request_v1_value in requests_value:
-            request = make_request_from_v1(
-                method=request_v1_value['method'],
-                params=request_v1_value['params'],
-                session_key=request_v1_value.get('session_key'),
-                environment=request_v1_value.get('environment'),
-            )
-            response = DISPATCHER.process_single_request(request)
-            response_v1_list.append(response_to_v1_json_value(response))
+    # avoid call `setup_rpcd` repeatly
+    global setup_rpcd
+    del setup_rpcd
+
+
+setup_rpcd()
+
+
+__all__ = [
+    'DISPATCHER',
+    'views'
+]
\ No newline at end of file
-
-        if close_old_connections:
-            close_old_connections()
-        return HttpResponse(json.dumps(response_v1_list), {'content_type': 'application/json'})
\ No newline at end of file

adapter/rpcd/views.py

+# -*- coding: utf-8 -*-
+import json
+
+from django.views import View
+from django.http.response import HttpResponse, JsonResponse
+
+from gm_rpcd.internals.protocol.request import make_request_from_v1
+from gm_rpcd.internals.protocol.response import response_to_v1_json_value
+from gm_rpcd.version import VERSION
+
+from . import DISPATCHER
+
+try:
+    from django.db import close_old_connections
+except ImportError:
+    close_old_connections = None
+
+# The followings are Django class-based views for gm_rpcd
+
+class PingView(View):
+    def post(self, request, *args, **kwargs):
+        return HttpResponse('pong')
+
+
+class ServerVersionView(View):
+    def get(self, request, *args, **kwargs):
+        return JsonResponse({
+            'name': 'gm-rpcd',
+            'version': VERSION
+        })
+
+
+class SystemStatusesView(View):
+    def get(self, request, *args, **kwargs):
+        data = DISPATCHER.system_statuses(request)
+        if close_old_connections:
+            close_old_connections()
+        return JsonResponse(data)
+
+
+class V1OnceView(View):
+    def post(self, request, *args, **kwargs):
+        requests_dict = json.loads(request.POST.get('requests'))
+
+        method = requests_dict.get('method')
+        params = requests_dict.get('params')
+        session_key = requests_dict.get('session_key')
+        environment = requests_dict.get('environment')
+
+        request = make_request_from_v1(
+            method=method,
+            params=params,
+            session_key=session_key,
+            environment=environment,
+        )
+        response = DISPATCHER.process_single_request(request)
+        response_value = response_to_v1_json_value(response)
+        if close_old_connections:
+            close_old_connections()
+
+        return HttpResponse(json.dumps(response_value), {'content_type': 'application/json'})
+
+
+class V1BatchView(View):
+    def post(self, request, *args, **kwargs):
+        requests_json = request.POST.get('requests')
+        requests_value = json.loads(requests_json)
+        response_v1_list = []
+        for request_v1_value in requests_value:
+            request = make_request_from_v1(
+                method=request_v1_value['method'],
+                params=request_v1_value['params'],
+                session_key=request_v1_value.get('session_key'),
+                environment=request_v1_value.get('environment'),
+            )
+            # print(DISPATCHER._method_table, DISPATCHER._method_table._MethodTable__method_map)
+            response = DISPATCHER.process_single_request(request)
+            response_v1_list.append(response_to_v1_json_value(response))
+
+        if close_old_connections:
+            close_old_connections()
+        return HttpResponse(json.dumps(response_v1_list), {'content_type': 'application/json'})

adapter/tracer/__init__.py

+# -*- coding: utf-8 -*-
+import json
+
+
+PROJECT_NAME = 'courier'
+
+def setup_tracer(request):
+    try:
+        seqs = json.loads(request.POST['requests'])
+        tracer = seqs[0]['environment'].get('trace_context')
+    except (KeyError, IndexError, AttributeError, ValueError):
+        tracer = None
+
+    if not tracer:
+        return {
+            'service_name': PROJECT_NAME,
+            'sampled': False,
+            'span_name': '*'
+        }
+    return {
+        'service_name': PROJECT_NAME,
+        'sampled': tracer['sampled'],
+        'debug': tracer.get('debug', False),
+        'trace_id': tracer['trace_id'],
+        'span_id': tracer['span_id'],
+        'span_name': seqs[0]['method'],
+    }

adapter/tracer/middleware.py

+# -*- coding: utf-8 -*-
+import six
+from gm_tracer import middleware
+from gm_tracer.context import current_tracer
+from gm_tracer.contrib.django import unpatch
+from gm_tracer.middleware import patch_packages
+
+
+class TracerMiddleware(middleware.TracerMiddleware):
+    def process_response(self, request, response):
+        tracer = current_tracer()
+        tracer.add_binary_annotation({
+            'span.type': 'rpc',
+            'http.code': six.text_type(response.status_code),
+        })
+
+        tracer.stop()
+        if tracer.need_report():
+            record = tracer.convert_to_pb_repr()
+            self.logger(record)
+
+        if tracer.debug:
+            unpatch(**patch_packages)
+        # purge_current_tracer()
+
+        return response

app_conf.xml

 <?xml version="1.0" encoding="utf-8"?>
 <gm_rpcd_config>
 	<info config_name="app" version="1.0"/>
-	<config name="application_name" value="poll"/>
+	<config name="application_name" value="courier"/>
 	<config name="service_list">
 		<element value="courier"/>
 	</config>

courier/management_interface/initialize.py

 def initialize():
-    pass
+    # fire all binds
+    # See: gm_rpcd.internals.initializations.initialize:52
+    from message import views

courier/settings.py

@@ -40,6 +40,7 @@ INSTALLED_APPS = [
 ]
 
 MIDDLEWARE = [
+    'gm_tracer.middleware.TracerMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -47,6 +48,7 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'adapter.tracer.middleware.TracerMiddleware',
 ]
 
 ROOT_URLCONF = 'courier.urls'
@@ -75,9 +77,17 @@ WSGI_APPLICATION = 'courier.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    }
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'zhengxing_test',
+        'USER': 'work',
+        'PASSWORD': 'Gengmei1',
+        'HOST': 'mysql-server',
+        'PORT': '3306',
+        'OPTIONS': {
+            "init_command": "SET foreign_key_checks = 0;",
+            "charset": "utf8mb4",
+        },
+    },
 }
 
 
@@ -205,3 +215,10 @@ if DEBUG:
             'level': 'DEBUG',
         }
     }),
+
+GMTRACER_PARSER = 'adapter.tracer.setup_tracer'
+
+try:
+    from courier.settings_local import *
+except ModuleNotFoundError:
+    pass

courier/urls.py

@@ -14,10 +14,15 @@ Including another URLconf
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path, re_path
-from adapter.rpcd import V1Batch
+from django.urls import path
+from adapter.rpcd import views
 
 urlpatterns = [
+    path('ping', views.PingView.as_view()),
+    path('version', views.ServerVersionView.as_view()),
+    path('system/statuses', views.SystemStatusesView.as_view()),
+    path('v1/once', views.V1OnceView.as_view()),
+    path('v1/batch', views.V1BatchView.as_view()),
+
     path('admin/', admin.site.urls),
-    path('api/batch', V1Batch.as_view()),
 ]

docker-compose.yml

+version: '2'
+services:
+  courier:
+    build: .
+    container_name: courier
+    ports:
+      - "8000:8000"
+    volumes:
+      - "$PWD:/srv/apps/courier"
+    tty: true
+    command: /bin/sh
\ No newline at end of file

message/__init__.py

+# -*- coding: utf-8 -*-

message/views.py

+# -*- coding: utf-8 -*-
+from gm_rpcd.all import bind
+
+@bind('courier/message/conversation/list')
+def conversation_list():
+    return {}
+
+

requirements.txt

@@ -7,6 +7,11 @@ git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git
 #git+ssh://git@git.wanmeizhensuo.com/system/kafka-python.git
 
 Django==3.0.1
+PyMySQL==0.9.3
+mysqlclient==1.4.6
+redis==3.3.11
+django-redis==4.11.0
 raven==6.10.0
 kafka-python==1.4.7
 gunicorn==20.0.4
+djangorestframework==3.11.0
\ No newline at end of file