[A]rpc adapter, logging system, deploy config files

.drone.yml

 kind: pipeline
-name: hermes
+name: courier
 
 steps:
 - name: build
@@ -8,7 +8,7 @@ steps:
     registry: ccr.ccs.tencentyun.com
     username: 100006655717
     password: 541882233Zy
-    repo: ccr.ccs.tencentyun.com/mike-test/hermes
+    repo: ccr.ccs.tencentyun.com/mike-test/courier
     tags:
     - latest
     # - v0.0.1-test

Dockerfile

-FROM ccr.ccs.tencentyun.com/gm-base/gm-alpine:v1.3
+FROM ccr.ccs.tencentyun.com/gm-base/gm-alpine:v1.3 as ssh-keys
 
-MAINTAINER Mike [zhangyu@igengmei.com]
+FROM python:3.8.1-alpine
 
-WORKDIR /srv/apps/hermes/
+WORKDIR /srv/apps/courier/
+COPY --from=ssh-keys /root/.ssh /root/.ssh
+RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
 
-COPY ./requirements.txt /tmp
+COPY ./requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . .
+
+ENV GM_RPCD_MODE="deploy"
+ENV GM_RPCD_DEPLOY_CONF_PATH "/srv/apps/courier/deploy.conf"
+CMD gunicorn courier.wsgi:application -c deploy/gunicorn.py
 
-RUN apk add --no-cache --virtual .build-deps  \
-                bzip2-dev \
-                coreutils \
-                dpkg-dev dpkg \
-                expat-dev \
-                findutils \
-                gcc \
-                gdbm-dev \
-                libc-dev \
-                libffi-dev \
-                libnsl-dev \
-                libressl-dev \
-                libtirpc-dev \
-                linux-headers \
-                make \
-                ncurses-dev \
-                pax-utils \
-                readline-dev \
-                sqlite-dev \
-                tcl-dev \
-                tk \
-                tk-dev \
-                xz-dev \
-                zlib-dev \
-                # 业务相关依赖和安装工具
-                linux-headers \
-                python3-dev \
-                librdkafka-dev \
-                mariadb-client \
-                mariadb-dev \
-                git \
-                openssh \
-                bash \
-                build-base \
-                \
-                # 取消ssh第一次链接的确认
-                && echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config \
-                && apk add --no-cache  mariadb-connector-c-dev libxml2-dev libxslt-dev librdkafka-dev \
-                && apk add --no-cache jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \
-                \
-                && git clone https://github.com/pyenv/pyenv.git ~/.pyenv \
-                && echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.ashrc \
-                && echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.ashrc \
-                && echo -e 'if command -v pyenv 1>/dev/null 2>&1; then\n  eval "$(pyenv init -)"\nfi' >> ~/.ashrc \
-                && source ~/.ashrc \
-                \
-                && git clone https://github.com/pyenv/pyenv-virtualenv.git $(pyenv root)/plugins/pyenv-virtualenv \
-                && echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.ashrc \
-                \
-                && source ~/.ashrc \
-                \
-                && pyenv install 3.8.1 \
-                && pyenv virtualenv 3.8.1 hermes \
-                && pyenv activate hermes \
-                \
-                && pip install --no-cache-dir -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com -r /tmp/requirements.txt \
-                && mkdir -p /data/log/hermes/apps
 
-COPY . /srv/apps/hermes/
-WORKDIR /srv/apps/hermes/
\ No newline at end of file

README.md

+
+
+###### python manage.py runserver
+export GM_RPCD_MODE=deploy;export GM_RPCD_DEPLOY_CONF_PATH=`pwd`/deploy.conf; python manage.py runserver
+
+
+###### gunicorn
+export GM_RPCD_MODE=deploy;export GM_RPCD_DEPLOY_CONF_PATH=`pwd`/deploy.conf; gunicorn courier.wsgi:application -c deploy/gunicorn.py
+

adapter/__init__.py

+# -*- coding: utf-8 -*-

adapter/rpcd.py

+# -*- coding: utf-8 -*-
+import json
+import os
+import uuid
+
+from django.conf import settings
+from django.views import View
+from django.http.response import HttpResponse
+
+from gm_rpcd.internals.configuration.model import Config, environ, from_property
+from gm_rpcd.internals.configuration.model_base import literal, xml_text, xml_text_list
+from gm_rpcd.internals.protocol.request import make_request_from_v1
+from gm_rpcd.internals.protocol.response import response_to_v1_json_value
+from gm_rpcd.internals.utils import once
+
+try:
+    from django.db import close_old_connections
+except ImportError:
+    close_old_connections = None
+
+class DjangoRpcdConfig(Config):
+    process_unique_id = literal(uuid.uuid4().hex)
+    app_conf_path = environ('GM_RPCD_APP_CONF_PATH').default(os.path.abspath('app_conf.xml'))
+    deploy_conf_path = environ('GM_RPCD_DEPLOY_CONF_PATH')
+
+    mode = environ('GM_RPCD_MODE')
+    is_develop_mode = mode.eq('develop')
+    is_deploy_mode = mode.eq('deploy')
+
+    application_name = xml_text('app', 'application_name')
+    service_list = xml_text_list('app', 'service_list').default_func(list)
+    initializer_list = xml_text_list('app', 'initializer_list').default_func(list)
+    statuses_func = xml_text('app', 'statuses')
+
+    # give app a chance for using customized request info extractor, if not
+    # set, use default one GMRpcdRequestInfoExtractor
+    request_info_extractor_cls = xml_text('app', 'request_info_extractor').default(None)
+
+    log_dir_for_develop = xml_text('develop', 'log_dir').default(
+        os.path.abspath('.gm_rpcd.log'))
+    log_dir_for_deploy = xml_text('deploy', 'log_dir')
+
+    sentry_dsn_for_deploy = xml_text('deploy', 'sentry_dsn')
+
+    log_encoding = literal('utf-8')
+
+    @from_property
+    @property
+    def log_dir(self):
+        if self.is_develop_mode:
+            return self.log_dir_for_develop
+        if self.is_deploy_mode:
+            return self.log_dir_for_deploy
+        raise AttributeError('invalid mode')
+
+    @from_property
+    @property
+    def local_request_log_path(self):
+        return os.path.join(self.log_dir, 'gm_rpcd_request.log')
+
+    @from_property
+    @property
+    def local_request_error_log_path(self):
+        return os.path.join(self.log_dir, 'gm_rpcd_request_error.log')
+
+######################## PATCH ##########################
+from gm_rpcd.internals.configuration import model
+model.__dict__['config'] = DjangoRpcdConfig().wrapped
+
+from gm_rpcd.internals import configuration
+configuration.__dict__['config'] = model.__dict__['config']
+# ###################### end PATCH ##########################
+
+class V1Batch(View):
+
+    def post(self, request, *args, **kwargs):
+        from gm_rpcd.internals.initializations import initialize
+        DISPATCHER = initialize().dispatcher
+
+        requests_json = request.POST.get('requests')
+        requests_value = json.loads(requests_json)
+
+        response_v1_list = []
+        for request_v1_value in requests_value:
+            request = make_request_from_v1(
+                method=request_v1_value['method'],
+                params=request_v1_value['params'],
+                session_key=request_v1_value.get('session_key'),
+                environment=request_v1_value.get('environment'),
+            )
+            response = DISPATCHER.process_single_request(request)
+            response_v1_list.append(response_to_v1_json_value(response))
+
+        if close_old_connections:
+            close_old_connections()
+        return HttpResponse(json.dumps(response_v1_list), {'content_type': 'application/json'})
\ No newline at end of file

app_conf.xml

+<?xml version="1.0" encoding="utf-8"?>
+<gm_rpcd_config>
+	<info config_name="app" version="1.0"/>
+	<config name="application_name" value="poll"/>
+	<config name="service_list">
+		<element value="courier"/>
+	</config>
+	<config name="initializer_list">
+		<element value="courier.management_interface.initialize:initialize"/>
+	</config>
+</gm_rpcd_config>

courier/asgi.py

+"""
+ASGI config for courier project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'courier.settings')
+
+application = get_asgi_application()

courier/management_interface/__init__.py

+# -*- coding: utf-8 -*-

courier/management_interface/initialize.py

+def initialize():
+    pass

courier/settings.py

+"""
+Django settings for courier project.
+
+Generated by 'django-admin startproject' using Django 3.0.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/3.0/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'd2#)2cd!efrj@to&45yn=+=82-b*&of3i*&fxgfq%v53h1&1qx'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    # 'django.contrib.sessions',
+    'django.contrib.messages',
+    # 'django.contrib.staticfiles',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    # 'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'courier.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'courier.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/3.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.0/howto/static-files/
+
+STATIC_URL = '/static/'
+
+
+
+#######################################################
+####################### LOGGING #######################
+#######################################################
+# https://docs.djangoproject.com/en/2.2/topics/logging/
+# django.utils.log:18,   django.conf.global_settings:559
+DJANGO_LOG_PATH = '/data/log/courier/app'
+if not os.path.exists(DJANGO_LOG_PATH):
+    try:
+        os.makedirs(DJANGO_LOG_PATH, exist_ok=True)
+    except PermissionError:
+        DJANGO_LOG_PATH = os.path.join(BASE_DIR, 'log')
+        os.makedirs(DJANGO_LOG_PATH, exist_ok=True)
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse',
+        },
+        'require_debug_true': {
+            '()': 'django.utils.log.RequireDebugTrue',
+        },
+    },
+    'formatters': {
+        "verbose": {
+            "format": "[%(asctime)-23s]:%(levelname)s:@%(pathname)s:%(lineno)d, "
+                      "%(funcName)s: %(message)s",
+            # "datefmt": "%Y-%m-%d %H:%M:%S.%f%z"
+        }
+    },
+    'handlers': {
+        'console': {
+            'level': 'INFO',
+            'filters': ['require_debug_true'],
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
+        },
+        'django': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': os.path.join(DJANGO_LOG_PATH, 'django.log'),
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 365,
+            'formatter': 'verbose'
+        },
+        'default': {
+            'level': 'DEBUG',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': os.path.join(DJANGO_LOG_PATH, 'default.log'),
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 365,
+            'formatter': 'verbose'
+        }
+    },
+    'loggers': {
+        '': {
+            'handlers': ['default', 'console'],
+            'level': 'DEBUG' if DEBUG else 'WARNING', # default level is warning
+        },
+        'django': {
+            'handlers': ['django', 'console'],
+            'level': 'DEBUG' if DEBUG else 'WARNING',
+            'propagate': False
+        },
+        'django.request': {
+            'handlers': ['django', 'console'],
+            'level': 'DEBUG' if DEBUG else 'WARNING',
+            'propagate': False
+        },
+    }
+}
+
+# optimize SQL
+if DEBUG:
+    LOGGING['loggers'].update({
+        'django.db.backends': {
+            'handlers': ['console'],
+            'propagate': True,
+            'level': 'DEBUG',
+        }
+    }),

courier/urls.py

+"""courier URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path, re_path
+from adapter.rpcd import V1Batch
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+    path('api/batch', V1Batch.as_view()),
+]

courier/wsgi.py

+"""
+WSGI config for courier project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'courier.settings')
+
+application = get_wsgi_application()

deploy.conf

+<?xml version="1.0" encoding="utf-8"?>
+<gm_rpcd_config>
+    <info config_name="deploy" version="1.0"/>
+    <config name="log_dir" value="/data/log/courier/app"/>
+</gm_rpcd_config>

deploy/gunicorn.py

+import multiprocessing
+
+workers = multiprocessing.cpu_count() + 1
+bind = '0.0.0.0:8000'
+proc_name = 'courier'
+#pidfile = '/var/courier/gunicorn.pid'
+timeout = 3600
+#accesslog = '/var/log/courier/access.log'
+#access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' # default
+#access_log_format = '"%({X-Real-IP}i)s" %(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
+#errorlog = '/var/log/courier/error.log'
+preload_app = True
+#worker_class = 'gevent'
+access_log_format = '%(t)s %(p)s %(h)s "%(r)s" %(s)s %(L)s %(b)s "%(f)s" "%(a)s"'

manage.py

+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'courier.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

requirements.txt

+git+ssh://git@git.wanmeizhensuo.com/backend/gm-logging.git
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git
+git+ssh://git@git.wanmeizhensuo.com/backend/helios.git
+git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git
+#git+ssh://git@git.wanmeizhensuo.com/system/kafka-python.git
+
+Django==3.0.1
+raven==6.10.0
+kafka-python==1.4.7
+gunicorn==20.0.4
\ No newline at end of file