爬虫数据入库

.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*~
+# C extensions
+*.so
+.vscode/*
+
+# Distribution / packaging
+.Python
+.vscode
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+.idea/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+
+# config
+fabfile.py
+settings.online.py
+/gaia/settings.py
+settings_local.py
+media/
+log/
+crawldata/
+conf/
+/static
+.vagrant/
+Vagrantfile
+
+*.DS_Store
+dump.rdb
+# .gitignore for yangchuncheng
+api/management/commands/ycc*
+settings_override*
+.script/
+.tmp.sql
+.env
+*.pem
+/gaia/hospital_list_settings.py
+coverage_html/
+gaia/rpcd.json
+*.swp
+
+dbmw_deploy/config.dir/
+sms_config.json
+app_conf.xml
\ No newline at end of file

README.md

+## alpha saturn
+
+python版本 3.6
+
+安装依赖
+
+	pip install -r requirements/dev.txt
+

_celery.py

+import os
+
+import raven
+from celery import Celery
+from django.conf import settings
+from raven.contrib.celery import register_signal, register_logger_signal
+
+# set the default Django settings module for the 'celery' program.
+# os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings.settings')
+
+
+class _Celery(Celery):
+    """wrap for celery.Celery."""
+
+    def on_configure(self):
+        # check if sentry settings provided
+        if not settings.SENTRY_CELERY_ENDPOINT:
+            return
+
+        client = raven.Client(settings.SENTRY_CELERY_ENDPOINT)
+
+        # register a custom filter to filter out duplicate logs
+        register_logger_signal(client)
+
+        # hook into the Celery error handler
+        register_signal(client)
+
+
+app = _Celery('saturn_tasks')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

api/__init__.py

+from .tasks import *
+from _celery import app as celery_app

api/apps.py

+
+from django.apps import AppConfig
+
+
+class SaturnConfig(AppConfig):
+    name = 'api'

api/tasks.py

+from celery import shared_task
+from engine.rpc import get_current_rpc_invoker

api/urls.py

+
+from django.conf.urls import url
+
+from .views import user
+from .views import topic
+
+urlpatterns = [
+    url(r'^v1/update_grasp_status$', user.UpdateGraspStatus.as_view(), name='update_grasp_status$'),
+    url(r'^v1/create_topic_for_batch$', topic.CreateTopicForBatch.as_view(), name='create_topic_for_batch'),
+]

api/views/topic.py

+import json
+
+
+from api.views.base_view import BaseView
+
+
+class CreateTopicForBatch(BaseView):
+    """
+    内部使用，批量建帖
+    """
+    def post(self, request):
+
+        user_id = request.POST.get("user_id", 0)
+        topic_list = json.loads(request.POST.get("topic_list", '[]'))
+
+        if not user_id:
+            return self.parameter_invalid_response()
+
+        if not topic_list:
+            return self.ok()
+
+        tag_names = []
+        for item in topic_list:
+            tag_names.extend(item.get("tags", []))
+            item["user_id"] = user_id
+
+        # 先创建标签
+        _tag_error, _tag_data = self.call_rpc(
+            "venus/community/tag/batch_create_tag_by_name",
+            tag_names=list(filter(None, set(tag_names)))
+        )
+        if _tag_error:
+            return self.error(_tag_error)
+
+        # 更新发帖
+        for item in topic_list:
+            item["tag_ids"] = [_tag_data.get(tag_name, 0) for tag_name in item.get("tags", [])]
+
+        create_err, result = self.call_rpc(
+            "venus/community/topic/batch_create_for_inner",
+            topic_list=topic_list
+        )
+        if create_err:
+            return self.error(create_err)
+
+        return self.ok(data=result)

api/views/user.py

+
+from api.views.base_view import BaseView
+
+
+class UpdateGraspStatus(BaseView):
+    """
+    更新用户爬取状态
+    """
+    def post(self, request):
+        user_id = int(request.POST.get("user_id", 0))
+        relation_account_id = request.POST.get("relation_account_id", "")
+        error, _data = self.call_rpc(
+            "venus/community/user/update_grasp_status",
+            user_id=user_id,
+            relation_account_id=relation_account_id
+        )
+        if error:
+            return self.error(error)
+
+        return self.ok(data=_data)

app_conf.xml.bak

+<?xml version="1.0" encoding="utf-8"?>
+<gm_rpcd_config>
+    <info config_name="app" version="1.0"/>
+    <config name="sentry_dsn" value="http://39f1236352c04a80826878f459ef1b05:333fbf94113c4879addaf656f233eb8e@sentry.igengmei.com/138"/>
+    <config name="application_name" value="earth"/>
+    <config name="service_list">
+        <element value="earth"/>
+    </config>
+    <config name="initializer_list">
+        <element value="earth.django_init"/>
+    </config>
+</gm_rpcd_config>

engine/__init__.py

+
+from django.conf import settings
+
+from gm_tracer.tracer import hit_for_sample
+
+
+def setup_tracer(request):
+    # 采样率默认1%
+    sample_rate = getattr(settings,'TRACE_SAMPLE_RATE', 1.0)
+    sampled = hit_for_sample(sample_rate)
+    return {
+        'service_name': 'backend',
+        'sampled': sampled,
+        'span_name': request.path
+    }

engine/cache.py

+import redis
+from django.conf import settings
+
+
+class RedisWithoutprefixProxy(object):
+    _hacked_methods = set([
+        'get', 'mget', 'hget', 'hgetall', 'rpop'
+    ])
+
+    def __getattribute__(self, name):
+        try:
+            return super(RedisWithoutprefixProxy, self).__getattribute__(name)
+        except AttributeError:
+            f = getattr(self.redis,  name)
+
+            if name in RedisWithoutprefixProxy._hacked_methods:
+                def wrapper(k, *args, **kwargs):
+                    data = f(k, *args, **kwargs)
+                    # bug fix for py35, json.loads does accept bytes!
+                    if type(data) == bytes:
+                        data = data.decode()
+                    return data
+                return wrapper
+            return f
+
+    def __init__(self, conf):
+        self.__pool = redis.ConnectionPool(**conf)
+        self.redis = redis.StrictRedis(connection_pool=self.__pool)

engine/exception_info.py

+
+import inspect
+import traceback
+import sys
+
+
+def get_exception_info():
+    '''
+    获取stacktrace
+    '''
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    exc = traceback.format_exception(exc_type, exc_value, exc_traceback)
+    info = {'exception': '%s' % exc}
+    return info
+
+
+def build_func_args_item(stack_info):
+    '''
+    创建单调函数调用记录
+    '''
+    func_name = stack_info[3]
+    args_info = inspect.getargvalues(stack_info[0])
+    frame = stack_info[0]
+    item = {'function': func_name,
+            'object': '%s' % frame.f_locals.get('self'),
+            'arguments': inspect.formatargvalues(args_info.args, args_info.varargs, args_info.keywords, args_info.locals)}
+    return item
+
+
+def get_func_args():
+    '''
+    获取外层函数调用参数，注意调用栈顺序，最多取最后3层调用
+    '''
+    stacks = inspect.stack()[:3]
+    result = []
+    for stack in stacks:
+        item = build_func_args_item(stack)
+        result.append(item)
+    return result

engine/logger.py

+
+import logging
+import traceback
+import inspect
+import json
+import sys
+
+import raven
+
+from django.conf import settings
+
+import gm_logging
+import gm_logging.utils
+from gm_rpcd.all import context
+from gm_rpcd.features.raven import get_raven_client
+from gm_rpcd.internals.dynamic_scopes import DynamicVariableUnboundError
+from gm_logging.utils import get_exception_logging_func
+
+from engine.network import get_client_ip
+
+
+info_logger = logging.getLogger('info_logger')
+error_logger = logging.getLogger('error_logger')
+profile_logger = logging.getLogger('profile_logger')
+exception_logger = logging.getLogger('exception_logger')
+
+cpc_click_consumer_logger = logging.getLogger('cpc_click_consumer_logger')
+auth_logger = logging.getLogger('auth_logger')
+wechat_logger = logging.getLogger('wechat_logger')
+
+def debug(view_func):
+    """将POST请求数据存到log
+
+    :param view_func:
+    :return:
+    """
+
+    def wrap(request):
+        info_logger.info(request.POST.dict())
+        return view_func(request)
+
+    return wrap
+
+
+def _build_json_format(result):
+    """
+    创建json格式输出
+    """
+    return json.dumps(result, ensure_ascii=True, indent=2)
+
+
+def _get_exception_info():
+    """
+    获取stacktrace
+    """
+    try:
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        exc = traceback.format_exception(exc_type, exc_value, exc_traceback)
+        info = {'exception': exc}
+        return info
+    except:
+        logging.error('_get_exception_info error')
+        return {}
+
+
+def _build_func_args_item(stack_info):
+    """
+    创建单调函数调用记录
+    """
+    try:
+        func_name = stack_info[3]
+        args_info = inspect.getargvalues(stack_info[0])
+        frame = stack_info[0]
+        item = {'function': func_name,
+                'object': '%s' % frame.f_locals.get('self'),
+                'arguments': inspect.formatargvalues(args_info.args, args_info.varargs, args_info.keywords,
+                                                     args_info.locals)}
+        return item
+    except:
+        logging.error('_build_func_args_item error')
+        return {}
+
+
+def _get_func_args():
+    """
+    获取外层函数调用参数，注意调用栈顺序，最多取最后3层调用
+    """
+    try:
+        stacks = inspect.stack()[:3]
+        result = []
+        for stack in stacks:
+            item = _build_func_args_item(stack)
+            result.append(item)
+        return result
+    except:
+        logging.error('_get_func_args error')
+        return []
+
+
+def print_exception():
+    """
+    直接输出异常信息
+    """
+    print(_build_json_format(_get_exception_info()))
+
+
+_sentry_client = None
+try:
+    _sentry_client = get_raven_client()
+except:
+    pass
+
+if _sentry_client is None:
+    _sentry_client = raven.Client(settings.SENTRY_CELERY_ENDPOINT)
+
+# 从gm-logging中取得通用的logging_exception方法，logging_excepiton的原型如下：
+# logging_exception(exception_logger, sentry_client=None, send_to_sentry=True, **kwargs)
+logging_exception = get_exception_logging_func(exception_logger, _sentry_client)
+
+
+def get_logging_exception_json():
+    func_args = _get_func_args()
+    exc = _get_exception_info()
+    error_json = _build_json_format({'stacktrace': exc, 'func_args': func_args})
+    return error_json
+
+
+class RequestInfoExtractor(gm_logging.RequestInfoExtractor):
+
+    __cl_type_map = {
+        'iPhone': 'ios',
+        'android': 'android',
+    }
+
+    @classmethod
+    def __get_cl_type(cls, cl_type):
+        return cls.__cl_type_map.get(cl_type, cl_type)
+
+    def __init__(self):
+        self.__local_ip = gm_logging.utils.get_local_ip()
+        self.__hostname = gm_logging.utils.get_hostname()
+
+    def get_request_info(self, request):
+        request_info = gm_logging.RequestInfo.create(
+            log_host=self.__local_ip,
+            hostname=self.__hostname,
+
+            module='saturn',
+
+            cl_type=self.__get_cl_type(request.GET.get('platform')),
+            cl_ver=request.GET.get('version'),
+            cl_os_ver=request.GET.get('os_version'),
+            phone_model=request.GET.get('model'),
+            cl_id=request.GET.get('device_id') or request.GET.get('idfa'),
+            cl_ios_idfv=request.GET.get('idfv'),
+            channel=request.GET.get('channel'),
+            session_id=request.session.session_key,
+            user_ip=get_client_ip(request),
+            gm_request_id=request.META.get('HTTP_X_GM_REQUEST_ID'),
+
+            http_verb=request.method,
+            action=request.path,
+        )
+
+        extra = {
+            'cl_longitude': request.GET.get('lng'),
+            'cl_latitude': request.GET.get('lat'),
+            'cl_city_id': request.GET.get('current_city_id'),
+            'phone_id': request.GET.get('phone_id'),
+        }
+        try:
+            # 兼容旧版 gm-logging
+            request_info.update(**extra)
+        except Exception:
+            pass
+        return request_info
+

engine/middleware/__init__.py

+
+from .log import TrackingMiddleware, LoggingMiddleware, TrackingCityMiddleware
+from .profile import ProfileMiddleware
+from .rpc import HeliosMiddleware
+from .crt import CTRTrackMiddleware

engine/middleware/crt.py

+
+
+class CTRTrackMiddleware(object):
+
+    def process_response(self, request, response):
+        """
+        CTR track middleware.
+        :param request:
+        :param response:
+        :return:
+        """
+        if hasattr(request, 'ctr_track'):
+
+            req = request.GET.dict()
+            req.update(request.POST.dict())
+            try:
+                res = json.loads(response.content).get('data', {})
+            except ValueError:
+                res = {}
+
+            for record in request.ctr_track:
+                data = record.copy()
+
+                try:
+                    exposure_data = [
+                        {'business_type': key, 'list_ids': value} for key, value in data.pop('data')(req, res).items()
+                    ]
+                except KeyError:
+                    exposure_data = None  # response is broken
+
+                if exposure_data:
+                    data.update({
+                        'is_exposure': 1,
+                        'exposure_data': exposure_data
+                    })
+
+                for key, value in data.items():
+                    try:
+                        v = value(req, res) if callable(value) else value
+                        data.update({key: v})
+                    except KeyError:
+                        # json response broken
+                        pass
+
+                request.logger.app(**data)
+
+        return response

engine/middleware/log.py

+
+import time
+import logging
+import json
+import uuid
+import random
+import datetime
+
+from django.conf import settings
+from django.utils.http import cookie_date
+
+from engine.logger import exception_logger, profile_logger
+from engine.exception_info import get_exception_info, get_func_args
+
+
+class TrackingMiddleware(object):
+    """set/get tracking cookie."""
+
+    def process_request(self, request):
+        """check if tracking cookie exist"""
+        if settings.TRACKING_COOKIE_NAME in request.COOKIES:
+            request._has_tracking_cookie = True
+            request.tracking_cookie = request.COOKIES[settings.TRACKING_COOKIE_NAME]
+
+        else:
+            request._has_tracking_cookie = False
+            request.tracking_cookie = uuid.uuid1().hex + str(random.randrange(1, 10000))
+
+    def process_response(self, request, response):
+        if not request._has_tracking_cookie:
+            expires_time = time.time() + settings.TRACKING_COOKIE_AGE
+            expires = cookie_date(expires_time)
+            response.set_cookie(
+                settings.TRACKING_COOKIE_NAME,
+                request.tracking_cookie,
+                max_age=settings.TRACKING_COOKIE_AGE,
+                expires=expires,
+            )
+
+        return response
+
+
+class LoggingMiddleware(object):
+
+    def process_request(self, request):
+        request.start_time = time.time()
+
+    def process_response(self, request, response):
+        execute_time = time.time() - request.start_time
+        path = request.get_full_path()
+        gm_request_id = request.META.get('HTTP_X_GM_REQUEST_ID')
+
+        try:
+            items = {}
+            if request.method == 'GET':
+                items = request.GET.dict()
+
+            elif request.method == 'POST':
+                items = request.POST.dict()
+
+            items = json.dumps(items)
+        except:
+            items = ''
+
+        profile_logger.info(
+            json.dumps(
+                {
+                    'timestamp': datetime.datetime.fromtimestamp(
+                        request.start_time
+                    ).strftime('%Y-%m-%dT%H:%M:%S+08:00'),
+                    'request_method': request.method,
+                    'path': path,
+                    'execute_time': execute_time,
+                    'items': items,
+                    'gm_request_id': gm_request_id
+                }
+                , ensure_ascii=True
+            )
+        )
+        return response
+
+    def process_exception(self, request, exception):
+        gm_request_id = request.META.get('HTTP_X_GM_REQUEST_ID')
+        exc = get_exception_info()
+        func_args = get_func_args()
+        exception_logger.info(
+            json.dumps(
+                {
+                    'gm_request_id': gm_request_id,
+                    'stacktrace': exc,
+                    'func_args': func_args,
+                    'timestamp': datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S+08:00'),
+                }
+                , ensure_ascii=True
+            )
+        )
+
+
+class TrackingCityMiddleware(object):
+    def process_request(self, request):
+        current_city_id = request.GET.get('current_city_id', '')
+        request.logger.app(tracking_id=request.tracking_cookie, current_city_id=current_city_id)
+

engine/middleware/profile.py

+
+import os
+import errno
+import cProfile
+import datetime
+import random
+
+
+log_dir = os.path.join(os.path.dirname(__file__), '..', 'profile_log')
+try:
+    os.mkdir(log_dir)
+except OSError as e:
+    if e.errno != errno.EEXIST:
+        raise
+
+
+class Profile(object):
+
+    def __init__(self, name=None):
+        if name is None:
+            name = '{}{}'.format(
+                datetime.datetime.now().strftime('%Y-%m-%dT%H_%M_%S'),
+                random.randrange(1000000),
+            )
+        self._name = name
+        self._filename = os.path.join(log_dir, name)
+        self._profile = cProfile.Profile(self._filename)
+
+    def start(self):
+        self._profile.start()
+
+    def stop(self):
+        self._profile.stop()
+        self._profile.close()
+
+    def __enter__(self):
+        self.start()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+
+    @property
+    def profile(self):
+        return self._profile
+
+
+class ProfileMiddleware(object):
+
+    def process_request(self, request):
+        request._tux_profile = Profile()
+        request._tux_profile.start()
+
+    def process_response(self, request, response):
+        request._tux_profile.stop()
+

engine/middleware/rpc.py

+
+import time
+import logging
+import json
+import uuid
+import random
+import datetime
+import threading
+
+from django.conf import settings
+from django.utils.http import cookie_date
+from django.contrib.sessions.middleware import SessionMiddleware
+
+from gm_logging.django.middleware import get_client_info_of_request
+
+from engine.rpc_base import rpc_invoker
+
+
+class HeliosController(object):
+
+    __local = threading.local()
+
+    @classmethod
+    def enter_request(cls, request):
+        # integrity with gm_logging
+        client_info = get_client_info_of_request(request)
+
+        rpc_invoker = cls.__get_base_rpc_invoker().with_config(
+            session_key=request.session.session_key,
+            client_info=client_info,
+        )
+
+        request.rpc = rpc_invoker
+        cls.__local.request = request
+
+    @classmethod
+    def exit_request(cls, request):
+        cls.__local.request = None
+
+    @classmethod
+    def get_current_rpc_invoker(cls):
+        request = getattr(cls.__local, 'request', None)
+        if request is None:
+            # write log here
+            return cls.__get_base_rpc_invoker()
+        return request.rpc
+
+    @classmethod
+    def __get_base_rpc_invoker(cls):
+        return settings.RPC_INVOKER
+
+    @classmethod
+    def get_current_request(cls):
+        return getattr(cls.__local, 'request', None)
+
+
+class HeliosMiddleware(object):
+
+    def process_request(self, request):
+        HeliosController.enter_request(request)
+
+    def process_response(self, request, response):
+        HeliosController.exit_request(request)
+        return response

engine/network.py

+
+def get_client_ip(request):
+    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+    if x_forwarded_for:
+        ip = x_forwarded_for.split(',')[0]
+    else:
+        ip = request.META.get('REMOTE_ADDR')
+    return ip
\ No newline at end of file

engine/response.py

+
+import json
+
+from django.http import HttpResponse
+
+
+def json_response(result, status=200, extra_headers=None):
+
+    if extra_headers is None:
+        extra_headers = {}
+
+    if 'extra' not in result:
+        result['extra'] = {}
+
+    if 'data' not in result:
+        result['data'] = {}
+
+    response = HttpResponse(
+        json.dumps(result),
+        content_type="application/json; charset=UTF-8",
+        status=status,
+    )
+    for header_key, header_value in extra_headers.items():
+        response[header_key] = header_value
+
+    return response
+
+
+def get_error_message(code, extra=None):
+
+    result = {'error': 0}
+
+    if code >= 10000:
+        result['error'] = 1
+
+    result['error_code'] = code
+
+    msg = ERROR_CODE.getDesc(code)
+    if extra:
+        msg += ' %s' % extra
+    result['message'] = msg
+
+    return result
\ No newline at end of file

engine/rpc.py

+
+import functools
+import logging
+
+from django.conf import settings
+from django.core.wsgi import WSGIHandler
+from django.core.cache import cache
+from django.db import signals
+
+from gm_rpcd import all
+from engine.middleware.rpc import HeliosController
+from engine.rpc_base import rpc_invoker
+
+_all__ = ['rpc_invoker', 'bind', 'Bind', 'get_current_rpc_invoker']
+
+
+def bind(endpoint, **options):
+
+    endpoint = 'saturn/' + endpoint
+    return all.bind(endpoint, **options)
+
+
+def get_current_rpc_invoker():
+    return HeliosController.get_current_rpc_invoker()
+
+
+class Context(object):
+
+    @property
+    def user(self):
+        key = 'session:{}'.format(all.context.session_id)
+        user_info = cache.get(key)
+        if not user_info:
+            user_info = all.context.rpc['passport/info/by_session']().unwrap()
+            cache.set(key, user_info, timeout=180)
+        return user_info
+
+    @property
+    def rpc(self):
+        return all.context.rpc
+
+
+class Bind(object):
+    project = 'saturn'
+    app = ''
+    exception_map = {}
+
+    def _bind(self, endpoint, with_context=False):
+        def _wrap(func):
+            @functools.wraps(func)
+            def _wrapper(*args, **kwargs):
+                self.request_start(*args, **kwargs)
+                try:
+                    if with_context:
+                        data = func(Context(), *args, **kwargs)
+                    else:
+                        data = func(*args, **kwargs)
+                except Exception as e:
+                    raise self.exception_map.get(type(e), e)
+                finally:
+                    self.request_end(*args, **kwargs)
+
+                return self.format_data(data)
+
+            return all.bind('{}/{}/{}'.format(self.project, self.app, endpoint))(_wrapper)
+
+        return _wrap
+
+    def request_start(self, *args, **kwargs):
+        signals.request_started.send(sender=WSGIHandler)
+
+    def request_end(self, *args, **kwargs):
+        signals.request_finished.send(sender=WSGIHandler)
+
+    def format_data(self, data):
+        return data
+
+    def bind_context(self, endpoint):
+        return self._bind(endpoint, with_context=True)
+
+    def __call__(self, endpoint):
+        return self._bind(endpoint)

engine/rpc_base.py

+
+from django.conf import settings
+
+from helios.rpc import create_default_invoker
+
+rpc_invoker = create_default_invoker(
+    debug=settings.DEBUG
+).with_config(
+    dump_curl=settings.DEBUG
+)

libs/user.py

+import json
+from functools import wraps
+
+from django.http import HttpResponse
+from alpha_types.venus.error import ERROR as CODES
+from helios.rpc import RPCFaultException
+
+from libs.utils import DictWrapperUseDot
+from engine.logger import auth_logger
+
+
+def get_user_by_request(request):
+    """获取用户信息"""
+
+    try:
+        user_info = request.rpc["venus/account/user/self_user_detail"]().unwrap()
+    except:
+        return None
+
+    if not user_info["logined"]:
+        return None
+
+    info = {
+        "id": user_info["user_id"],
+        "user_id": user_info["user_id"],
+        "nick_name": user_info["nick_name"],
+        "profile_pic": user_info["profile_pic"],
+        "city_id": user_info["city_id"],
+        "gender": user_info["gender"],
+        "register_time": user_info["register_time"],
+    }
+
+    return DictWrapperUseDot(info)
+
+
+def auth(request):
+    user_info = get_user_by_request(request)
+    if not user_info:
+        return False
+
+    request.user = user_info
+
+    return True
+
+
+def login_required(view_func):
+    """登录校验"""
+
+    @wraps(view_func)
+    def _wrapped_view(request, *args, **kwargs):
+
+        if auth(request):
+            return view_func(request, *args, **kwargs)
+
+        request_data = {'request': {key: value for key, value in request.REQUEST.items()}}
+        auth_logger.error(json.dumps(request_data))
+
+        result = {
+            'error': 1,
+            'message': CODES.getDesc(CODES.LOGIN_REQUIRED),
+            'error_code': CODES.LOGIN_REQUIRED,
+        }
+
+        return HttpResponse(status=403, content=json.dumps(result))
+
+    return _wrapped_view
+

libs/utils.py

+
+
+class DictWrapperUseDot(object):
+    """wrap a dict, get key by dot syntax."""
+
+    def __init__(self, _dict):
+        assert isinstance(_dict, dict)
+        self.__data = {}
+
+        for k, v in _dict.items():
+            if isinstance(v, dict):
+                self.__data[k] = DictWrapperUseDot(v)
+            else:
+                self.__data[k] = v
+
+    def __getattr__(self, attr):
+        return self.__data[attr]
+

manage.py

+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == '__main__':
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        )
+    execute_from_command_line(sys.argv)

requirements.txt

+PyMySQL==0.9.2
+Django==1.8
+celery==4.2.0
+kafka-python
+# line_profiler==1.0
+django-cors-headers==2.4.0
+pycrypto==2.6.1
+lxml
+mistune==0.8.4
+python-geohash==0.8.5
+
+gunicorn==19.9.0
+gevent==1.3.7
+
+
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-rpcd.git@v0.2.0
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-logging.git@v0.8.1
+git+ssh://git@git.wanmeizhensuo.com/backend/helios.git@v0.6.3
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-config.git@v0.1.2#egg=gm-config==0.1.2
+git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-types.git@master
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-shield.git@1.0.0
+git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@dev
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-protocol.git@master
+git+ssh://git@git.wanmeizhensuo.com/backend/gm-upload.git@v0.74

saturn/django_init.py

+
+import os
+import django
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings.settings')
+django.setup()

saturn/urls.py

+
+"""saturn URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/2.1/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.conf.urls import url, include
+# from django.urls import path, include
+
+from api.urls import urlpatterns
+
+urlpatterns = [
+    # url('admin/', admin.site.urls),
+    url(r'^api/', include('api.urls'))
+]

saturn/views.py

+
+from api.views import *

saturn/wsgi.py

+
+"""
+WSGI config for saturn project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings.settings')
+
+application = get_wsgi_application()

settings/base.py

+
+import helios
+
+from .log_settings import LOG_DIR
+
+SENTRY_CELERY_ENDPOINT = 'http://45b0ef6a79204d6988b59c32c9dd4bef:fa018677fb4f49be88d2ba61cdc7ee32@sentry.igengmei.com/135'
+USER_COOKIE_NAME = 'sessionid'
+SESSION_COOKIE_AGE = 60 * 60 * 24 * 600  # Age of cookie, in seconds (default: 300 weeks).
+SESSION_COOKIE_PATH = '/'
+SESSION_COOKIE_SECURE = False
+SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_DOMAIN_IGENGMEI = '.alpha.env'
+
+START_NUM_UPLIMIT = 1200
+
+def GM_LOGGING():
+    from engine.logger import RequestInfoExtractor
+    return {
+        'request_info_extractor_class': RequestInfoExtractor,
+        'log': {
+            'basedir': LOG_DIR,
+            'prefix': 'saturn',
+            'buffered': not DEBUG,
+        }
+    }
+
+GMTRACER_PARSER = 'engine.setup_tracer'
+
+DEBUG = False
+
+RPC_INVOKER = helios.rpc.create_default_invoker(debug=DEBUG)
+
+TRACKING_COOKIE_NAME = '_gtid'
+TRACKING_COOKIE_AGE = 365 * 24 * 60 * 60
+
+API_HOST = "http://backend.alpha.env"
+WEB_API_HOST = "http://m.alpha.env"
+
+# wechat_pub
+WECHATPUB_APPID = 'wx007f571568e5186e'
+WECHATPUB_APPSECRET = 'c241098e95f3a0b3e654d5ca5b370db0'
+
+QQ_MAP_KEYS = [
+    'QVMBZ-KZFRJ-VQ5F6-FMJIJ-SCJN6-WNFFS',
+    '2RJBZ-EWTW3-ANB3A-3O6KS-SDYRK-EQFFO',
+    '2LZBZ-K353F-ZQKJG-JGT7Z-PQYD2-YVBLE',
+    'D7FBZ-SRGKG-AU3QI-IGMXR-63EU3-3PBOA',
+    'H52BZ-JH2WX-EAB4T-TXW23-SNBLQ-GNFNQ',
+    'RL2BZ-CASW3-MJ53C-3JCUR-IEGRO-TABLU',
+    'NJIBZ-L2H3U-ZWLV6-4W3PK-H7DFF-OSFUM',
+    'ZOBBZ-FP2WX-CA74G-TZC24-MPSL7-22BD4',
+    'TENBZ-AZ53P-JRTDP-LUIQ2-KNVE6-XNF4X',
+    'L7GBZ-JKMWP-HKJDV-LQPEC-QQ2M7-D6BAT',
+    'PFPBZ-63ZRR-AB2W7-WHADL-LSDEH-CNFRN',
+    '2EEBZ-6RYWD-EF546-H7Z5U-HKFLT-JWB2L',
+    'KXHBZ-3HZ3O-OSIWQ-SVMHT-DMLBE-J6FJD',
+    'CUNBZ-QAL3X-BPX4M-7BL77-YG5TZ-SCBPI',
+    'QM5BZ-Z2RRX-ZZR4M-ZGBYO-X5BH5-X2BK2',
+]
+
+QINIU_ACCESS_KEY = "UPCOYIJkZOMcdd9FDzpBqYjzWUh55fBpVi3AhWpL"
+QINIU_SECRET_KEY = "z5YvpDDSam_JE345Z8J_f3TufzelOW2VOGNoBl9e"
+GM_UPLOAD_ENV = "alpha"
+
+QINIU_WATERMARK_BUCKET = 'alpha'
+QINIU_FACE_BUCKET = 'alpha-s'
+
+FACE_TOP_N_CNT = 3
+
+REDIS_GROUP = {
+    'desc_cache': {'host': '127.0.0.1', 'port': 6379, 'db': 1},
+    'qq_cache': {'host': '127.0.0.1', 'port': 6379, 'db': 1},
+}
+
+OPERTATION_POSITION = 3  # 运营卡片位置
+PORTRAIT_POSITION = 0  # 拟合头像卡片位置
+
+CELERY_TIMEZONE = 'Asia/Shanghai'
+CELERY_TASK_SERIALIZER = "json"
+BROKER_URL = 'redis://127.0.0.1:6379/5'
+FACE_SHARE_URL = 'http://earth.iyanzhi.com/alpha/fitting/'
+FACE_SHARE_DESCRIPTION = '在颜值打卡APP，这里的颜值很不错！'
+FACE_IMG_URL = 'http://alpha-s.iyanzhi.com/'
+
+IMAGE_SUFFIX = '-aspectscale'  # 瀑布流图片后缀

settings/core.py

+"""
+Django settings for saturn project.
+
+Generated by 'django-admin startproject' using Django 1.8.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.8/ref/settings/
+"""
+
+import os
+
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 't!*m42n6c^v4e+g%h*+%sc=-znrdi&^og**x-uf$2_*+@vwxd1'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+# DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+# interface decrypt for import user_key
+ENCRYPT_KEY = 'Up[K+ub%pliOnsO5UavFBd)cw5VcyHSX'
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'api',
+    'corsheaders'
+]
+
+MIDDLEWARE_CLASSES = [
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+
+    # 'gm_logging.django.middleware.LoggingMiddleware',
+    # 'gm_tracer.middleware.TracerMiddleware',
+    # 'gm_tracer.middleware.TracerExtraMiddleware',
+    # 'gm_shield.django.middleware.GMShield',
+    # 'helios.DjangoL5dMiddleware',
+
+    # 'engine.middleware.LoggingMiddleware',
+    # 'engine.middleware.TrackingMiddleware',
+    # 'engine.middleware.TrackingCityMiddleware',
+    # 'engine.middleware.HeliosMiddleware',
+    # 'engine.middleware.CTRTrackMiddleware',
+]
+
+ROOT_URLCONF = 'saturn.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'saturn.wsgi.application'
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.8/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'Asia/Shanghai'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.8/howto/static-files/
+
+STATIC_URL = '/static/'
+
+
+# 跨域问题
+CORS_ALLOW_CREDENTIALS = True
+CORS_ORIGIN_ALLOW_ALL = True
+CORS_ORIGIN_WHITELIST = (
+    '*',
+)
+
+CORS_ALLOW_METHODS = (
+    'DELETE',
+    'GET',
+    'OPTIONS',
+    'PATCH',
+    'POST',
+    'PUT',
+    'VIEW',
+)
+
+CORS_ALLOW_HEADERS = (
+    'XMLHttpRequest',
+    'X_FILENAME',
+    'accept-encoding',
+    'authorization',
+    'content-type',
+    'dnt',
+    'origin',
+    'user-agent',
+    'x-csrftoken',
+    'x-requested-with',
+    'Pragma',
+)

settings/log_settings.py

+import os
+
+LOG_DIR = '/data/log/saturn/app/'
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+
+    'formatters': {
+        'verbose': {
+            'format': '%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'profile': {
+            'format': '%(asctime)s %(message)s'
+        },
+        'consumer': {
+            'format': '%(asctime)s %(levelname)s Process ID:%(process)d %(module)s.%(funcName)s Line:%(lineno)d %(message)s'
+        },
+        'raw': {
+            'format': '%(message)s'
+        },
+    },
+
+    'handlers': {
+        'null': {
+            'level': 'DEBUG',
+            'class': 'logging.NullHandler',
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
+        },
+        'default': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'default.log'),
+            'formatter': 'verbose',
+        },
+        'info_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'info.log'),
+            'formatter': 'verbose',
+        },
+        'error_handler': {
+            'level': 'ERROR',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'error.log'),
+            'formatter': 'verbose',
+        },
+        'profile_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'profile.log'),
+            'formatter': 'profile',
+            'maxBytes': 500 * 1024 * 1024,  # 500M
+        },
+
+        'exception_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'exception.log'),
+            'formatter': 'verbose',
+        },
+
+        'tracer_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'tracer.log'),
+            'formatter': 'raw',
+        },
+
+        'auth_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'auth.log'),
+            'formatter': 'verbose',
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 30,
+        },
+        'wechat_handler': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': os.path.join(LOG_DIR, 'wechat.log'),
+            'formatter': 'verbose',
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 30,
+        },
+    },
+
+    'loggers': {
+        'django': {
+            'handlers': ['default'],
+            'propagate': True,
+            'level': 'INFO',
+        },
+        'django.request': {
+            'handlers': ['error_handler'],
+            'level': 'ERROR',
+            'propagate': False,
+        },
+        'info_logger': {
+            'handlers': ['info_handler', 'console'],
+            'level': 'DEBUG',
+            'propagate': False,
+        },
+        'error_logger': {
+            'handlers': ['error_handler'],
+            'level': 'ERROR',
+            'propagate': False,
+        },
+        'profile_logger': {
+            'handlers': ['profile_handler'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+
+        'exception_logger': {
+            'handlers': ['exception_handler'],
+            'level': 'ERROR',
+            'propagate': False,
+        },
+
+        'gm_tracer.subscribe': {
+            'handlers': ['tracer_handler'],
+            'propagate': False,
+            'level': 'INFO'
+        },
+
+        'auth_logger': {
+            'handlers': ['auth_handler'],
+            'propagate': False,
+            'level': 'INFO'
+        },
+        'wechat_logger': {
+            'handlers': ['wechat_handler'],
+            'propagate': False,
+            'level': 'INFO'
+        },
+    }
+}
+

settings/settings.py

+
+from .core import *
+from .log_settings import *
+from .base import *
+from .settings_local import *

settings/settings_local.py.bak

+import helios
+
+DEBUG = True
+
+# QINIU_ACCESS_KEY = "ln5otz9s46t3LcpAFXcA1faYO8y1K34f6cvVBXiz"
+# QINIU_SECRET_KEY = "hDZJAfVecAn0qj3pCONyd9ba-cTiekQZs2HIJalI"
+
+SENTRY_CELERY_ENDPOINT = 'http://2ee4f9dbe16747da9ac504cc62deace2:37bcd9464f1d47a99b2b72e777f322f4@sentry.gengmei.cc/26'
+
+# ALLOWED_HOSTS = ["xxx"]
+
+USER_COOKIE_NAME = 'sessionid'
+
+RPC_INVOKER = helios.rpc.create_default_invoker(debug=DEBUG)
+
+DEBUG_PROPAGATE_EXCEPTIONS = True
+
+TMP_STATIC_DOMAIN = "earth.igengmei.com"
+SESSION_COOKIE_DOMAIN_IGENGMEI = '127.0.0.1'