gm serializer init

.gitignore

+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Pycharm
+atlassian-ide-plugin.xml
+.idea/

README.md

+
+#### 更美业务服务化拆分与序列化组件
+* 适用场景一：业务拆分为独立服务时, 提供WeakRelField(弱关系外键)替代ForeignKey, 支持惰性加载，批量加载，ObjCache和RedisCache两层缓存；
+* 适用场景二：对象序列化时，提供弱Schema，处理相关Field(包括WeakRelField);
+
+#### 配置
+
+> settings.py
+
+```python
+from rpc.context import get_gaia_local_invoker
+RPC_INVOKER = get_gaia_local_invoker()
+```
+
+#### 使用示例
+
+```python
+from gm_serializer import serializers
+from gm_serializer.models import WeakRelField
+
+
+# 社区拆分之后，将 user(ForeignKey) 改为 WeakRelField
+class Question(models.Model):
+    user = WeakRelField(type=int, url='api/user/get_fundamental_info_by_user_ids', args='user_ids')
+    title = models.CharField(max_length=128, null=False, verbose_name=u'问题')
+    content = models.TextField(null=True, verbose_name=u'描述')
+
+
+class QuestionSerializer(serializers.GmModelSerializer):
+    user = serializers.WeakRelField(
+        validators = [serializers.BlacklistValidator]
+    )
+    content = serializers.TextField(
+        validators = [serializers.FilterWordValidator]
+    )
+    
+    class Meta:
+        model = Question
+        lazy_fields = ('user',)
+        fields = '__all__'
+```
+

gm_serializer/__init__.py

+__version__ = '0.0.1'
+__author__ = 'gushitong@gmei.com'

gm_serializer/cache.py

+
+import json
+import redis
+from django.conf import settings
+
+
+class _PoolMixin(object):
+    _pool = redis.ConnectionPool(**settings.REDIS['gm_serializer'])
+    _client = redis.Redis(connection_pool=_pool)
+
+
+class WeakObjCache(_PoolMixin):
+
+    prefix = ''
+
+    def __init__(self, prefix):
+        self.prefix = prefix
+
+    def __getitem__(self, item):
+        """
+
+        :param item:
+        :return:
+        """
+        key = "%s:%s" % (self.prefix, str(item))
+        obj = self._client.get(key)
+        return json.loads(obj) if obj else None
+
+    def __setitem__(self, key, value):
+        """
+
+        :param key:
+        :param value:
+        :return:
+        """
+        key = "%s:%s" % (self.prefix, str(key))
+        if isinstance(value, dict):
+            value = json.dumps(value)
+        return self._client.set(key, value, 30*60)
+
+
+

gm_serializer/error.py

+
+
+class LazyObjNotExist(Exception):
+    pass
+
+
+class MultiLazyObjFound(Exception):
+    pass
+

gm_serializer/logger.py

+
+import logging
+
+logger = logging.getLogger('gm_serializer')

gm_serializer/manager.py

+
+from gm_serializer.cache import WeakObjCache
+from gm_serializer.logger import logger
+from gm_serializer.utils import RPCMixin, DictWrapper
+
+
+class WeakCacheManager(RPCMixin):
+
+    def __init__(self, name=None, value=None, value_lst=None, url=None, args=None):
+        self.name = name
+        self.value = value
+        self.value_lst = value_lst
+        self.url = url
+        self.args = args
+
+    @property
+    def cache(self):
+        return WeakObjCache(self.name)
+
+    def get_object(self):
+        return self.cache[self.value]
+
+    def get_queryset(self):
+        values = filter(lambda i: i is not None, [self.cache[key] for key in set(self.value_lst)])
+        if len(values) == len(set(self.value_lst)):
+            return values
+        return []
+
+
+class WeakRelManager(WeakCacheManager):
+
+    def get_object(self):
+        """
+        get object from remote rpc
+        :return:
+        """
+        obj = super(WeakRelManager, self).get_object()
+
+        if obj:
+            logger.debug("WeakRel RedisCache: %s %s" % (self.name, self.value))
+            return DictWrapper(obj)
+
+        kwargs = {
+            self.args: [self.value],
+        }
+        logger.debug("WeakRel RpcCall: %s %s=%s" % (self.url, self.args, self.value))
+        obj = self.call_rpc(self.url, **kwargs)[0]
+        self.cache[obj['id']] = obj
+        return DictWrapper(obj)
+
+    def get_queryset(self):
+        queryset = super(WeakRelManager, self).get_queryset()
+        if queryset:
+            logger.debug("WeakRel RedisCache: %s %s" % (self.name, self.value_lst))
+            return [DictWrapper(obj) for obj in queryset]
+
+        kwargs = {
+            self.args: self.value_lst,
+        }
+        logger.debug("WeakRel RpcCall: %s %s=%s" % (self.url, self.args, self.value_lst))
+        data = self.call_rpc(self.url, **kwargs)
+
+        for obj in data:
+            self.cache[obj['id']] = obj
+
+        return [DictWrapper(d) for d in data]

gm_serializer/models/__init__.py

+
+from .field import WeakRelField

gm_serializer/models/descriptor.py

+
+from gm_serializer.manager import DictWrapper
+from gm_serializer.logger import logger
+
+
+class WeakDescriptor(object):
+
+    def __init__(self, field_with_rel):
+        self.field = field_with_rel
+
+    @property
+    def cache_name(self):
+        return "_%s_cache_" % self.field.attname
+
+    @property
+    def lazy_name(self):
+        return "_%s_lazy_" % self.field.attname
+
+    def __get__(self, instance, cls):
+        """
+
+        :param instance:
+        :param cls:
+        :return:
+        """
+        if instance is None:
+            return self
+
+        if getattr(instance, self.lazy_name, False):
+            return getattr(instance, "%s_id" % self.field.attname)
+
+        try:
+            lazy_obj = getattr(instance, self.cache_name)
+            logger.debug("WeakRel ObjCache: %s:%s" % (self.field.attname, lazy_obj['id']))
+        except AttributeError:
+            lazy_obj = self.field.relation.get_object()
+            setattr(instance, self.cache_name, lazy_obj)
+        return lazy_obj
+
+    def __set__(self, instance, value):
+        """
+
+        :param instance:
+        :param value:
+        :return:
+        """
+        attname = self.field.attname
+        relation = self.field.relation
+
+        if value is None and self.field.null is False:
+            raise ValueError("Null value not allowed.")
+
+        if isinstance(value, DictWrapper):
+            setattr(instance, self.cache_name, value)
+            return
+
+        setattr(relation, "name", attname)
+        setattr(relation, "value", value)
+        setattr(instance, "%s_id" % attname, value)
+
+        if hasattr(instance, self.cache_name) and getattr(instance, self.cache_name)['id'] != value:
+            obj_cache = self.field.relation.get_object()
+            setattr(instance, "%s_id" % attname, value)
+            setattr(instance, self.cache_name, obj_cache)
+
+

gm_serializer/models/field.py

+from django.core import exceptions
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from gm_serializer.models.descriptor import WeakDescriptor
+from gm_serializer.manager import WeakRelManager
+
+INT = 'INT'
+VARCHAR = 'VARCHAR'
+
+
+class WeakRelField(models.Field):
+
+    default_error_messages = {
+        "invalid": _("'%(value)s' is not a valid %(type)")
+    }
+    relation_class = WeakRelManager
+    descriptor_class = WeakDescriptor
+
+    def __init__(self, *args, **kwargs):
+        """
+
+        :param args:
+        :param kwargs:
+        """
+        self.url = kwargs.pop('url')
+        self.type = kwargs.pop('type', int)
+        self.args = kwargs.pop('args', 'id')
+        self.batch = kwargs.pop('batch', True)
+        self.relation = self.relation_class(
+            url=self.url,
+            args=self.args,
+        )
+        super(WeakRelField, self).__init__(*args, **kwargs)
+
+    def get_internal_type(self):
+        return "VirtualForeignKey"
+
+    def db_type(self, connection):
+        if self.type is int:
+            return INT
+        elif self.type is str:
+            return VARCHAR
+        raise RuntimeError("Unsupported data type for VirtualForeignKey: %s" % self.type)
+
+    def get_db_prep_value(self, value, connection, prepared=False):
+        """
+        lazy foreign key db prep value.
+        :param value:
+        :param connection:
+        :param prepared:
+        :return:
+        """
+        if value is None:
+            return None
+
+        if hasattr(value, 'id'):
+            return self.type(value.id)
+
+        try:
+            return self.type(value)
+        except (TypeError, ValueError):
+            raise exceptions.ValidationError(
+                self.error_messages['invalid'],
+                code='invalid',
+                params={'value': value, 'type': str(self.type)}
+            )
+
+    def contribute_to_class(self, cls, name, **kwargs):
+        """
+        :param cls:
+        :param name:
+        :param kwargs:
+        :return:
+        """
+        super(WeakRelField, self).contribute_to_class(cls, name, **kwargs)
+        setattr(cls, self.name, self.descriptor_class(self))
+
+    def deconstruct(self):
+        """
+        :param
+        :return:
+        """
+        name, path, args, kwargs = super(WeakRelField, self).deconstruct()
+        kwargs.update({
+            'url': self.url, 'type': self.type, 'args': self.args, 'batch': self.batch
+        })
+        return name, path, args, kwargs
+

gm_serializer/serializers/__init__.py

+
+from .serializer import GmModelSerializer
+from .validators import BlacklistValidator, FilterWordValidator
+

gm_serializer/serializers/serializer.py

+
+from django.db import models
+from rest_framework import serializers
+from rest_framework.serializers import ListSerializer
+
+from gm_serializer.error import LazyObjNotExist, MultiLazyObjFound
+
+LIST_SERIALIZER_KWARGS = list(serializers.LIST_SERIALIZER_KWARGS)
+LIST_SERIALIZER_KWARGS.extend(['lazy', 'lazy_field_ids'])
+
+
+class GmModelSerializer(serializers.ModelSerializer):
+
+    @classmethod
+    def many_init(cls, *args, **kwargs):
+        allow_empty = kwargs.pop('allow_empty', None)
+        lazy = kwargs.pop('lazy', False)
+        child_serializer = cls(*args, **kwargs)
+        list_kwargs = {
+            'child': child_serializer,
+            'lazy': lazy,
+            'lazy_fields': cls.get_laze_fields(*args, **kwargs)
+        }
+
+        if allow_empty is not None:
+            list_kwargs['allow_empty'] = allow_empty
+
+        list_kwargs.update({
+            key: value for key, value in kwargs.items()
+            if key in LIST_SERIALIZER_KWARGS
+        })
+
+        meta = getattr(cls, 'Meta', None)
+        list_serializer_class = getattr(meta, 'list_serializer_class', GmListSerializer)
+        return list_serializer_class(*args, **list_kwargs)
+
+    @classmethod
+    def get_laze_fields(cls, *args, **kwargs):
+        lazy_fields = {}
+        meta = getattr(cls, 'Meta', None)
+        model = getattr(meta, 'model')
+        lazy_field_names = getattr(meta, 'lazy_fields', [])
+        for field_name in lazy_field_names:
+            field = getattr(model, field_name).field
+            rel = field.relation_class(
+                name=field_name,
+                url=field.url,
+                args=field.args,
+                value_lst=cls.get_field_value_list(field_name, args[0])
+            )
+            lazy_fields[field_name] = rel.get_queryset
+        return lazy_fields
+
+    @classmethod
+    def get_field_value_list(cls, field_name, obj_list):
+        return [getattr(obj, "%s_id" % field_name) for obj in obj_list]
+
+
+class GmListSerializer(ListSerializer):
+
+    def __init__(self, *args, **kwargs):
+        self.lazy = kwargs.pop('lazy', False)
+        self.lazy_fields = kwargs.pop('lazy_fields', None)
+        super(GmListSerializer, self).__init__(*args, **kwargs)
+
+    def to_representation(self, data):
+        iterable = data.all() if isinstance(data, models.Manager) else data
+        if self.lazy:
+            iterable = self.lazy_iterable(iterable)
+        else:
+            iterable = self.unlazy_iterable(iterable)
+        return [
+            self.child.to_representation(item) for item in iterable
+        ]
+
+    def lazy_iterable(self, iterable):
+        """
+
+        :param iterable:
+        :return:
+        """
+        for field in self.lazy_fields.keys():
+            for obj in iterable:
+                setattr(obj, '_%s_lazy_' % field, True)
+        return iterable
+
+    def unlazy_iterable(self, iterable):
+        """
+
+        :param iterable:
+        :return:
+        """
+        for field, future in self.lazy_fields.items():
+            assert callable(future)
+            futures = future()
+            iterable = self.zip_iterable(field, iterable, futures)
+        return iterable
+
+    @classmethod
+    def zip_iterable(cls, field, iterable, futures):
+        for i in range(len(iterable)):
+            lazy_id = getattr(iterable[i], "%s_id" % field)
+            future = filter(lambda item: item['id'] == lazy_id, futures)
+            if len(future) > 1:
+                raise MultiLazyObjFound
+            elif len(future) == 1:
+                setattr(iterable[i], field, future[0])
+            else:
+                raise LazyObjNotExist
+        return iterable
+
+    def update(self, instance, validated_data):
+        pass

gm_serializer/serializers/validators.py

+from rest_framework import serializers
+
+from gm_serializer.utils import RPCMixin
+
+
+class BlacklistValidator(RPCMixin):
+    def __call__(self, value):
+        try:
+            r = self.call_rpc('api/user/in_blacklist', user_id=value)
+        except:
+            return False
+
+        if r:
+            raise serializers.ValidationError("User in blacklist.")
+
+
+class FilterWordValidator(RPCMixin):
+
+    def __init__(self, filter_type=None):
+        self.filter_type = filter_type
+
+    def __call__(self, value):
+        try:
+            r = self.call_rpc('api/filterWord/list', filter_type=self.filter_type)
+        except:
+            return False
+        if r:
+            raise serializers.ValidationError("Content contains sensitive words.")
+
+
+

gm_serializer/utils.py

+
+from django.conf import settings
+
+
+class DictWrapper(dict):
+
+    def __getattribute__(self, item):
+        """
+
+        :param item:
+        :return:
+        """
+        if item in self:
+            if isinstance(self[item], dict):
+                return DictWrapper(self[item])
+            return self[item]
+        raise AttributeError
+
+
+class RPCMixin(object):
+
+    @classmethod
+    def rpc_invoker(cls):
+        return settings.RPC_INVOKER
+
+    def call_rpc(self, api_endpoint, **kwargs):
+        """call rpc.
+
+        get rpc invoker from current ctx, this method only support call rpc sequentially.
+        """
+        result = self.rpc_invoker()[api_endpoint](**kwargs)
+        return result.unwrap()
+
+

setup.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from setuptools import find_packages
+from setuptools import setup
+import os
+import io
+import re
+import ast
+
+# requirements
+install_requires = [
+    "djangorestframework==3.6.0",
+    "redis",
+    "django",
+]
+
+
+here = os.path.abspath(os.path.dirname(__file__))
+with io.open(os.path.join(here, 'gm_serializer', '__init__.py')) as f:
+    version = None
+    for line in f.readlines():
+        match = re.match(r"""__version__ *= *(.*)""", line)
+        if not match:
+            continue
+        content = match.groups()[0]
+        version = ast.literal_eval(content)
+
+
+setup(name="gm-serializer",
+      version=version,
+      description="gengmei serializer",
+      author="gushitong",
+      author_email="gushitong@gmei.com",
+      packages=find_packages(),
+      url="http://git.gengmei.cc/gushitong/gm-serializer",
+      install_requires=install_requires)