Commit 8bfcc1b0 authored by gushitong's avatar gushitong

Add manager, ttl suppport.

parent 8bf30ec9
......@@ -60,3 +60,5 @@ target/
# Pycharm
atlassian-ide-plugin.xml
.idea/
migrations/
db.sqlite3
\ No newline at end of file
#### 更美业务服务化拆分与序列化组件
* 适用场景一:业务拆分为独立服务时, 提供WeakRelField(弱关系外键)替代ForeignKey, 支持惰性加载,批量加载,ObjCache和RedisCache两层缓存;
* 适用场景二:对象序列化时,提供弱Schema,处理相关Field(包括WeakRelField);
#### Cross Database ForeignKey - MagicField
* 适用场景一:业务拆分为独立服务时, 提供MagicField(弱关系外键)替代ForeignKey, 支持惰性加载,批量加载,ObjCache和RedisCache两层缓存;
* 适用场景二:对象序列化时,提供弱Schema,处理相关Field(包括MagicField);
#### 配置
> settings.py
```python
from rpc.context import get_gaia_local_invoker
RPC_INVOKER = get_gaia_local_invoker()
```
#### 使用示例
```python
from gm_serializer import serializers
from gm_serializer.models import WeakRelField
from django.db import models
from gm_serializer import fields, serializers
class UserManager(object):
def __init__(self):
self.url = 'api/user/get'
def __call__(self, pk_list):
# mock return user instance from rpc call.
return [{'id': pk, 'username': "username %s" % pk} for pk in pk_list]
class Post(models.Model):
"""
This is demonstrate app model.
To explain how to use gm-serializer in your project.
`user` is more like a cross-database foreign-key relation.
"""
# 社区拆分之后,将 user(ForeignKey) 改为 WeakRelField
class Question(models.Model):
user = WeakRelField(type=int, url='api/user/get_fundamental_info_by_user_ids', args='user_ids')
title = models.CharField(max_length=128, null=False, verbose_name=u'问题')
content = models.TextField(null=True, verbose_name=u'描述')
user = fields.MagicField(type=int, manager=UserManager, ttl=60*60)
title = models.CharField(max_length=128)
content = models.CharField(max_length=255)
create_at = models.DateTimeField(auto_now_add=True)
update_at = models.DateTimeField(auto_now=True)
class QuestionSerializer(serializers.GmModelSerializer):
user = serializers.WeakRelField(validators = [
serializers.BlacklistValidator
])
content = serializers.TextField(validators = [
serializers.FilterWordValidator
])
class PostSerializer(serializers.GmModelSerializer):
user = serializers.MagicField()
class Meta:
model = Question
model = Post
lazy_fields = ('user',)
fields = '__all__'
fields = ('user', 'title', 'content', 'create_at')
```
```python
>>> from answer.models import Question
>>> from answer.serializers.question import QuestionSerializer
>>> q = Question.objects.first()
>>> q.user
{u'city_id': None, u'user_rights_level': u'1', u'topic_count': 0, u'city_name': None, u'portrait': u'http://pic.gmei.com/user/icon7@2x.png-thumb', u'country_tag_id': None, u'nickname': u'18*******4*', u'id': 2500, u'membership_level': u'0', u'level_icon': u'http://hera.s.gmei.com/2017/06/12/d2068222ba', u'constellation': u'', u'membership_icon': u'', u'vote_count': u'0', u'city_tag_id': None, u'constellation_icon': u'', u'person_id': u'55729f2e5aa111e5a2a200163e00488a', u'province_tag_id': None}
>>> qs = Question.objects.all()[:2]
>>>
>>> QuestionSerializer(qs, many=True).data
[OrderedDict([(u'id', 1), ('user', u"{u'city_id': None, u'user_rights_level': u'1', u'topic_count': u'0', u'city_name': None, u'portrait': u'http://pic.gmei.com/user/icon7@2x.png-thumb', u'country_tag_id': None, u'nickname': u'18*******4*', u'id': 2500, u'membership_level': u'0', u'level_icon': u'http://hera.s.gmei.com/2017/06/12/d2068222ba', u'constellation': u'', u'membership_icon': u'', u'vote_count': u'0', u'city_tag_id': None, u'constellation_icon': u'', u'person_id': u'55729f2e5aa111e5a2a200163e00488a', u'province_tag_id': None}"), ('title', u'auto question title'), ('content', u'auto question XuraDFEiam'), ('cover_url', None), ('is_online', True), ('is_recommend', False), ('like_num', 0), ('create_time', '2017-06-27T17:55:41.404296'), ('update_time', '2017-07-04T16:55:59.389776'), ('is_spam', False), ('problem_id', None), ('city_id', None)]), OrderedDict([(u'id', 2), ('user', u"{u'city_id': u'beijing', u'user_rights_level': u'1', u'topic_count': 0, u'city_name': u'\\u5317\\u4eac', u'portrait': u'http://pic.gmei.com/userextra/2016/09/08/165c678aa2-thumb', u'country_tag_id': 259, u'nickname': u'xxxxxxx', u'id': 1, u'membership_level': u'1', u'level_icon': u'', u'constellation': u'', u'membership_icon': u'', u'vote_count': u'0', u'city_tag_id': 328, u'constellation_icon': u'', u'person_id': u'e03b55b85a9911e5a2a200163e00488a', u'province_tag_id': 264}"), ('title', u'auto question title'), ('content', u'auto question XuraDFEiam'), ('cover_url', None), ('is_online', True), ('is_recommend', False), ('like_num', 0), ('create_time', '2017-06-27T17:55:41.404296'), ('update_time', '2017-06-27T17:55:41.423131'), ('is_spam', False), ('problem_id', None), ('city_id', None)])]
>>> QuestionSerializer(qs, many=True, lazy=True).data
[OrderedDict([(u'id', 1), ('user', 2500L), ('title', u'auto question title'), ('content', u'auto question XuraDFEiam'), ('cover_url', None), ('is_online', True), ('is_recommend', False), ('like_num', 0), ('create_time', '2017-06-27T17:55:41.404296'), ('update_time', '2017-07-04T16:55:59.389776'), ('is_spam', False), ('problem_id', None), ('city_id', None)]), OrderedDict([(u'id', 2), ('user', 1L), ('title', u'auto question title'), ('content', u'auto question XuraDFEiam'), ('cover_url', None), ('is_online', True), ('is_recommend', False), ('like_num', 0), ('create_time', '2017-06-27T17:55:41.404296'), ('update_time', '2017-06-27T17:55:41.423131'), ('is_spam', False), ('problem_id', None), ('city_id', None)])]
>>>
>>> serializer = QuestionSerializer(data={"user": 1000, "title": "title", "content": "content"})
>>> serializer.is_valid()
True
>>> serializer.save()
class MagicFieldTestCase(TestCase):
def setUp(self):
for i in range(10):
Post.objects.create(user=i, title="article %s" % i, content="body %s" % i)
def test_get_object(self):
post = Post.objects.first()
self.assertEqual(post.user, {'id': 0, 'username': 'username 0'})
def test_get_queryset(self):
"""
pass
:return:
"""
posts = Post.objects.all()[:10]
serializer = PostSerializer(posts, many=True)
for idx, obj in enumerate(serializer.data):
self.assertEqual(obj['user'], {'id': idx, 'username': 'username %s' % idx})
def test_get_queryset_lazy(self):
posts = Post.objects.all()[:10]
serializer = PostSerializer(posts, many=True, lazy=True)
for idx, obj in enumerate(serializer.data):
self.assertEqual(obj['user'], idx)
```
__version__ = '0.0.1'
__version__ = '0.0.2'
__author__ = 'gushitong@gmei.com'
......@@ -3,18 +3,21 @@ import json
import redis
from django.conf import settings
__all__ = []
class _PoolMixin(object):
_pool = redis.ConnectionPool(**settings.REDIS['gm_serializer'])
_client = redis.Redis(connection_pool=_pool)
class WeakObjCache(_PoolMixin):
class RedisCache(_PoolMixin):
prefix = ''
def __init__(self, prefix):
def __init__(self, prefix, ttl=3600):
self.prefix = prefix
self.ttl = ttl
def __getitem__(self, item):
"""
......@@ -36,7 +39,7 @@ class WeakObjCache(_PoolMixin):
key = "%s:%s" % (self.prefix, str(key))
if isinstance(value, dict):
value = json.dumps(value)
return self._client.set(key, value, 30*60)
return self._client.set(key, value, self.ttl)
from .field import MagicField
from gm_serializer.cache import WeakObjCache
from gm_serializer.cache import RedisCache
from gm_serializer.logger import logger
from gm_serializer.utils import RPCMixin, DictWrapper
class WeakCacheManager(RPCMixin):
class CacheAccessor(RPCMixin):
def __init__(self, name=None, value=None, value_lst=None, url=None, args=None):
def __init__(self, name=None, manager=None, pk_list=None, ttl=None):
"""
Cache accessor.
:param name: field name, and cache prefix.
:param manager: magic object manager.
:param pk_list: magic object list.
:param ttl: magic object redis cache ttl
"""
self.name = name
self.value = value
self.value_lst = value_lst
self.url = url
self.args = args
self.manager = manager() if type(manager) == type else manager
self.pk_list = pk_list
self.ttl = ttl
@property
def cache(self):
return WeakObjCache(self.name)
return RedisCache(self.name, ttl=self.ttl)
def get_object(self):
return self.cache[self.value]
"""
get object from cache.
:return:
"""
assert len(self.pk_list) == 1
return self.cache[self.pk_list[0]]
def get_queryset(self):
values = filter(lambda i: i is not None, [self.cache[key] for key in set(self.value_lst)])
if len(values) == len(set(self.value_lst)):
"""
get list object from cache.
:return:
"""
values = filter(lambda i: i is not None, [self.cache[pk] for pk in set(self.pk_list)])
if len(values) == len(set(self.pk_list)):
return values
return []
class WeakRelManager(WeakCacheManager):
class MagicAccessor(CacheAccessor):
def get_object(self):
"""
get object from remote rpc
:return:
"""
obj = super(WeakRelManager, self).get_object()
obj = super(MagicAccessor, self).get_object()
if obj:
logger.debug("WeakRel RedisCache: %s %s" % (self.name, self.value))
logger.debug("Magic Hit RedisCache: %s" % self.pk_list)
return DictWrapper(obj)
kwargs = {
self.args: [self.value],
}
logger.debug("WeakRel RpcCall: %s %s=%s" % (self.url, self.args, self.value))
obj = self.call_rpc(self.url, **kwargs)[0]
logger.debug("Magic CacheMiss: %s" % self.pk_list)
obj = self.manager(pk_list=self.pk_list)[0]
self.cache[obj['id']] = obj
return DictWrapper(obj)
def get_queryset(self):
queryset = super(WeakRelManager, self).get_queryset()
"""
:return:
"""
queryset = super(MagicAccessor, self).get_queryset()
if queryset:
logger.debug("WeakRel RedisCache: %s %s" % (self.name, self.value_lst))
logger.debug("Magic Hit RedisCache: %s %s" % (self.name, self.pk_list))
return [DictWrapper(obj) for obj in queryset]
kwargs = {
self.args: self.value_lst,
}
logger.debug("WeakRel RpcCall: %s %s=%s" % (self.url, self.args, self.value_lst))
data = self.call_rpc(self.url, **kwargs)
logger.debug("Magic CacheMiss: %s" % self.pk_list)
data = self.manager(pk_list=self.pk_list)
for obj in data:
self.cache[obj['id']] = obj
......
......@@ -2,7 +2,7 @@ from gm_serializer.logger import logger
from gm_serializer.utils import DictWrapper
class WeakDescriptor(object):
class MagicDescriptor(object):
def __init__(self, field_with_rel):
self.field = field_with_rel
......@@ -30,9 +30,9 @@ class WeakDescriptor(object):
try:
lazy_obj = getattr(instance, self.cache_name)
logger.debug("WeakRel ObjCache: %s:%s" % (self.field.attname, lazy_obj['id']))
logger.debug("Magic Hit ObjCache: %s:%s" % (self.field.attname, lazy_obj['id']))
except AttributeError:
lazy_obj = self.field.relation.get_object()
lazy_obj = self.field.accessor.get_object()
setattr(instance, self.cache_name, lazy_obj)
return lazy_obj
......@@ -44,7 +44,7 @@ class WeakDescriptor(object):
:return:
"""
attname = self.field.attname
relation = self.field.relation
accessor = self.field.accessor
if value is None and self.field.null is False:
raise ValueError("Null value not allowed.")
......@@ -53,12 +53,12 @@ class WeakDescriptor(object):
setattr(instance, self.cache_name, value)
return
setattr(relation, "name", attname)
setattr(relation, "value", value)
setattr(accessor, "name", attname)
setattr(accessor, "pk_list", [value])
setattr(instance, "%s_id" % attname, value)
if hasattr(instance, self.cache_name) and getattr(instance, self.cache_name)['id'] != value:
obj_cache = self.field.relation.get_object()
obj_cache = self.field.manager.get_object()
setattr(instance, "%s_id" % attname, value)
setattr(instance, self.cache_name, obj_cache)
......
......@@ -2,20 +2,20 @@ from django.core import exceptions
from django.db import models
from django.utils.translation import ugettext_lazy as _
from gm_serializer.models.descriptor import WeakDescriptor
from gm_serializer.models.manager import WeakRelManager
from gm_serializer.fields.descriptor import MagicDescriptor
from gm_serializer.fields.accessor import MagicAccessor
INT = 'INT'
VARCHAR = 'VARCHAR'
class WeakRelField(models.Field):
class MagicField(models.Field):
default_error_messages = {
"invalid": _("'%(value)s' is not a valid %(type)")
}
relation_class = WeakRelManager
descriptor_class = WeakDescriptor
accessor_class = MagicAccessor
descriptor_class = MagicDescriptor
def __init__(self, *args, **kwargs):
"""
......@@ -23,15 +23,14 @@ class WeakRelField(models.Field):
:param args:
:param kwargs:
"""
self.url = kwargs.pop('url')
self.type = kwargs.pop('type', int)
self.args = kwargs.pop('args', 'id')
self.batch = kwargs.pop('batch', True)
self.relation = self.relation_class(
url=self.url,
args=self.args,
self.ttl = kwargs.pop('ttl', None)
self.manager = kwargs.pop('manager')
self.accessor = self.accessor_class(
manager=self.manager,
)
super(WeakRelField, self).__init__(*args, **kwargs)
super(MagicField, self).__init__(*args, **kwargs)
def get_internal_type(self):
return "VirtualForeignKey"
......@@ -73,7 +72,7 @@ class WeakRelField(models.Field):
:param kwargs:
:return:
"""
super(WeakRelField, self).contribute_to_class(cls, name, **kwargs)
super(MagicField, self).contribute_to_class(cls, name, **kwargs)
setattr(cls, self.name, self.descriptor_class(self))
def deconstruct(self):
......@@ -81,9 +80,10 @@ class WeakRelField(models.Field):
:param
:return:
"""
name, path, args, kwargs = super(WeakRelField, self).deconstruct()
name, path, args, kwargs = super(MagicField, self).deconstruct()
kwargs.update({
'url': self.url, 'type': self.type, 'args': self.args, 'batch': self.batch
'type': self.type, 'manager': self.manager,
'ttl': self.ttl,
})
return name, path, args, kwargs
from django.db import models
from gm_serializer import fields, serializers
class UserManager(object):
def __init__(self):
self.url = 'api/user/get'
def __call__(self, pk_list):
# mock return user instance from rpc call.
return [{'id': pk, 'username': "username %s" % pk} for pk in pk_list]
class Post(models.Model):
"""
This is demonstrate app model.
To explain how to use gm-serializer in your project.
`weak_rel` is more like a cross-database foreign-key relation.
"""
user = fields.MagicField(type=int, manager=UserManager, ttl=60*60)
title = models.CharField(max_length=128)
content = models.CharField(max_length=255)
create_at = models.DateTimeField(auto_now_add=True)
update_at = models.DateTimeField(auto_now=True)
class PostSerializer(serializers.GmModelSerializer):
user = serializers.MagicField()
class Meta:
model = Post
lazy_fields = ('user',)
fields = ('user', 'title', 'content', 'create_at')
from .field import WeakRelField
from rest_framework.serializers import *
from .serializer import GmModelSerializer
from .field import MagicField
from .validator import BlacklistValidator, FilterWordValidator
import json
from rest_framework import serializers
class MagicField(serializers.Field):
def to_representation(self, value):
return json.loads(json.dumps(value))
def to_internal_value(self, data):
return data
......@@ -42,13 +42,12 @@ class GmModelSerializer(serializers.ModelSerializer):
lazy_field_names = getattr(meta, 'lazy_fields', [])
for field_name in lazy_field_names:
field = getattr(model, field_name).field
rel = field.relation_class(
mgr = field.accessor_class(
name=field_name,
url=field.url,
args=field.args,
value_lst=cls.get_field_value_list(field_name, args[0])
manager=field.manager,
pk_list=cls.get_field_value_list(field_name, args[0])
)
lazy_fields[field_name] = rel.get_queryset
lazy_fields[field_name] = mgr.get_queryset
return lazy_fields
@classmethod
......
from django.test import TestCase
from .models import Post, PostSerializer
class MagicFieldTestCase(TestCase):
def setUp(self):
for i in range(10):
Post.objects.create(user=i, title="article %s" % i, content="body %s" % i)
def test_get_object(self):
post = Post.objects.first()
self.assertEqual(post.user, {'id': 0, 'username': 'username 0'})
def test_get_queryset(self):
"""
pass
:return:
"""
posts = Post.objects.all()[:10]
serializer = PostSerializer(posts, many=True)
for idx, obj in enumerate(serializer.data):
self.assertEqual(obj['user'], {'id': idx, 'username': 'username %s' % idx})
def test_get_queryset_lazy(self):
posts = Post.objects.all()[:10]
serializer = PostSerializer(posts, many=True, lazy=True)
for idx, obj in enumerate(serializer.data):
self.assertEqual(obj['user'], idx)
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "serializer.settings")
try:
from django.core.management import execute_from_command_line
except ImportError:
# The above import may fail for some other reason. Ensure that the
# issue is really that Django is missing to avoid masking other
# exceptions on Python 2.
try:
import django
except ImportError:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
)
raise
execute_from_command_line(sys.argv)
"""
Django settings for serializer project.
Generated by 'django-admin startproject' using Django 1.11.2.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'v*(%hx%(ylwaf#a64q$^-*g@93qg6acn!fr)y%dfxgar*18soo'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'gm_serializer'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'serializer.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'serializer.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/static/'
REDIS = {
'gm_serializer': {'host': 'localhost', 'port': 6379, 'db': 1},
}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(asctime)s %(levelname)s %(module)s.%(funcName)s Line:%(lineno)d %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'
},
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
},
'loggers': {
'gm_serializer': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': True,
}
},
}
"""serializer URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.conf.urls import url, include
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
urlpatterns = [
url(r'^admin/', admin.site.urls),
]
"""
WSGI config for serializer project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "serializer.settings")
application = get_wsgi_application()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment