1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function
import string
import inspect
import functools
try:
import cPickle as pickle
except ImportError:
import pickle
import redis
from django.conf import settings
from utils.common import big_data_iter
from qa.cache.cache_v2 import RedisWrapper # todo RedisWrapper 需要独立出来
class ViewRecordBase(object):
def __init__(self, model_name, redis_conf):
self.model_name = model_name
self.__pool = redis.ConnectionPool(**redis_conf)
self.client = redis.StrictRedis(connection_pool=self.__pool)
def __getitem__(self, item):
return self.client.hget(self.model_name, item) or 0
def __setitem__(self, key, value):
return self.client.hset(self.model_name, key, value)
def view_hmget(self, item_list):
result = []
for _ids in big_data_iter(item_list):
_vs = self.client.hmget(self.model_name, _ids)
result.extend(_vs)
return result
class RedisCache(object):
def __init__(self, client):
self.client = client
def get(self, key):
data = self.client.get(key)
if data is not None:
data = pickle.loads(data)
return data
def set(self, key, value, expire=60):
data = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)
self.client.set(key, data, expire)
def invalidate(self, key):
self.client.delete(key)
def __call__(self, key_str, key_args=None, expire=60):
def _cache(fn):
@functools.wraps(fn)
def _wrap(*args, **kwargs):
argvalues = inspect.getcallargs(fn, *args, **kwargs)
cache_key = self.get_cache_key(key_str, key_args, argvalues)
cache_data = self.get(cache_key)
if cache_data is None:
cache_data = fn(*args, **kwargs)
self.set(cache_key, cache_data, expire)
return cache_data
return _wrap
return _cache
@classmethod
def get_key_args(cls, key_str):
"""从`key_str`里自动获取args"""
args = []
for item in string.Formatter().parse(key_str):
if item[1] is None:
continue
if not item[1]:
raise Exception(u'不能使用位置参数 如{}, {0}')
args.append(item[1])
if not args:
return None
return args
@classmethod
def get_cache_key(cls, key_str, key_args=None, argvalues=None):
"""生成缓存key
:param key_str: @str, 缓存key, 如果有动态参数, 在`key_args`里指定, 字符串按`Format String Syntax`
:param key_args: @list, 缓存key的参数名k列表，列表的元素有三种写法:
1. 元素是字符串，说明key_str 的参数名和取值的参数名一致，直接获取
2. 元素是列表，长度为2
第一个元素是字符串，为key_str 的参数名和取值的参数名
第二个元素是函数，以获取到的值为参数的函数
3. 元素是列表，长度为3
第一个元素和第二个元素是字符串，分别表示key_str的参数名和取值的参数名
第三个元素是函数，以获取到的值为参数的函数
[key1, key2, key3...]
[(key1, fn1), (key2, fn2)...]
[(str_key1, val_key1, fn1), (str_key2, val_key2, fn2)...]
:param argvalues: @dict 参数值
"""
if key_args is None:
key_args = cls.get_key_args(key_str)
if key_args is not None and isinstance(key_args, (list, tuple)):
params = {}
for item in key_args:
if isinstance(item, (list, tuple)):
item_len = len(item)
if item_len == 2:
# item 有两个元素，string 里的参数名与取值的参数名一致
k, fnk = item[:2]
str_k = val_k = k
elif item_len >= 3:
# item 有三个元素，按(str参数名, 取值参数名, 处理函数)来处理
str_k, val_k, fnk = item[:3]
params[str_k] = fnk(argvalues[val_k])
else:
str_k = val_k = item
params[str_k] = argvalues[val_k]
cache_key = key_str.format(**params)
else:
cache_key = key_str
return cache_key
common_cache = RedisWrapper('__common_cache', settings.REDIS_TOPIC_1ST['page_cache'])
picture_tool_cache = RedisWrapper('__picture_tool', settings.REDIS_TOPIC_1ST['page_cache'])
common_cache_wrapper = RedisCache(common_cache)