[Update] 更新缓存机制

apps/ops/templates/ops/command_execution_create.html

@@ -5,17 +5,17 @@
 
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/ztree/awesomeStyle/awesome.css' %}" rel="stylesheet">
+    <link rel="stylesheet" href="{% static 'js/plugins/xterm/xterm.css' %}" />
+    <link href="{% static 'css/plugins/codemirror/codemirror.css' %}" rel="stylesheet">
+    <link href="{% static 'css/plugins/codemirror/ambiance.css' %}" rel="stylesheet">
+    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <script type="text/javascript" src="{% static 'js/plugins/ztree/jquery.ztree.all.min.js' %}"></script>
     <script type="text/javascript" src="{% static 'js/plugins/ztree/jquery.ztree.exhide.min.js' %}"></script>
     <script src="{% static 'js/jquery.form.min.js' %}"></script>
     <script src="{% static 'js/plugins/xterm/xterm.js' %}"></script>
-    <link rel="stylesheet" href="{% static 'js/plugins/xterm/xterm.css' %}" />
     <script src="{% static 'js/plugins/xterm/addons/fit/fit.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/codemirror.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/mode/shell/shell.js' %}"></script>
-    <link href="{% static 'css/plugins/codemirror/codemirror.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/codemirror/ambiance.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
 	<style type="text/css">
         .xterm .xterm-screen canvas {

apps/perms/api/user_group_permission.py

@@ -108,7 +108,7 @@ class UserGroupGrantedNodesWithAssetsAsTreeApi(ListAPIView):
         group = get_object_or_404(UserGroup, id=user_group_id)
         util = AssetPermissionUtil(group)
         if self.system_user_id:
-            util.filter_permission_with_system_user(system_user=self.system_user_id)
+            util.filter_permissions(system_users=self.system_user_id)
         nodes = util.get_nodes_with_assets()
         for node, assets in nodes.items():
             data = parse_node_to_tree_node(node)

apps/perms/api/user_permission.py

@@ -35,10 +35,11 @@ __all__ = [
 ]
 
 
-class UserPermissionMixin:
+class UserPermissionCacheMixin:
     cache_policy = '0'
     RESP_CACHE_KEY = '_PERMISSION_RESPONSE_CACHE_{}'
     CACHE_TIME = settings.ASSETS_PERM_CACHE_TIME
+    _object = None
 
     @staticmethod
     def change_org_if_need(request, kwargs):
@@ -51,56 +52,82 @@ class UserPermissionMixin:
     def get_object(self):
         return None
 
-    def get(self, request, *args, **kwargs):
-        self.change_org_if_need(request, kwargs)
-        self.cache_policy = request.GET.get('cache_policy', '0')
+    # 内部使用可控制缓存
+    def _get_object(self):
+        if not self._object:
+            self._object = self.get_object()
+        return self._object
 
-        obj = self.get_object()
-        if obj is None:
-            return super().get(request, *args, **kwargs)
-        request_path_md5 = md5(request.get_full_path().encode()).hexdigest()
-        obj_id = str(obj.id)
-        expire_cache_key = '{}_{}'.format(obj_id, '*')
-        if self.CACHE_TIME <= 0 or \
-                self.cache_policy in AssetPermissionUtil.CACHE_POLICY_MAP[0]:
-            return super().get(request, *args, **kwargs)
-        elif self.cache_policy in AssetPermissionUtil.CACHE_POLICY_MAP[2]:
-            self.expire_cache_response(expire_cache_key)
+    def get_object_id(self):
+        obj = self._get_object()
+        if obj:
+            return str(obj.id)
+        return None
+
+    def get_request_md5(self):
+        full_path = self.request.get_full_path()
+        return md5(full_path.encode()).hexdigest()
 
+    def get_meta_cache_id(self):
+        obj = self._get_object()
         util = AssetPermissionUtil(obj, cache_policy=self.cache_policy)
         meta_cache_id = util.cache_meta.get('id')
-        cache_id = '{}_{}_{}'.format(obj_id, request_path_md5, meta_cache_id)
+        return meta_cache_id
+
+    def get_response_cache_id(self):
+        obj_id = self.get_object_id()
+        request_md5 = self.get_request_md5()
+        meta_cache_id = self.get_meta_cache_id()
+        resp_cache_id = '{}_{}_{}'.format(obj_id, request_md5, meta_cache_id)
+        return resp_cache_id
+
+    def get_response_from_cache(self):
+        resp_cache_id = self.get_response_cache_id()
         # 没有数据缓冲
+        meta_cache_id = self.get_meta_cache_id()
         if not meta_cache_id:
-            response = super().get(request, *args, **kwargs)
-            self.set_cache_response(cache_id, response)
-            return response
-        # 从响应缓冲里获取响应
-        response = self.get_cache_response(cache_id)
-        if not response:
-            response = super().get(request, *args, **kwargs)
-            self.set_cache_response(cache_id, response)
-        return response
-
-    def get_cache_response(self, _id):
-        if not _id:
             return None
-        key = self.RESP_CACHE_KEY.format(_id)
+        # 从响应缓冲里获取响应
+        key = self.RESP_CACHE_KEY.format(resp_cache_id)
         data = cache.get(key)
         if not data:
             return None
-        return Response(data)
+        logger.debug("Get user permission from cache: {}".format(self.get_object()))
+        response = Response(data)
+        return response
 
-    def expire_cache_response(self, _id):
-        key = self.RESP_CACHE_KEY.format(_id)
-        cache.delete(key)
+    def expire_response_cache(self):
+        obj_id = self.get_object_id()
+        expire_cache_id = '{}_{}'.format(obj_id, '*')
+        key = self.RESP_CACHE_KEY.format(expire_cache_id)
+        cache.delete_pattern(key)
 
-    def set_cache_response(self, _id, response):
-        key = self.RESP_CACHE_KEY.format(_id)
+    def set_response_to_cache(self, response):
+        resp_cache_id = self.get_response_cache_id()
+        key = self.RESP_CACHE_KEY.format(resp_cache_id)
         cache.set(key, response.data, self.CACHE_TIME)
 
+    def get(self, request, *args, **kwargs):
+        self.change_org_if_need(request, kwargs)
+        self.cache_policy = request.GET.get('cache_policy', '0')
+
+        obj = self._get_object()
+        if obj is None:
+            return super().get(request, *args, **kwargs)
+
+        if AssetPermissionUtil.is_not_using_cache(self.cache_policy):
+            return super().get(request, *args, **kwargs)
+        elif AssetPermissionUtil.is_refresh_cache(self.cache_policy):
+            self.expire_response_cache()
+
+        resp = self.get_response_from_cache()
+        if not resp:
+            resp = super().get(request, *args, **kwargs)
+            self.set_response_to_cache(resp)
+        return resp
 
-class UserGrantedAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
+
+class UserGrantedAssetsApi(UserPermissionCacheMixin, AssetsFilterMixin, ListAPIView):
     """
     用户授权的所有资产
     """
@@ -110,7 +137,6 @@ class UserGrantedAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
 
     def get_object(self):
         user_id = self.kwargs.get('pk', '')
-
         if user_id:
             user = get_object_or_404(User, id=user_id)
         else:
@@ -134,7 +160,7 @@ class UserGrantedAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
         return super().get_permissions()
 
 
-class UserGrantedNodesApi(UserPermissionMixin, ListAPIView):
+class UserGrantedNodesApi(UserPermissionCacheMixin, ListAPIView):
     """
     查询用户授权的所有节点的API, 如果是超级用户或者是 app，切换到root org
     """
@@ -161,7 +187,7 @@ class UserGrantedNodesApi(UserPermissionMixin, ListAPIView):
         return super().get_permissions()
 
 
-class UserGrantedNodesWithAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
+class UserGrantedNodesWithAssetsApi(UserPermissionCacheMixin, AssetsFilterMixin, ListAPIView):
     """
     用户授权的节点并带着节点下资产的api
     """
@@ -202,17 +228,12 @@ class UserGrantedNodesWithAssetsApi(UserPermissionMixin, AssetsFilterMixin, List
         return super().get_permissions()
 
 
-class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionMixin, ListAPIView):
+class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView):
     serializer_class = TreeNodeSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
     show_assets = True
     system_user_id = None
 
-    def get(self, request, *args, **kwargs):
-        self.show_assets = request.query_params.get('show_assets', '1') == '1'
-        self.system_user_id = request.query_params.get('system_user')
-        return super().get(request, *args, **kwargs)
-
     def get_permissions(self):
         if self.kwargs.get('pk') is None:
             self.permission_classes = (IsValidUser,)
@@ -226,13 +247,19 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionMixin, ListAPIView):
             user = get_object_or_404(User, id=user_id)
         return user
 
+    def list(self, request, *args, **kwargs):
+        resp = super().list(request, *args, **kwargs)
+        return resp
+
     def get_queryset(self):
         queryset = []
+        self.show_assets = self.request.query_params.get('show_assets', '1') == '1'
+        self.system_user_id = self.request.query_params.get('system_user')
         user = self.get_object()
         util = AssetPermissionUtil(user, cache_policy=self.cache_policy)
         if self.system_user_id:
-            util.filter_permission_with_system_user(
-                system_user=self.system_user_id
+            util.filter_permissions(
+                system_users=self.system_user_id
             )
         nodes = util.get_nodes_with_assets()
         for node, assets in nodes.items():
@@ -247,7 +274,7 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionMixin, ListAPIView):
         return queryset
 
 
-class UserGrantedNodeAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
+class UserGrantedNodeAssetsApi(UserPermissionCacheMixin, AssetsFilterMixin, ListAPIView):
     """
     查询用户授权的节点下的资产的api, 与上面api不同的是，只返回某个节点下的资产
     """
@@ -283,7 +310,7 @@ class UserGrantedNodeAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIVi
         return super().get_permissions()
 
 
-class UserGrantedNodeChildrenApi(UserPermissionMixin, ListAPIView):
+class UserGrantedNodeChildrenApi(UserPermissionCacheMixin, ListAPIView):
     """
     获取用户自己授权节点下子节点的api
     """
@@ -369,7 +396,7 @@ class UserGrantedNodeChildrenApi(UserPermissionMixin, ListAPIView):
             return self.get_children_queryset()
 
 
-class ValidateUserAssetPermissionApi(UserPermissionMixin, APIView):
+class ValidateUserAssetPermissionApi(UserPermissionCacheMixin, APIView):
     permission_classes = (IsOrgAdminOrAppUser,)
 
     def get(self, request, *args, **kwargs):

apps/perms/utils.py

@@ -3,6 +3,9 @@
 from __future__ import absolute_import, unicode_literals
 import uuid
 from collections import defaultdict
+import json
+from hashlib import md5
+
 from django.utils import timezone
 from django.db.models import Q
 from django.core.cache import cache
@@ -101,8 +104,8 @@ class AssetPermissionUtil:
         "SystemUser": get_node_permissions,
     }
 
-    CACHE_KEY = '_ASSET_PERM_CACHE_{}_{}'
-    CACHE_META_KEY = '_ASSET_PERM_META_KEY_{}'
+    CACHE_KEY = '_ASSET_PERM_CACHE_{obj_id}_{filter_id}_{resource}'
+    CACHE_META_KEY = '_ASSET_PERM_META_KEY_{obj_id}_{filter_id}'
     CACHE_TIME = settings.ASSETS_PERM_CACHE_TIME
     CACHE_POLICY_MAP = (('0', 'never'), ('1', 'using'), ('2', 'refresh'))
 
@@ -110,11 +113,31 @@ class AssetPermissionUtil:
         self.object = obj
         self.obj_id = str(obj.id)
         self._permissions = None
+        self._permissions_id = None  # 标记_permission的唯一值
         self._assets = None
+        self._filter_id = ''  # 当通过filter更改 permission是标记
         self.cache_policy = cache_policy
-        self.node_key = self.CACHE_KEY.format(self.obj_id, 'NODES_WITH_ASSETS')
-        self.asset_key = self.CACHE_KEY.format(self.obj_id, 'ASSETS')
-        self.system_key = self.CACHE_KEY.format(self.obj_id, 'SYSTEM_USER')
+
+    @classmethod
+    def is_not_using_cache(cls, cache_policy):
+        return cls.CACHE_TIME == 0 or cache_policy in cls.CACHE_POLICY_MAP[0]
+
+    @classmethod
+    def is_using_cache(cls, cache_policy):
+        return cls.CACHE_TIME != 0 and cache_policy in cls.CACHE_POLICY_MAP[1]
+
+    @classmethod
+    def is_refresh_cache(cls, cache_policy):
+        return cache_policy in cls.CACHE_POLICY_MAP[2]
+
+    def _is_not_using_cache(self):
+        return self.is_not_using_cache(self.cache_policy)
+
+    def _is_using_cache(self):
+        return self.is_using_cache(self.cache_policy)
+
+    def _is_refresh_cache(self):
+        return self.is_refresh_cache(self.cache_policy)
 
     @property
     def permissions(self):
@@ -126,8 +149,10 @@ class AssetPermissionUtil:
         self._permissions = permissions
         return permissions
 
-    def filter_permission_with_system_user(self, system_user):
-        self._permissions = self.permissions.filter(system_users=system_user)
+    def filter_permissions(self, **filters):
+        filters_json = json.dumps(filters, sort_keys=True)
+        self._permissions = self.permissions.filter(**filters)
+        self._filter_id = md5(filters_json.encode()).hexdigest()
 
     def get_nodes_direct(self):
         """
@@ -169,6 +194,24 @@ class AssetPermissionUtil:
         self._assets = assets
         return self._assets
 
+    def get_cache_key(self, resource):
+        return self.CACHE_KEY.format(
+            obj_id=self.obj_id, filter_id=self._filter_id,
+            resource=resource
+        )
+
+    @property
+    def node_key(self):
+        return self.get_cache_key('NODES_WITH_ASSETS')
+
+    @property
+    def asset_key(self):
+        return self.get_cache_key('ASSETS')
+
+    @property
+    def system_key(self):
+        return self.get_cache_key('SYSTEM_USER')
+
     def get_assets_from_cache(self):
         cached = cache.get(self.asset_key)
         if not cached:
@@ -177,9 +220,9 @@ class AssetPermissionUtil:
         return cached
 
     def get_assets(self):
-        if self.CACHE_TIME <= 0 or self.cache_policy in self.CACHE_POLICY_MAP[1]:
+        if self._is_not_using_cache():
             return self.get_assets_from_cache()
-        elif self.cache_policy in self.CACHE_POLICY_MAP[2]:
+        elif self._is_refresh_cache():
             self.expire_cache()
             return self.get_assets_from_cache()
         else:
@@ -206,9 +249,9 @@ class AssetPermissionUtil:
         return cached
 
     def get_nodes_with_assets(self):
-        if self.CACHE_TIME <= 0 or self.cache_policy in self.CACHE_POLICY_MAP[1]:
+        if self._is_using_cache():
             return self.get_nodes_with_assets_from_cache()
-        elif self.cache_policy in self.CACHE_POLICY_MAP[2]:
+        elif self._is_refresh_cache():
             self.expire_cache()
             return self.get_nodes_with_assets_from_cache()
         else:
@@ -229,21 +272,27 @@ class AssetPermissionUtil:
         return cached
 
     def get_system_users(self):
-        if self.CACHE_TIME <= 0 or self.cache_policy in self.CACHE_POLICY_MAP[1]:
+        if self._is_using_cache():
             return self.get_system_user_from_cache()
-        elif self.cache_policy in self.CACHE_POLICY_MAP[2]:
+        elif self._is_refresh_cache():
             self.expire_cache()
             return self.get_system_user_from_cache()
         else:
             return self.get_system_user_without_cache()
 
+    def get_meta_cache_key(self):
+        key = self.CACHE_META_KEY.format(
+            obj_id=str(self.object.id), filter_id=self._filter_id
+        )
+        return key
+
     @property
     def cache_meta(self):
-        key = self.CACHE_META_KEY.format(str(self.object.id))
+        key = self.get_meta_cache_key()
         return cache.get(key) or {}
 
-    def set_cache_meta(self):
-        key = self.CACHE_META_KEY.format(str(self.object.id))
+    def set_meta_to_cache(self):
+        key = self.get_meta_cache_key()
         meta = {
             'id': str(uuid.uuid4()),
             'datetime': timezone.now(),
@@ -252,7 +301,7 @@ class AssetPermissionUtil:
         cache.set(key, meta, self.CACHE_TIME)
 
     def expire_cache_meta(self):
-        key = self.CACHE_META_KEY.format(str(self.object.id))
+        key = self.get_meta_cache_key()
         cache.delete(key)
 
     def update_cache(self):
@@ -262,7 +311,7 @@ class AssetPermissionUtil:
         cache.set(self.asset_key, assets, self.CACHE_TIME)
         cache.set(self.node_key, nodes, self.CACHE_TIME)
         cache.set(self.system_key, system_users, self.CACHE_TIME)
-        self.set_cache_meta()
+        self.set_meta_to_cache()
 
     def expire_cache(self):
         """