Dev (#2544)

* [Update] 更新翻译 (#2541)

* [Update] 更新缓存机制

* [Update] 增加task最大允许事件，并设置命令最大运行时间为60s

apps/assets/templates/assets/asset_asset_user_list.html

@@ -46,7 +46,7 @@
                                         <tr>
                                             <th class="text-center"><input type="checkbox" class="ipt_check_all"></th>
                                             <th class="text-center">{% trans 'Username' %}</th>
-                                            <th class="text-center">{% trans 'Version' %}</th>
+                                            <th class="text-center">{% trans 'Password version' %}</th>
                                             <th class="text-center">{% trans 'Reachable' %}</th>
                                             <th class="text-center">{% trans 'Date updated' %}</th>
                                             <th class="text-center">{% trans 'Action' %}</th>

apps/common/permissions.py

@@ -70,6 +70,14 @@ class IsCurrentUserOrReadOnly(permissions.BasePermission):
         return obj == request.user
 
 
+class LoginRequiredMixin(UserPassesTestMixin):
+    def test_func(self):
+        if self.request.user.is_authenticated:
+            return True
+        else:
+            return False
+
+
 class AdminUserRequiredMixin(UserPassesTestMixin):
     def test_func(self):
         if not self.request.user.is_authenticated:

apps/jumpserver/settings.py

@@ -475,6 +475,7 @@ CELERY_WORKER_REDIRECT_STDOUTS = True
 CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
 # CELERY_WORKER_HIJACK_ROOT_LOGGER = True
 CELERY_WORKER_MAX_TASKS_PER_CHILD = 40
+CELERY_TASK_SOFT_TIME_LIMIT = 3600
 
 # Cache use redis
 CACHES = {

apps/ops/tasks.py

@@ -4,6 +4,7 @@ import subprocess
 
 from django.conf import settings
 from celery import shared_task, subtask
+from celery.exceptions import SoftTimeLimitExceeded
 from django.utils import timezone
 
 from common.utils import get_logger, get_object_or_none
@@ -38,11 +39,14 @@ def run_ansible_task(tid, callback=None, **kwargs):
         logger.error("No task found")
 
 
-@shared_task
+@shared_task(soft_time_limit=60)
 def run_command_execution(cid, **kwargs):
     execution = get_object_or_none(CommandExecution, id=cid)
     if execution:
-        execution.run()
+        try:
+            execution.run()
+        except SoftTimeLimitExceeded:
+            print("HLLL")
     else:
         logger.error("Not found the execution id: {}".format(cid))
 

apps/ops/templates/ops/command_execution_create.html

@@ -5,17 +5,17 @@
 
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/ztree/awesomeStyle/awesome.css' %}" rel="stylesheet">
+    <link rel="stylesheet" href="{% static 'js/plugins/xterm/xterm.css' %}" />
+    <link href="{% static 'css/plugins/codemirror/codemirror.css' %}" rel="stylesheet">
+    <link href="{% static 'css/plugins/codemirror/ambiance.css' %}" rel="stylesheet">
+    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <script type="text/javascript" src="{% static 'js/plugins/ztree/jquery.ztree.all.min.js' %}"></script>
     <script type="text/javascript" src="{% static 'js/plugins/ztree/jquery.ztree.exhide.min.js' %}"></script>
     <script src="{% static 'js/jquery.form.min.js' %}"></script>
     <script src="{% static 'js/plugins/xterm/xterm.js' %}"></script>
-    <link rel="stylesheet" href="{% static 'js/plugins/xterm/xterm.css' %}" />
     <script src="{% static 'js/plugins/xterm/addons/fit/fit.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/codemirror.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/mode/shell/shell.js' %}"></script>
-    <link href="{% static 'css/plugins/codemirror/codemirror.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/codemirror/ambiance.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
 	<style type="text/css">
         .xterm .xterm-screen canvas {
@@ -82,6 +82,7 @@
 <script>
 var zTree, show = 0;
 var systemUserId = null;
+var treeUrl = "{% url 'api-perms:my-nodes-assets-as-tree' %}?cache_policy=1";
 
 function initTree() {
     var setting = {
@@ -110,14 +111,17 @@ function initTree() {
             onCheck: onCheck
         }
     };
-    var url = "{% url 'api-perms:my-nodes-assets-as-tree' %}?cache_policy=1";
     if (systemUserId) {
-        url += '&system_user=' + systemUserId
+        url = treeUrl + '&system_user=' + systemUserId
     }
 
     $.get(url, function(data, status){
         $.fn.zTree.init($("#assetTree"), setting, data);
         zTree = $.fn.zTree.getZTreeObj("assetTree");
+        rootNodeAddDom(zTree, function () {
+            treeUrl = treeUrl.replace('cache_policy=1', 'cache_policy=2');
+            initTree();
+        });
     });
 }
 
@@ -250,7 +254,8 @@ function execute() {
         method: 'POST',
         flash_message: false,
         success: function (resp) {
-            term.write("{% trans 'Pending' %}" + "...\r\n");
+            var msg = "{% trans 'Pending' %}";
+            term.write(msg + "...\r\n");
             log_url = resp.log_url;
             int = setInterval(function () {
                 writeExecutionOutput()

apps/ops/views/command.py

@@ -5,6 +5,7 @@ from django.utils.translation import ugettext as _
 from django.conf import settings
 from django.views.generic import ListView, TemplateView
 
+from common.permissions import AdminUserRequiredMixin, LoginRequiredMixin
 from common.mixins import DatetimeSearchMixin
 from ..models import CommandExecution
 from ..forms import CommandExecutionForm
@@ -15,7 +16,7 @@ __all__ = [
 ]
 
 
-class CommandExecutionListView(DatetimeSearchMixin, ListView):
+class CommandExecutionListView(AdminUserRequiredMixin, DatetimeSearchMixin, ListView):
     template_name = 'ops/command_execution_list.html'
     model = CommandExecution
     paginate_by = settings.DISPLAY_PER_PAGE
@@ -50,7 +51,7 @@ class CommandExecutionListView(DatetimeSearchMixin, ListView):
         return super().get_context_data(**kwargs)
 
 
-class CommandExecutionStartView(TemplateView):
+class CommandExecutionStartView(LoginRequiredMixin, TemplateView):
     template_name = 'ops/command_execution_create.html'
     form_class = CommandExecutionForm
 

apps/perms/api/user_group_permission.py

@@ -108,7 +108,7 @@ class UserGroupGrantedNodesWithAssetsAsTreeApi(ListAPIView):
         group = get_object_or_404(UserGroup, id=user_group_id)
         util = AssetPermissionUtil(group)
         if self.system_user_id:
-            util.filter_permission_with_system_user(system_user=self.system_user_id)
+            util.filter_permissions(system_users=self.system_user_id)
         nodes = util.get_nodes_with_assets()
         for node, assets in nodes.items():
             data = parse_node_to_tree_node(node)

apps/perms/api/user_permission.py

@@ -35,10 +35,11 @@ __all__ = [
 ]
 
 
-class UserPermissionMixin:
+class UserPermissionCacheMixin:
     cache_policy = '0'
     RESP_CACHE_KEY = '_PERMISSION_RESPONSE_CACHE_{}'
     CACHE_TIME = settings.ASSETS_PERM_CACHE_TIME
+    _object = None
 
     @staticmethod
     def change_org_if_need(request, kwargs):
@@ -51,56 +52,82 @@ class UserPermissionMixin:
     def get_object(self):
         return None
 
-    def get(self, request, *args, **kwargs):
-        self.change_org_if_need(request, kwargs)
-        self.cache_policy = request.GET.get('cache_policy', '0')
+    # 内部使用可控制缓存
+    def _get_object(self):
+        if not self._object:
+            self._object = self.get_object()
+        return self._object
 
-        obj = self.get_object()
-        if obj is None:
-            return super().get(request, *args, **kwargs)
-        request_path_md5 = md5(request.get_full_path().encode()).hexdigest()
-        obj_id = str(obj.id)
-        expire_cache_key = '{}_{}'.format(obj_id, '*')
-        if self.CACHE_TIME <= 0 or \
-                self.cache_policy in AssetPermissionUtil.CACHE_POLICY_MAP[0]:
-            return super().get(request, *args, **kwargs)
-        elif self.cache_policy in AssetPermissionUtil.CACHE_POLICY_MAP[2]:
-            self.expire_cache_response(expire_cache_key)
+    def get_object_id(self):
+        obj = self._get_object()
+        if obj:
+            return str(obj.id)
+        return None
+
+    def get_request_md5(self):
+        full_path = self.request.get_full_path()
+        return md5(full_path.encode()).hexdigest()
 
+    def get_meta_cache_id(self):
+        obj = self._get_object()
         util = AssetPermissionUtil(obj, cache_policy=self.cache_policy)
         meta_cache_id = util.cache_meta.get('id')
-        cache_id = '{}_{}_{}'.format(obj_id, request_path_md5, meta_cache_id)
+        return meta_cache_id
+
+    def get_response_cache_id(self):
+        obj_id = self.get_object_id()
+        request_md5 = self.get_request_md5()
+        meta_cache_id = self.get_meta_cache_id()
+        resp_cache_id = '{}_{}_{}'.format(obj_id, request_md5, meta_cache_id)
+        return resp_cache_id
+
+    def get_response_from_cache(self):
+        resp_cache_id = self.get_response_cache_id()
         # 没有数据缓冲
+        meta_cache_id = self.get_meta_cache_id()
         if not meta_cache_id:
-            response = super().get(request, *args, **kwargs)
-            self.set_cache_response(cache_id, response)
-            return response
-        # 从响应缓冲里获取响应
-        response = self.get_cache_response(cache_id)
-        if not response:
-            response = super().get(request, *args, **kwargs)
-            self.set_cache_response(cache_id, response)
-        return response
-
-    def get_cache_response(self, _id):
-        if not _id:
             return None
-        key = self.RESP_CACHE_KEY.format(_id)
+        # 从响应缓冲里获取响应
+        key = self.RESP_CACHE_KEY.format(resp_cache_id)
         data = cache.get(key)
         if not data:
             return None
-        return Response(data)
+        logger.debug("Get user permission from cache: {}".format(self.get_object()))
+        response = Response(data)
+        return response
 
-    def expire_cache_response(self, _id):
-        key = self.RESP_CACHE_KEY.format(_id)
-        cache.delete(key)
+    def expire_response_cache(self):
+        obj_id = self.get_object_id()
+        expire_cache_id = '{}_{}'.format(obj_id, '*')
+        key = self.RESP_CACHE_KEY.format(expire_cache_id)
+        cache.delete_pattern(key)
 
-    def set_cache_response(self, _id, response):
-        key = self.RESP_CACHE_KEY.format(_id)
+    def set_response_to_cache(self, response):
+        resp_cache_id = self.get_response_cache_id()
+        key = self.RESP_CACHE_KEY.format(resp_cache_id)
         cache.set(key, response.data, self.CACHE_TIME)
 
+    def get(self, request, *args, **kwargs):
+        self.change_org_if_need(request, kwargs)
+        self.cache_policy = request.GET.get('cache_policy', '0')
+
+        obj = self._get_object()
+        if obj is None:
+            return super().get(request, *args, **kwargs)
+
+        if AssetPermissionUtil.is_not_using_cache(self.cache_policy):
+            return super().get(request, *args, **kwargs)
+        elif AssetPermissionUtil.is_refresh_cache(self.cache_policy):
+            self.expire_response_cache()
+
+        resp = self.get_response_from_cache()
+        if not resp:
+            resp = super().get(request, *args, **kwargs)
+            self.set_response_to_cache(resp)
+        return resp
 
-class UserGrantedAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
+
+class UserGrantedAssetsApi(UserPermissionCacheMixin, AssetsFilterMixin, ListAPIView):
     """
     用户授权的所有资产
     """
@@ -110,7 +137,6 @@ class UserGrantedAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
 
     def get_object(self):
         user_id = self.kwargs.get('pk', '')
-
         if user_id:
             user = get_object_or_404(User, id=user_id)
         else:
@@ -134,7 +160,7 @@ class UserGrantedAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
         return super().get_permissions()
 
 
-class UserGrantedNodesApi(UserPermissionMixin, ListAPIView):
+class UserGrantedNodesApi(UserPermissionCacheMixin, ListAPIView):
     """
     查询用户授权的所有节点的API, 如果是超级用户或者是 app，切换到root org
     """
@@ -161,7 +187,7 @@ class UserGrantedNodesApi(UserPermissionMixin, ListAPIView):
         return super().get_permissions()
 
 
-class UserGrantedNodesWithAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
+class UserGrantedNodesWithAssetsApi(UserPermissionCacheMixin, AssetsFilterMixin, ListAPIView):
     """
     用户授权的节点并带着节点下资产的api
     """
@@ -202,17 +228,12 @@ class UserGrantedNodesWithAssetsApi(UserPermissionMixin, AssetsFilterMixin, List
         return super().get_permissions()
 
 
-class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionMixin, ListAPIView):
+class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView):
     serializer_class = TreeNodeSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
     show_assets = True
     system_user_id = None
 
-    def get(self, request, *args, **kwargs):
-        self.show_assets = request.query_params.get('show_assets', '1') == '1'
-        self.system_user_id = request.query_params.get('system_user')
-        return super().get(request, *args, **kwargs)
-
     def get_permissions(self):
         if self.kwargs.get('pk') is None:
             self.permission_classes = (IsValidUser,)
@@ -226,13 +247,19 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionMixin, ListAPIView):
             user = get_object_or_404(User, id=user_id)
         return user
 
+    def list(self, request, *args, **kwargs):
+        resp = super().list(request, *args, **kwargs)
+        return resp
+
     def get_queryset(self):
         queryset = []
+        self.show_assets = self.request.query_params.get('show_assets', '1') == '1'
+        self.system_user_id = self.request.query_params.get('system_user')
         user = self.get_object()
         util = AssetPermissionUtil(user, cache_policy=self.cache_policy)
         if self.system_user_id:
-            util.filter_permission_with_system_user(
-                system_user=self.system_user_id
+            util.filter_permissions(
+                system_users=self.system_user_id
             )
         nodes = util.get_nodes_with_assets()
         for node, assets in nodes.items():
@@ -247,7 +274,7 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionMixin, ListAPIView):
         return queryset
 
 
-class UserGrantedNodeAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIView):
+class UserGrantedNodeAssetsApi(UserPermissionCacheMixin, AssetsFilterMixin, ListAPIView):
     """
     查询用户授权的节点下的资产的api, 与上面api不同的是，只返回某个节点下的资产
     """
@@ -283,7 +310,7 @@ class UserGrantedNodeAssetsApi(UserPermissionMixin, AssetsFilterMixin, ListAPIVi
         return super().get_permissions()
 
 
-class UserGrantedNodeChildrenApi(UserPermissionMixin, ListAPIView):
+class UserGrantedNodeChildrenApi(UserPermissionCacheMixin, ListAPIView):
     """
     获取用户自己授权节点下子节点的api
     """
@@ -369,7 +396,7 @@ class UserGrantedNodeChildrenApi(UserPermissionMixin, ListAPIView):
             return self.get_children_queryset()
 
 
-class ValidateUserAssetPermissionApi(UserPermissionMixin, APIView):
+class ValidateUserAssetPermissionApi(UserPermissionCacheMixin, APIView):
     permission_classes = (IsOrgAdminOrAppUser,)
 
     def get(self, request, *args, **kwargs):

apps/perms/utils.py

@@ -3,6 +3,9 @@
 from __future__ import absolute_import, unicode_literals
 import uuid
 from collections import defaultdict
+import json
+from hashlib import md5
+
 from django.utils import timezone
 from django.db.models import Q
 from django.core.cache import cache
@@ -101,8 +104,8 @@ class AssetPermissionUtil:
         "SystemUser": get_node_permissions,
     }
 
-    CACHE_KEY = '_ASSET_PERM_CACHE_{}_{}'
-    CACHE_META_KEY = '_ASSET_PERM_META_KEY_{}'
+    CACHE_KEY_PREFIX = '_ASSET_PERM_CACHE_'
+    CACHE_META_KEY_PREFIX = '_ASSET_PERM_META_KEY_'
     CACHE_TIME = settings.ASSETS_PERM_CACHE_TIME
     CACHE_POLICY_MAP = (('0', 'never'), ('1', 'using'), ('2', 'refresh'))
 
@@ -110,11 +113,31 @@ class AssetPermissionUtil:
         self.object = obj
         self.obj_id = str(obj.id)
         self._permissions = None
+        self._permissions_id = None  # 标记_permission的唯一值
         self._assets = None
+        self._filter_id = 'None'  # 当通过filter更改 permission是标记
         self.cache_policy = cache_policy
-        self.node_key = self.CACHE_KEY.format(self.obj_id, 'NODES_WITH_ASSETS')
-        self.asset_key = self.CACHE_KEY.format(self.obj_id, 'ASSETS')
-        self.system_key = self.CACHE_KEY.format(self.obj_id, 'SYSTEM_USER')
+
+    @classmethod
+    def is_not_using_cache(cls, cache_policy):
+        return cls.CACHE_TIME == 0 or cache_policy in cls.CACHE_POLICY_MAP[0]
+
+    @classmethod
+    def is_using_cache(cls, cache_policy):
+        return cls.CACHE_TIME != 0 and cache_policy in cls.CACHE_POLICY_MAP[1]
+
+    @classmethod
+    def is_refresh_cache(cls, cache_policy):
+        return cache_policy in cls.CACHE_POLICY_MAP[2]
+
+    def _is_not_using_cache(self):
+        return self.is_not_using_cache(self.cache_policy)
+
+    def _is_using_cache(self):
+        return self.is_using_cache(self.cache_policy)
+
+    def _is_refresh_cache(self):
+        return self.is_refresh_cache(self.cache_policy)
 
     @property
     def permissions(self):
@@ -126,8 +149,10 @@ class AssetPermissionUtil:
         self._permissions = permissions
         return permissions
 
-    def filter_permission_with_system_user(self, system_user):
-        self._permissions = self.permissions.filter(system_users=system_user)
+    def filter_permissions(self, **filters):
+        filters_json = json.dumps(filters, sort_keys=True)
+        self._permissions = self.permissions.filter(**filters)
+        self._filter_id = md5(filters_json.encode()).hexdigest()
 
     def get_nodes_direct(self):
         """
@@ -169,6 +194,25 @@ class AssetPermissionUtil:
         self._assets = assets
         return self._assets
 
+    def get_cache_key(self, resource):
+        cache_key = self.CACHE_KEY_PREFIX + '{obj_id}_{filter_id}_{resource}'
+        return cache_key.format(
+            obj_id=self.obj_id, filter_id=self._filter_id,
+            resource=resource
+        )
+
+    @property
+    def node_key(self):
+        return self.get_cache_key('NODES_WITH_ASSETS')
+
+    @property
+    def asset_key(self):
+        return self.get_cache_key('ASSETS')
+
+    @property
+    def system_key(self):
+        return self.get_cache_key('SYSTEM_USER')
+
     def get_assets_from_cache(self):
         cached = cache.get(self.asset_key)
         if not cached:
@@ -177,9 +221,9 @@ class AssetPermissionUtil:
         return cached
 
     def get_assets(self):
-        if self.CACHE_TIME <= 0 or self.cache_policy in self.CACHE_POLICY_MAP[1]:
+        if self._is_not_using_cache():
             return self.get_assets_from_cache()
-        elif self.cache_policy in self.CACHE_POLICY_MAP[2]:
+        elif self._is_refresh_cache():
             self.expire_cache()
             return self.get_assets_from_cache()
         else:
@@ -206,9 +250,9 @@ class AssetPermissionUtil:
         return cached
 
     def get_nodes_with_assets(self):
-        if self.CACHE_TIME <= 0 or self.cache_policy in self.CACHE_POLICY_MAP[1]:
+        if self._is_using_cache():
             return self.get_nodes_with_assets_from_cache()
-        elif self.cache_policy in self.CACHE_POLICY_MAP[2]:
+        elif self._is_refresh_cache():
             self.expire_cache()
             return self.get_nodes_with_assets_from_cache()
         else:
@@ -229,21 +273,28 @@ class AssetPermissionUtil:
         return cached
 
     def get_system_users(self):
-        if self.CACHE_TIME <= 0 or self.cache_policy in self.CACHE_POLICY_MAP[1]:
+        if self._is_using_cache():
             return self.get_system_user_from_cache()
-        elif self.cache_policy in self.CACHE_POLICY_MAP[2]:
+        elif self._is_refresh_cache():
             self.expire_cache()
             return self.get_system_user_from_cache()
         else:
             return self.get_system_user_without_cache()
 
+    def get_meta_cache_key(self):
+        cache_key = self.CACHE_META_KEY_PREFIX + '{obj_id}_{filter_id}'
+        key = cache_key.format(
+            obj_id=str(self.object.id), filter_id=self._filter_id
+        )
+        return key
+
     @property
     def cache_meta(self):
-        key = self.CACHE_META_KEY.format(str(self.object.id))
+        key = self.get_meta_cache_key()
         return cache.get(key) or {}
 
-    def set_cache_meta(self):
-        key = self.CACHE_META_KEY.format(str(self.object.id))
+    def set_meta_to_cache(self):
+        key = self.get_meta_cache_key()
         meta = {
             'id': str(uuid.uuid4()),
             'datetime': timezone.now(),
@@ -252,8 +303,9 @@ class AssetPermissionUtil:
         cache.set(key, meta, self.CACHE_TIME)
 
     def expire_cache_meta(self):
-        key = self.CACHE_META_KEY.format(str(self.object.id))
-        cache.delete(key)
+        cache_key = self.CACHE_META_KEY_PREFIX + '{obj_id}_*'
+        key = cache_key.format(obj_id=str(self.object.id))
+        cache.delete_pattern(key)
 
     def update_cache(self):
         assets = self.get_assets_without_cache()
@@ -262,7 +314,7 @@ class AssetPermissionUtil:
         cache.set(self.asset_key, assets, self.CACHE_TIME)
         cache.set(self.node_key, nodes, self.CACHE_TIME)
         cache.set(self.system_key, system_users, self.CACHE_TIME)
-        self.set_cache_meta()
+        self.set_meta_to_cache()
 
     def expire_cache(self):
         """
@@ -270,17 +322,19 @@ class AssetPermissionUtil:
         缓存，以免造成不统一的情况
         :return:
         """
-        key = self.CACHE_KEY.format(str(self.object.id), '*')
+        cache_key = self.CACHE_KEY_PREFIX + '{obj_id}_*'
+        key = cache_key.format(obj_id='*')
         cache.delete_pattern(key)
         self.expire_cache_meta()
 
-    def expire_all_cache_meta(self):
-        key = self.CACHE_META_KEY.format('*')
+    @classmethod
+    def expire_all_cache_meta(cls):
+        key = cls.CACHE_META_KEY_PREFIX + '*'
         cache.delete_pattern(key)
 
     @classmethod
     def expire_all_cache(cls):
-        key = cls.CACHE_KEY.format('*', '*')
+        key = cls.CACHE_KEY_PREFIX + '*'
         cache.delete_pattern(key)