Config (#3502)

* [Update] 修改config

* [Update] 移动存储设置到到terminal中

* [Update] 修改permission 查看

* [Update] pre merge

* [Update] 录像存储

* [Update] 命令存储

* [Update] 添加存储测试可连接性

* [Update] 修改 meta 值的 key 为大写

* [Update] 修改 Terminal 相关 Storage 配置

* [Update] 删除之前获取录像/命令存储的代码

* [Update] 修改导入失败

* [Update] 迁移文件添加default存储

* [Update] 删除之前代码，添加help_text信息

* [Update] 删除之前代码

* [Update] 删除之前代码

* [Update] 抽象命令/录像存储 APIView

* [Update] 抽象命令/录像存储 APIView 1

* [Update] 抽象命令/录像存储 DictField

* [Update] 抽象命令/录像存储列表页面

* [Update] 修复CustomDictField的bug

* [Update] RemoteApp 页面添加 hidden

* [Update] 用户页面添加用户关联授权

* [Update] 修改存储测试可连接性 target

* [Update] 修改配置

* [Update] 修改存储前端 Form 渲染逻辑

* [Update] 修改存储细节

* [Update] 统一存储类型到 const 文件

* [Update] 修改迁移文件及Model，创建默认存储

* [Update] 修改迁移文件及Model初始化默认数据

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 限制删除默认存储配置，只允许创建扩展的存储类型

* [Update] 修改ip字段长度

* [Update] 修改ip字段长度

* [Update] 修改一些css

* [Update] 修改关联

* [Update] 添加操作日志定时清理

* [Update] 修改记录syslog的instance encoder

* [Update] 忽略登录产生的操作日志

* [Update] 限制更新存储时不覆盖原有AK SK 等字段

* [Update] 修改迁移文件添加comment字段

* [Update] 修改迁移文件

* [Update] 添加 comment 字段

* [Update] 修改默认存储no -> null

* [Update] 修改细节

* [Update] 更新翻译（存储配置

* [Update] 修改定时任务注册，修改系统用户资产、节点关系api

* [Update] 添加监控磁盘任务

* [Update] 修改session

* [Update] 拆分serializer

* [Update] 还原setting原来的manager

apps/applications/serializers/remote_app.py

@@ -5,6 +5,7 @@
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
+from common.fields.serializer import CustomMetaDictField
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from .. import const
@@ -16,54 +17,9 @@ __all__ = [
 ]
 
 
-class RemoteAppParamsDictField(serializers.DictField):
-    """
-    RemoteApp field => params
-    """
-    @staticmethod
-    def filter_attribute(attribute, instance):
-        """
-        过滤掉params字段值中write_only特性的key-value值
-        For example, the chrome_password field is not returned when serializing
-        {
-            'chrome_target': 'http://www.jumpserver.org/',
-            'chrome_username': 'admin',
-            'chrome_password': 'admin',
-        }
-        """
-        for field in const.REMOTE_APP_TYPE_MAP_FIELDS[instance.type]:
-            if field.get('write_only', False):
-                attribute.pop(field['name'], None)
-        return attribute
-
-    def get_attribute(self, instance):
-        """
-        序列化时调用
-        """
-        attribute = super().get_attribute(instance)
-        attribute = self.filter_attribute(attribute, instance)
-        return attribute
-
-    @staticmethod
-    def filter_value(dictionary, value):
-        """
-        过滤掉不属于当前app_type所包含的key-value值
-        """
-        app_type = dictionary.get('type', const.REMOTE_APP_TYPE_CHROME)
-        fields = const.REMOTE_APP_TYPE_MAP_FIELDS[app_type]
-        fields_names = [field['name'] for field in fields]
-        no_need_keys = [k for k in value.keys() if k not in fields_names]
-        for k in no_need_keys:
-            value.pop(k)
-        return value
-
-    def get_value(self, dictionary):
-        """
-        反序列化时调用
-        """
-        value = super().get_value(dictionary)
-        value = self.filter_value(dictionary, value)
-        return value
+class RemoteAppParamsDictField(CustomMetaDictField):
+    type_map_fields = const.REMOTE_APP_TYPE_MAP_FIELDS
+    default_type = const.REMOTE_APP_TYPE_CHROME
 
 
 class RemoteAppSerializer(BulkOrgResourceModelSerializer):

apps/applications/templates/applications/remote_app_create_update.html

@@ -19,14 +19,14 @@
         <div class="hr-line-dashed"></div>
 
         {# chrome #}
-        <div class="chrome-fields">
+        <div class="chrome-fields hidden">
         {% bootstrap_field form.chrome_target layout="horizontal" %}
         {% bootstrap_field form.chrome_username layout="horizontal" %}
         {% bootstrap_field form.chrome_password layout="horizontal" %}
         </div>
 
         {# mysql workbench #}
-        <div class="mysql_workbench-fields">
+        <div class="mysql_workbench-fields hidden">
             {% bootstrap_field form.mysql_workbench_ip layout="horizontal" %}
             {% bootstrap_field form.mysql_workbench_name layout="horizontal" %}
             {% bootstrap_field form.mysql_workbench_username layout="horizontal" %}
@@ -34,14 +34,14 @@
         </div>
 
         {# vmware #}
-        <div class="vmware_client-fields">
+        <div class="vmware_client-fields hidden">
         {% bootstrap_field form.vmware_target layout="horizontal" %}
         {% bootstrap_field form.vmware_username layout="horizontal" %}
         {% bootstrap_field form.vmware_password layout="horizontal" %}
         </div>
 
         {# custom #}
-        <div class="custom-fields">
+        <div class="custom-fields hidden">
         {% bootstrap_field form.custom_cmdline layout="horizontal" %}
         {% bootstrap_field form.custom_target layout="horizontal" %}
         {% bootstrap_field form.custom_username layout="horizontal" %}

apps/applications/templates/applications/remote_app_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">
@@ -102,4 +97,4 @@ $(document).ready(function () {
     objectDelete($this, name, the_url, redirect_url);
 })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/api/__init__.py

@@ -2,6 +2,7 @@ from .admin_user import *
 from .asset import *
 from .label import *
 from .system_user import *
+from .system_user_relation import *
 from .node import *
 from .domain import *
 from .cmd_filter import *

apps/assets/api/admin_user.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from django.db import transaction
+from django.db.models import Count
 from django.shortcuts import get_object_or_404
 from rest_framework.response import Response
 from orgs.mixins.api import OrgBulkModelViewSet
@@ -44,6 +45,11 @@ class AdminUserViewSet(OrgBulkModelViewSet):
     serializer_class = serializers.AdminUserSerializer
     permission_classes = (IsOrgAdmin,)
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.annotate(_assets_amount=Count('assets'))
+        return queryset
+
 
 class AdminUserAuthApi(generics.UpdateAPIView):
     model = AdminUser

apps/assets/api/asset_user.py

@@ -114,7 +114,7 @@ class AssetUserExportViewSet(AssetUserViewSet):
     permission_classes = [IsOrgAdminOrAppUser]
 
     def get_permissions(self):
-        if settings.CONFIG.SECURITY_VIEW_AUTH_NEED_MFA:
+        if settings.SECURITY_VIEW_AUTH_NEED_MFA:
             self.permission_classes = [IsOrgAdminOrAppUser, NeedMFAVerify]
         return super().get_permissions()
 
@@ -124,7 +124,7 @@ class AssetUserAuthInfoApi(generics.RetrieveAPIView):
     permission_classes = [IsOrgAdminOrAppUser]
 
     def get_permissions(self):
-        if settings.CONFIG.SECURITY_VIEW_AUTH_NEED_MFA:
+        if settings.SECURITY_VIEW_AUTH_NEED_MFA:
             self.permission_classes = [IsOrgAdminOrAppUser, NeedMFAVerify]
         return super().get_permissions()
 

apps/assets/api/system_user.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 from django.shortcuts import get_object_or_404
-from django.conf import settings
 from rest_framework.response import Response
+from django.db.models import Count
 
 from common.serializers import CeleryTaskSerializer
 from common.utils import get_logger
@@ -50,6 +50,11 @@ class SystemUserViewSet(OrgBulkModelViewSet):
     serializer_class = serializers.SystemUserSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.annotate(_assets_amount=Count('assets'))
+        return queryset
+
 
 class SystemUserAuthInfoApi(generics.RetrieveUpdateDestroyAPIView):
     """

apps/assets/api/system_user_relation.py

+# -*- coding: utf-8 -*-
+#
+from django.db.models import F, Value
+from django.db.models.functions import Concat
+
+from common.permissions import IsOrgAdmin
+from orgs.mixins.api import OrgBulkModelViewSet
+from orgs.utils import current_org
+from .. import models, serializers
+
+__all__ = ['SystemUserAssetRelationViewSet', 'SystemUserNodeRelationViewSet']
+
+
+class RelationMixin(OrgBulkModelViewSet):
+    def get_queryset(self):
+        queryset = self.model.objects.all()
+        org_id = current_org.org_id()
+        if org_id is not None:
+            queryset = queryset.filter(systemuser__org_id=org_id)
+        queryset = queryset.annotate(systemuser_display=Concat(
+            F('systemuser__name'), Value('('), F('systemuser__username'),
+            Value(')')
+        ))
+        return queryset
+
+
+class SystemUserAssetRelationViewSet(RelationMixin):
+    serializer_class = serializers.SystemUserAssetRelationSerializer
+    model = models.SystemUser.assets.through
+    permission_classes = (IsOrgAdmin,)
+    filterset_fields = [
+        'id', 'asset', 'systemuser',
+    ]
+    search_fields = [
+        "id", "asset__hostname", "asset__ip",
+        "systemuser__name", "systemuser__username"
+    ]
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.annotate(
+            asset_display=Concat(
+                F('asset__hostname'), Value('('),
+                F('asset__ip'), Value(')')
+            )
+        )
+        return queryset
+
+
+class SystemUserNodeRelationViewSet(RelationMixin):
+    serializer_class = serializers.SystemUserNodeRelationSerializer
+    model = models.SystemUser.nodes.through
+    permission_classes = (IsOrgAdmin,)
+    filterset_fields = [
+        'id', 'node', 'systemuser',
+    ]
+    search_fields = [
+        "node__value", "systemuser__name", "systemuser_username"
+    ]
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset \
+            .annotate(node_key=F('node__key'))
+        return queryset

apps/assets/models/base.py

@@ -41,6 +41,7 @@ class AssetUser(OrgModelMixin):
     ASSET_USER_CACHE_TIME = 3600 * 24
 
     _prefer = "system_user"
+    _assets_amount = None
 
     @property
     def private_key_obj(self):
@@ -143,6 +144,8 @@ class AssetUser(OrgModelMixin):
 
     @property
     def assets_amount(self):
+        if self._assets_amount is not None:
+            return self._assets_amount
         cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id)
         cached = cache.get(cache_key)
         if not cached:

apps/assets/serializers/system_user.py

@@ -4,8 +4,10 @@ from rest_framework import serializers
 from django.utils.translation import ugettext_lazy as _
 
 from common.serializers import AdaptedBulkListSerializer
+from common.mixins.serializers import BulkSerializerMixin
 from common.utils import ssh_pubkey_gen
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
+from assets.models import Node
 from ..models import SystemUser
 from ..const import (
     GENERAL_FORBIDDEN_SPECIAL_CHARACTERS_PATTERN,
@@ -13,6 +15,12 @@ from ..const import (
 )
 from .base import AuthSerializer, AuthSerializerMixin
 
+__all__ = [
+    'SystemUserSerializer', 'SystemUserAuthSerializer',
+    'SystemUserSimpleSerializer', 'SystemUserAssetRelationSerializer',
+    'SystemUserNodeRelationSerializer',
+]
+
 
 class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     """
@@ -143,4 +151,43 @@ class SystemUserSimpleSerializer(serializers.ModelSerializer):
         fields = ('id', 'name', 'username')
 
 
+class RelationMixin(BulkSerializerMixin, serializers.Serializer):
+    systemuser_display = serializers.ReadOnlyField()
+
+    def get_field_names(self, declared_fields, info):
+        fields = super().get_field_names(declared_fields, info)
+        fields.extend(['systemuser', "systemuser_display"])
+        return fields
+
+    class Meta:
+        list_serializer_class = AdaptedBulkListSerializer
+
+
+class SystemUserAssetRelationSerializer(RelationMixin, serializers.ModelSerializer):
+    asset_display = serializers.ReadOnlyField()
+
+    class Meta(RelationMixin.Meta):
+        model = SystemUser.assets.through
+        fields = [
+            'id', "asset", "asset_display",
+        ]
+
+
+class SystemUserNodeRelationSerializer(RelationMixin, serializers.ModelSerializer):
+    node_display = serializers.SerializerMethodField()
+
+    class Meta(RelationMixin.Meta):
+        model = SystemUser.nodes.through
+        fields = [
+            'id', 'node', "node_display",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.tree = Node.tree()
 
+    def get_node_display(self, obj):
+        if hasattr(obj, 'node_key'):
+            return self.tree.get_node_full_tag(obj.node_key)
+        else:
+            return obj.node.full_value

apps/assets/templates/assets/_asset_list_modal.html

@@ -190,6 +190,16 @@ function setAssetModalOptions(options) {
     assetModalOption = options;
 }
 
+function initAssetTreeModel(selector) {
+    $(selector).parent().find(".select2-selection").on('click', function (e) {
+        if ($(e.target).attr('class') !== 'select2-selection__choice__remove'){
+            e.preventDefault();
+            e.stopPropagation();
+            $("#asset_list_modal").modal();
+        }
+    })
+}
+
 
 $(document).ready(function(){
 

apps/assets/templates/assets/_system_user.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -247,4 +243,4 @@ $(document).ready(function () {
 })
 
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/admin_user_assets.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/admin_user_create_update.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -87,4 +83,4 @@ $(document).ready(function () {
     })
 })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/admin_user_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/asset_bulk_update.html

@@ -32,13 +32,7 @@
 <script>
     $(document).ready(function () {
         $('.select2').select2();
-        $("#id_assets").parent().find(".select2-selection").on('click', function (e) {
-            if ($(e.target).attr('class') !== 'select2-selection__choice__remove'){
-                e.preventDefault();
-                e.stopPropagation();
-                $("#asset_list_modal").modal();
-            }
-        })
+        initAssetTreeModel("#id_assets");
     }).on('click', '.field-tag', function() {
         changeField(this);
     }).on('click', '#change_all', function () {

apps/assets/templates/assets/asset_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
     <link href='{% static "css/plugins/sweetalert/sweetalert.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
     <script src='{% static "js/plugins/sweetalert/sweetalert.min.js" %}'></script>
 {% endblock %}
 

apps/assets/templates/assets/asset_list.html

@@ -382,9 +382,6 @@ $(document).ready(function(){
     var data = {
         'resources': id_list
     };
-    function refreshPage() {
-        setTimeout( function () {window.location.reload();}, 300);
-    }
 
     function reloadTable() {
         asset_table.ajax.reload();

apps/assets/templates/assets/cmd_filter_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/cmd_filter_rule_create_update.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -91,4 +87,4 @@ $(document).ready(function(){
     formSubmit(props);
 })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/cmd_filter_rule_list.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/domain_create_update.html

@@ -25,9 +25,7 @@
 <script type="text/javascript">
 $(document).ready(function () {
     $('.select2').select2().off("select2:open");
-}).on('click', '.select2-selection__rendered', function (e) {
-    e.preventDefault();
-    $("#asset_list_modal").modal();
+    initAssetTreeModel('#id_assets');
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();
@@ -51,4 +49,4 @@ $(document).ready(function () {
     formSubmit(props);
  })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/domain_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">
@@ -138,4 +133,4 @@ $(document).ready(function(){
 })
 ;
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/domain_gateway_list.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/gateway_create_update.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -125,4 +121,4 @@ $(document).ready(function(){
     protocolChange();
 });
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/label_create_update.html

@@ -29,9 +29,7 @@ $(document).ready(function () {
     $('.select2').select2({
         closeOnSelect: false
     })
-}).on('click', '.select2-selection__rendered', function (e) {
-    e.preventDefault();
-    $("#asset_list_modal").modal();
+    initAssetTreeModel("#id_assets");
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();
@@ -55,4 +53,4 @@ $(document).ready(function () {
     formSubmit(props);
 })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/system_user_assets.html

@@ -4,9 +4,13 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
+    <style>
+    .table.node_edit {
+        margin-bottom: 0;
+    }
+    </style>
 {% endblock %}
+
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">
@@ -98,15 +102,8 @@
                                                 </td>
                                             </tr>
                                         </form>
-
-                                        {% for node in system_user.nodes.all|sort %}
-                                        <tr>
-                                          <td ><b class="bdg_node" data-gid={{ node.id }}>{{ node.full_value }}</b></td>
-                                          <td>
-                                              <button class="btn btn-danger pull-right btn-xs btn-remove-from-node" type="button"><i class="fa fa-minus"></i></button>
-                                          </td>
-                                        </tr>
-                                        {% endfor %}
+                                        <table class="table" id="node_list_table">
+                                        </table>
                                         </tbody>
                                     </table>
                                 </div>
@@ -120,91 +117,115 @@
 {% endblock %}
 {% block custom_foot_js %}
 <script>
+var assetsRelationUrl = "{% url 'api-assets:system-users-assets-relation-list' %}";
+var nodesRelationUrl = "{% url 'api-assets:system-users-nodes-relation-list' %}";
 
-function updateSystemUserNode(nodes) {
-    var the_url = "{% url 'api-assets:system-user-detail' pk=system_user.id %}";
-    var body = {
-        nodes: Object.assign([], nodes)
-    };
-    var success = function(data) {
-        // remove all the selected groups from select > option and rendered ul element;
-        $('.select2-selection__rendered').empty();
-        $('#node_selected').val('');
-        $.map(jumpserver.nodes_selected, function(node_name, index) {
-            $('#opt_' + index).remove();
-            // change tr html of user groups.
-            $('.node_edit tbody').append(
-                '<tr>' +
-                '<td><b class="bdg_node" data-gid="' + index + '">' + node_name + '</b></td>' +
-                '<td><button class="btn btn-danger btn-xs pull-right btn-remove-from-node" type="button"><i class="fa fa-minus"></i></button></td>' +
-                '</tr>'
-            )
-        });
-        // clear jumpserver.nodes_selected
-        jumpserver.nodes_selected = {};
-    };
+function getRelationUrl(type) {
+    var theUrl = "";
+    switch (type) {
+        case "asset":
+            theUrl = assetsRelationUrl;
+            break;
+        case "node":
+            theUrl = nodesRelationUrl;
+            break;
+    }
+    return theUrl;
+}
+
+function addObjects(objectsId, type) {
+    if (!objectsId || objectsId.length === 0) {
+        return
+    }
+    var theUrl = getRelationUrl(type);
+    var body = [];
+    objectsId.forEach(function (v) {
+        var data = {systemuser: "{{ object.id }}"};
+        data[type] = v;
+        body.push(data)
+    });
     requestApi({
-        url: the_url,
+        url: theUrl,
         body: JSON.stringify(body),
+        method: "POST",
+        success: reloadPage
+    });
+}
+
+function removeObject(objectId, type, success) {
+    if (!objectId)  {
+        return
+    }
+    var theUrl = getRelationUrl(type);
+    theUrl = setUrlParam(theUrl, 'systemuser', "{{ object.id }}");
+    theUrl = setUrlParam(theUrl, type, objectId);
+    if (!success) {
+        success = reloadPage
+    }
+    requestApi({
+        url: theUrl,
+        method: "DELETE",
         success: success
     });
 }
-jumpserver.nodes_selected = {};
+
+function initNodeTable() {
+    var theUrl = setUrlParam(nodesRelationUrl, "systemuser", "{{ object.id }}");
+    var options = {
+        ele: $('#node_list_table'),
+        toggle: true,
+        columnDefs: [
+             {targets: 1, createdCell: function (td, cellData) {
+                  var removeBtn = '<button class="btn btn-danger pull-right btn-xs btn-remove-from-node" data-uid="UID" type="button"><i class="fa fa-minus"></i></button>'
+                      .replace('UID', cellData);
+		  		  $(td).html(removeBtn);
+             }}
+        ],
+        ajax_url: theUrl,
+        dom: "trp",
+        hideDefaultDefs: true,
+        columns: [
+            {data: "node_display", orderable: false},
+            {data: "node", orderable: false},
+        ],
+    };
+    table = jumpserver.initServerSideDataTable(options);
+    return table
+}
+
 
 $(document).ready(function () {
-	$('.select2').select2()
-    nodesSelect2Init(".nodes-select2")
-	.on('select2:select', function(evt) {
-            var data = evt.params.data;
-            jumpserver.nodes_selected[data.id] = data.text;
-    })
-    .on('select2:unselect', function(evt) {
-        var data = evt.params.data;
-        delete jumpserver.nodes_selected[data.id];
-    });
+	$('.select2').select2();
+    nodesSelect2Init(".nodes-select2");
 	assetUserListUrl = setUrlParam(assetUserListUrl, "system_user_id", "{{ system_user.id }}");
     needPush = true;
     initAssetUserTable();
-
+    initNodeTable();
 })
 .on('click', '.btn-remove-from-node', function() {
     var $this = $(this);
-    var $tr = $this.closest('tr');
-    var $badge = $tr.find('.bdg_node');
-    var gid = $badge.data('gid');
-    var node_name = $badge.html() || $badge.text();
-    $('#groups_selected').append(
-        '<option value="' + gid + '" id="opt_' + gid + '">' + node_name + '</option>'
-    );
-    $tr.remove();
-    var nodes = $('.bdg_node').map(function () {
-        return $(this).data('gid');
-    }).get();
-    updateSystemUserNode(nodes);
+    var nodeId = $(this).data('uid');
+    var success = function() {
+        var $tr = $this.closest('tr');
+        $tr.remove();
+    };
+    removeObject(nodeId, "node", success)
 })
 .on('click', '#btn-add-to-node', function() {
-    if (Object.keys(jumpserver.nodes_selected).length === 0) {
-        return false;
-    }
-    var nodes = $('.bdg_node').map(function() {
-        return $(this).data('gid');
-    }).get();
-    $.map(jumpserver.nodes_selected, function(value, index) {
-        nodes.push(index);
-    });
-    updateSystemUserNode(nodes);
+    var nodes = $("#node_selected").val();
+    addObjects(nodes, "node");
 })
 .on('click', '.btn-push', function () {
-    var the_url = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
+    var theUrl = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
     var error = function (data) {
         alert(data)
     };
     var success = function (data) {
-        var task_id = data.task;
-        showCeleryTaskLog(task_id);
+        var taskId = data.task;
+        showCeleryTaskLog(taskId);
     };
     requestApi({
-        url: the_url,
+        url: theUrl,
         error: error,
         method: 'GET',
         success: success
@@ -213,39 +234,37 @@ $(document).ready(function () {
 .on('click', '.btn-push-auth', function () {
     var $this = $(this);
     var asset_id = $this.data('asset');
-    var the_url = "{% url 'api-assets:system-user-push-to-asset' pk=object.id aid=DEFAULT_PK %}";
-    the_url = the_url.replace("{{ DEFAULT_PK }}", asset_id);
+    var theUrl = "{% url 'api-assets:system-user-push-to-asset' pk=object.id aid=DEFAULT_PK %}";
+    theUrl = theUrl.replace("{{ DEFAULT_PK }}", asset_id);
     var success = function (data) {
-        var task_id = data.task;
-        showCeleryTaskLog(task_id);
+        var taskId = data.task;
+        showCeleryTaskLog(taskId);
     };
     var error = function (data) {
         alert(data)
     };
     requestApi({
-        url: the_url,
+        url: theUrl,
         method: 'GET',
         success: success,
         error: error
     })
 })
 .on('click', '.btn-test-connective', function () {
-    var the_url = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
+    var theUrl = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
     var error = function (data) {
         alert(data)
     };
     var success = function (data) {
-        var task_id = data.task;
-        showCeleryTaskLog(task_id);
+        var taskId = data.task;
+        showCeleryTaskLog(taskId);
     };
     requestApi({
-        url: the_url,
+        url: theUrl,
         error: error,
         method: 'GET',
         success: success
     });
 })
-
-
 </script>
 {% endblock %}

apps/assets/templates/assets/system_user_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/urls/api_urls.py

@@ -23,6 +23,8 @@ router.register(r'asset-users', api.AssetUserViewSet, 'asset-user')
 router.register(r'asset-users-info', api.AssetUserExportViewSet, 'asset-user-info')
 router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset')
+router.register(r'system-users-assets-relations', api.SystemUserAssetRelationViewSet, 'system-users-assets-relation')
+router.register(r'system-users-nodes-relations', api.SystemUserNodeRelationViewSet, 'system-users-nodes-relation')
 
 cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
 cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')

apps/audits/api.py

@@ -11,4 +11,4 @@ class FTPLogViewSet(OrgModelViewSet):
     model = FTPLog
     serializer_class = FTPLogSerializer
     permission_classes = (IsOrgAdminOrAppUser | IsOrgAuditor,)
-
+    http_method_names = ['get', 'post', 'head', 'options']

apps/audits/migrations/0007_auto_20191202_1010.py

+# Generated by Django 2.2.7 on 2019-12-02 02:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0006_auto_20190726_1753'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ftplog',
+            name='remote_addr',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Remote addr'),
+        ),
+        migrations.AlterField(
+            model_name='operatelog',
+            name='remote_addr',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Remote addr'),
+        ),
+        migrations.AlterField(
+            model_name='passwordchangelog',
+            name='remote_addr',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Remote addr'),
+        ),
+    ]

apps/audits/models.py

@@ -16,7 +16,7 @@ __all__ = [
 class FTPLog(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     user = models.CharField(max_length=128, verbose_name=_('User'))
-    remote_addr = models.CharField(max_length=15, verbose_name=_("Remote addr"), blank=True, null=True)
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     asset = models.CharField(max_length=1024, verbose_name=_("Asset"))
     system_user = models.CharField(max_length=128, verbose_name=_("System user"))
     operate = models.CharField(max_length=16, verbose_name=_("Operate"))
@@ -39,7 +39,7 @@ class OperateLog(OrgModelMixin):
     action = models.CharField(max_length=16, choices=ACTION_CHOICES, verbose_name=_("Action"))
     resource_type = models.CharField(max_length=64, verbose_name=_("Resource Type"))
     resource = models.CharField(max_length=128, verbose_name=_("Resource"))
-    remote_addr = models.CharField(max_length=15, verbose_name=_("Remote addr"), blank=True, null=True)
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     datetime = models.DateTimeField(auto_now=True)
 
     def __str__(self):
@@ -50,7 +50,7 @@ class PasswordChangeLog(models.Model):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     user = models.CharField(max_length=128, verbose_name=_('User'))
     change_by = models.CharField(max_length=128, verbose_name=_("Change by"))
-    remote_addr = models.CharField(max_length=15, verbose_name=_("Remote addr"), blank=True, null=True)
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     datetime = models.DateTimeField(auto_now=True)
 
     def __str__(self):

apps/audits/signals_handler.py

@@ -13,8 +13,8 @@ from common.utils import get_request_ip, get_logger, get_syslogger
 from users.models import User
 from authentication.signals import post_auth_failed, post_auth_success
 from terminal.models import Session, Command
-from terminal.backends.command.serializers import SessionCommandSerializer
-from . import models, serializers
+from common.utils.encode import model_to_json
+from . import models
 from .tasks import write_login_log_async
 
 logger = get_logger(__name__)
@@ -51,7 +51,10 @@ def create_operate_log(action, sender, resource):
 
 
 @receiver(post_save, dispatch_uid="my_unique_identifier")
-def on_object_created_or_update(sender, instance=None, created=False, **kwargs):
+def on_object_created_or_update(sender, instance=None, created=False, update_fields=None, **kwargs):
+    if instance._meta.object_name == 'User' and \
+            update_fields and 'last_login' in update_fields:
+        return
     if created:
         action = models.OperateLog.ACTION_CREATE
     else:
@@ -79,27 +82,20 @@ def on_user_change_password(sender, instance=None, **kwargs):
 def on_audits_log_create(sender, instance=None, **kwargs):
     if sender == models.UserLoginLog:
         category = "login_log"
-        serializer = serializers.LoginLogSerializer
     elif sender == models.FTPLog:
-        serializer = serializers.FTPLogSerializer
         category = "ftp_log"
     elif sender == models.OperateLog:
         category = "operation_log"
-        serializer = serializers.OperateLogSerializer
     elif sender == models.PasswordChangeLog:
         category = "password_change_log"
-        serializer = serializers.PasswordChangeLogSerializer
     elif sender == Session:
         category = "host_session_log"
-        serializer = serializers.SessionAuditSerializer
     elif sender == Command:
         category = "session_command_log"
-        serializer = SessionCommandSerializer
     else:
         return
 
-    s = serializer(instance=instance)
-    data = json_render.render(s.data).decode(errors='ignore')
+    data = model_to_json(instance)
     msg = "{} - {}".format(category, data)
     sys_logger.info(msg)
 

apps/audits/tasks.py

@@ -6,7 +6,7 @@ from django.conf import settings
 from celery import shared_task
 
 from ops.celery.decorator import register_as_period_task
-from .models import UserLoginLog
+from .models import UserLoginLog, OperateLog
 from .utils import write_login_log
 
 
@@ -22,6 +22,18 @@ def clean_login_log_period():
     UserLoginLog.objects.filter(datetime__lt=expired_day).delete()
 
 
+@register_as_period_task(interval=3600*24)
+@shared_task
+def clean_operation_log_period():
+    now = timezone.now()
+    try:
+        days = int(settings.LOGIN_LOG_KEEP_DAYS)
+    except ValueError:
+        days = 90
+    expired_day = now - datetime.timedelta(days=days)
+    OperateLog.objects.filter(datetime__lt=expired_day).delete()
+
+
 @shared_task
 def write_login_log_async(*args, **kwargs):
     write_login_log(*args, **kwargs)

apps/audits/templates/audits/ftp_log_list.html

@@ -5,8 +5,6 @@
 {% load common_tags %}
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         #search_btn {
             margin-bottom: 0;
@@ -14,6 +12,9 @@
         .form-control {
             height: 30px;
         }
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
     </style>
 {% endblock %}
 

apps/audits/templates/audits/operate_log_list.html

@@ -5,8 +5,6 @@
 {% load common_tags %}
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         #search_btn {
             margin-bottom: 0;

apps/audits/templates/audits/password_change_log_list.html

@@ -5,8 +5,6 @@
 {% load common_tags %}
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         #search_btn {
             margin-bottom: 0;

apps/authentication/backends/api.py

@@ -109,7 +109,7 @@ class AccessKeyAuthentication(authentication.BaseAuthentication):
 
 class AccessTokenAuthentication(authentication.BaseAuthentication):
     keyword = 'Bearer'
-    expiration = settings.TOKEN_EXPIRATION or 3600
+    # expiration = settings.TOKEN_EXPIRATION or 3600
     model = get_user_model()
 
     def authenticate(self, request):

apps/authentication/backends/ldap.py

 # coding:utf-8
 #
 
+import warnings
 import ldap
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
-from django_auth_ldap.backend import _LDAPUser, LDAPBackend
+from django_auth_ldap.backend import _LDAPUser, LDAPBackend, LDAPSettings
 from django_auth_ldap.config import _LDAPConfig, LDAPSearch, LDAPSearchUnion
 
 from users.utils import construct_user_email
@@ -17,7 +18,6 @@ class LDAPAuthorizationBackend(LDAPBackend):
     """
     Override this class to override _LDAPUser to LDAPUser
     """
-
     @staticmethod
     def user_can_authenticate(user):
         """
@@ -27,18 +27,26 @@ class LDAPAuthorizationBackend(LDAPBackend):
         is_valid = getattr(user, 'is_valid', None)
         return is_valid or is_valid is None
 
-    def authenticate(self, request=None, username=None, password=None, **kwargs):
+    def pre_check(self, username):
+        if not settings.AUTH_LDAP:
+            return False
         logger.info('Authentication LDAP backend')
         if not username:
             logger.info('Authenticate failed: username is None')
-            return None
+            return False
         if settings.AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS:
             user_model = self.get_user_model()
             exist = user_model.objects.filter(username=username).exists()
             if not exist:
                 msg = 'Authentication failed: user ({}) is not in the user list'
                 logger.info(msg.format(username))
-                return None
+                return False
+        return True
+
+    def authenticate(self, request=None, username=None, password=None, **kwargs):
+        match = self.pre_check(username)
+        if not match:
+            return None
         ldap_user = LDAPUser(self, username=username.strip(), request=request)
         user = self.authenticate_ldap_user(ldap_user, password)
         logger.info('Authenticate user: {}'.format(user))

apps/authentication/backends/openid/views.py

@@ -25,7 +25,8 @@ __all__ = ['OpenIDLoginView', 'OpenIDLoginCompleteView']
 
 class OpenIDLoginView(RedirectView):
     def get_redirect_url(self, *args, **kwargs):
-        redirect_uri = settings.BASE_SITE_URL + str(settings.LOGIN_COMPLETE_URL)
+        redirect_uri = settings.BASE_SITE_URL + \
+                       str(settings.AUTH_OPENID_LOGIN_COMPLETE_URL)
         nonce = Nonce(
             redirect_uri=redirect_uri,
             next_path=self.request.GET.get('next')

apps/authentication/mixins.py

@@ -141,7 +141,7 @@ class AuthMixin:
             )
 
     def check_user_login_confirm_if_need(self, user):
-        if not settings.CONFIG.LOGIN_CONFIRM_ENABLE:
+        if not settings.LOGIN_CONFIRM_ENABLE:
             return
         confirm_setting = user.get_login_confirm_setting()
         if self.request.session.get('auth_confirm') or not confirm_setting:

apps/authentication/views/login.py

@@ -60,7 +60,8 @@ class UserLoginView(mixins.AuthMixin, FormView):
         # show jumpserver login page if request http://{JUMP-SERVER}/?admin=1
         if settings.AUTH_OPENID and not self.request.GET.get('admin', 0):
             query_string = request.GET.urlencode()
-            login_url = "{}?{}".format(settings.LOGIN_URL, query_string)
+            openid_login_url = reverse_lazy("authentication:openid:openid-login")
+            login_url = "{}?{}".format(openid_login_url, query_string)
             return redirect(login_url)
         request.session.set_test_cookie()
         return super().get(request, *args, **kwargs)

apps/common/drf/__init__.py

+# -*- coding: utf-8 -*-
+#

apps/common/filters.py → apps/common/drf/filters.py

@@ -7,7 +7,7 @@ from rest_framework.serializers import ValidationError
 from django.core.cache import cache
 import logging
 
-from . import const
+from common import const
 
 __all__ = ["DatetimeRangeFilter", "IDSpmFilter", "CustomFilter"]
 

apps/common/parsers/csv.py → apps/common/drf/parsers/csv.py

@@ -9,7 +9,7 @@ import unicodecsv
 from rest_framework.parsers import BaseParser
 from rest_framework.exceptions import ParseError
 
-from ..utils import get_logger
+from common.utils import get_logger
 
 logger = get_logger(__file__)
 

apps/common/renders/csv.py → apps/common/drf/renders/csv.py

@@ -9,7 +9,7 @@ from six import BytesIO
 from rest_framework.renderers import BaseRenderer
 from rest_framework.utils import encoders, json
 
-from ..utils import get_logger
+from common.utils import get_logger
 
 logger = get_logger(__file__)
 

apps/common/drf/routers.py

+# -*- coding: utf-8 -*-
+#
+from rest_framework_nested.routers import NestedMixin
+from rest_framework_bulk.routes import BulkRouter
+
+__all__ = ['BulkNestDefaultRouter']
+
+
+class BulkNestDefaultRouter(NestedMixin, BulkRouter):
+    pass

apps/common/fields/serializer.py

@@ -5,7 +5,10 @@ from rest_framework import serializers
 from django.utils import six
 
 
-__all__ = ['StringIDField', 'StringManyToManyField', 'ChoiceDisplayField']
+__all__ = [
+    'StringIDField', 'StringManyToManyField', 'ChoiceDisplayField',
+    'CustomMetaDictField'
+]
 
 
 class StringIDField(serializers.Field):
@@ -39,3 +42,63 @@ class DictField(serializers.DictField):
         if not value or not isinstance(value, dict):
             value = {}
         return super().to_representation(value)
+
+
+class CustomMetaDictField(serializers.DictField):
+    """
+    In use:
+    RemoteApp params field
+    CommandStorage meta field
+    ReplayStorage meta field
+    """
+    type_map_fields = {}
+    default_type = None
+    need_convert_key = False
+
+    def filter_attribute(self, attribute, instance):
+        fields = self.type_map_fields.get(instance.type, [])
+        for field in fields:
+            if field.get('write_only', False):
+                attribute.pop(field['name'], None)
+        return attribute
+
+    def get_attribute(self, instance):
+        """
+        序列化时调用
+        """
+        attribute = super().get_attribute(instance)
+        attribute = self.filter_attribute(attribute, instance)
+        return attribute
+
+    def convert_value_key(self, dictionary, value):
+        if not self.need_convert_key:
+            # remote app
+            return value
+        tp = dictionary.get('type')
+        _value = {}
+        for k, v in value.items():
+            prefix = '{}_'.format(tp)
+            _k = k
+            if k.lower().startswith(prefix):
+                _k = k.lower().split(prefix, 1)[1]
+            _k = _k.upper()
+            _value[_k] = value[k]
+        return _value
+
+    def filter_value_key(self, dictionary, value):
+        tp = dictionary.get('type', self.default_type)
+        fields = self.type_map_fields.get(tp, [])
+        fields_names = [field['name'] for field in fields]
+        no_need_keys = [k for k in value.keys() if k not in fields_names]
+        for k in no_need_keys:
+            value.pop(k)
+        return value
+
+    def get_value(self, dictionary):
+        """
+        反序列化时调用
+        """
+        value = super().get_value(dictionary)
+        value = self.convert_value_key(dictionary, value)
+        value = self.filter_value_key(dictionary, value)
+        return value

apps/common/mixins/api.py

@@ -3,11 +3,11 @@
 from django.http import JsonResponse
 from rest_framework.settings import api_settings
 
-from ..filters import IDSpmFilter, CustomFilter
+from common.drf.filters import IDSpmFilter, CustomFilter
 
 __all__ = [
     "JSONResponseMixin", "CommonApiMixin",
-    "IDSpmFilterMixin", "CommonApiMixin",
+    "IDSpmFilterMixin",
 ]
 
 

apps/common/signals_handlers.py

@@ -7,10 +7,13 @@ from django.conf import settings
 from django.dispatch import receiver
 from django.core.signals import request_finished
 from django.db import connection
+from django.conf import LazySettings
+from django.db.utils import ProgrammingError, OperationalError
 
 
 from common.utils import get_logger
 from .local import thread_local
+from .signals import django_ready
 
 pattern = re.compile(r'FROM `(\w+)`')
 logger = get_logger(__name__)
@@ -62,7 +65,15 @@ if settings.DEBUG and DEBUG_DB:
     request_finished.connect(on_request_finished_logging_db_query)
 
 
+@receiver(django_ready)
+def monkey_patch_settings(sender, **kwargs):
+    def monkey_patch_getattr(self, name):
+        val = getattr(self._wrapped, name)
+        if callable(val):
+            val = val()
+        return val
 
-
-
-
+    try:
+        LazySettings.__getattr__ = monkey_patch_getattr
+    except (ProgrammingError, OperationalError):
+        pass

apps/common/tasks.py

 from django.core.mail import send_mail
 from django.conf import settings
 from celery import shared_task
+
 from .utils import get_logger
 
 

apps/common/utils/common.py

@@ -9,6 +9,7 @@ import uuid
 from functools import wraps
 import time
 import ipaddress
+import psutil
 
 
 UUID_PATTERN = re.compile(r'\w{8}(-\w{4}){3}-\w{12}')
@@ -234,3 +235,10 @@ class lazyproperty:
             value = self.func(instance)
             setattr(instance, self.func.__name__, value)
             return value
+
+
+def get_disk_usage():
+    partitions = psutil.disk_partitions()
+    mount_points = [p.mountpoint for p in partitions]
+    usages = {p: psutil.disk_usage(p) for p in mount_points}
+    return usages

apps/common/utils/django.py

@@ -35,20 +35,3 @@ def date_expired_default():
         years = 70
     return timezone.now() + timezone.timedelta(days=365*years)
 
-
-def get_command_storage_setting():
-    default = settings.DEFAULT_TERMINAL_COMMAND_STORAGE
-    value = settings.TERMINAL_COMMAND_STORAGE
-    if not value:
-        return default
-    value.update(default)
-    return value
-
-
-def get_replay_storage_setting():
-    default = settings.DEFAULT_TERMINAL_REPLAY_STORAGE
-    value = settings.TERMINAL_REPLAY_STORAGE
-    if not value:
-        return default
-    value.update(default)
-    return value

apps/common/utils/encode.py

 # -*- coding: utf-8 -*-
 #
 import re
+import json
 from six import string_types
 import base64
 import os
 import time
 import hashlib
 from io import StringIO
+from itertools import chain
 
 import paramiko
 import sshpubkeys
@@ -15,6 +17,8 @@ from itsdangerous import (
     BadSignature, SignatureExpired
 )
 from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db.models.fields.files import FileField
 
 from .http import http_date
 
@@ -186,3 +190,35 @@ def get_signer():
 
 def ensure_last_char_is_ascii(data):
     remain = ''
+
+
+secret_pattern = re.compile(r'password|secret|key', re.IGNORECASE)
+
+
+def model_to_dict_pro(instance, fields=None, exclude=None):
+    from ..fields.model import EncryptMixin
+    opts = instance._meta
+    data = {}
+    for f in chain(opts.concrete_fields, opts.private_fields):
+        if not getattr(f, 'editable', False):
+            continue
+        if fields and f.name not in fields:
+            continue
+        if exclude and f.name in exclude:
+            continue
+        if isinstance(f, FileField):
+            continue
+        if isinstance(f, EncryptMixin):
+            continue
+        if secret_pattern.search(f.name):
+            continue
+        value = f.value_from_object(instance)
+        data[f.name] = value
+    return data
+
+
+def model_to_json(instance, sort_keys=True, indent=2, cls=None):
+    data = model_to_dict_pro(instance)
+    if cls is None:
+        cls = DjangoJSONEncoder
+    return json.dumps(data, sort_keys=sort_keys, indent=indent, cls=cls)

apps/common/utils/ipip/__init__.py

 # -*- coding: utf-8 -*-
 #
-from .ipdb import *
+from .utils import *

apps/common/utils/ipip/ipdb.py → apps/common/utils/ipip/utils.py

 # -*- coding: utf-8 -*-
 #
 import os
+from django.utils.translation import ugettext as _
 
 import ipdb
 
+__all__ = ['get_ip_city']
 ipip_db = None
 
 
 def get_ip_city(ip):
     global ipip_db
+    if not ip or not isinstance(ip, str):
+        return _("Invalid ip")
+    if ':' in ip:
+        return 'IPv6'
     if ipip_db is None:
         ipip_db_path = os.path.join(os.path.dirname(__file__), 'ipipfree.ipdb')
         ipip_db = ipdb.City(ipip_db_path)
     info = list(set(ipip_db.find(ip, 'CN')))
     if '' in info:
         info.remove('')
-    return ' '.join(info)
\ No newline at end of file
+    return ' '.join(info)

apps/jumpserver/const.py

 # -*- coding: utf-8 -*-
 #
+import os
+from .conf import ConfigManager
+
+__all__ = ['BASE_DIR', 'PROJECT_DIR', 'VERSION', 'CONFIG', 'DYNAMIC']
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROJECT_DIR = os.path.dirname(BASE_DIR)
 VERSION = '1.5.5'
+CONFIG = ConfigManager.load_user_config()
+DYNAMIC = ConfigManager.get_dynamic_config(CONFIG)

apps/jumpserver/context_processor.py

@@ -19,8 +19,8 @@ def jumpserver_processor(request):
         'SECURITY_COMMAND_EXECUTION': settings.SECURITY_COMMAND_EXECUTION,
         'SECURITY_MFA_VERIFY_TTL': settings.SECURITY_MFA_VERIFY_TTL,
         'FORCE_SCRIPT_NAME': settings.FORCE_SCRIPT_NAME,
-        'SECURITY_VIEW_AUTH_NEED_MFA': settings.CONFIG.SECURITY_VIEW_AUTH_NEED_MFA,
-        'LOGIN_CONFIRM_ENABLE': settings.CONFIG.LOGIN_CONFIRM_ENABLE,
+        'SECURITY_VIEW_AUTH_NEED_MFA': settings.SECURITY_VIEW_AUTH_NEED_MFA,
+        'LOGIN_CONFIRM_ENABLE': settings.LOGIN_CONFIRM_ENABLE,
     }
     return context
 

apps/jumpserver/settings/__init__.py

+# -*- coding: utf-8 -*-
+#
+from .base import *
+from .logging import *
+from .libs import *
+from .auth import *
+from .custom import *
+from ._xpack import *

apps/jumpserver/settings/_xpack.py

+# -*- coding: utf-8 -*-
+#
+
+import os
+from .. import const
+from .base import INSTALLED_APPS, TEMPLATES
+
+XPACK_DIR = os.path.join(const.BASE_DIR, 'xpack')
+XPACK_ENABLED = os.path.isdir(XPACK_DIR)
+XPACK_TEMPLATES_DIR = []
+XPACK_CONTEXT_PROCESSOR = []
+
+if XPACK_ENABLED:
+    from xpack.utils import get_xpack_templates_dir, get_xpack_context_processor
+    INSTALLED_APPS.append('xpack.apps.XpackConfig')
+    XPACK_TEMPLATES_DIR = get_xpack_templates_dir(const.BASE_DIR)
+    XPACK_CONTEXT_PROCESSOR = get_xpack_context_processor()
+    TEMPLATES[0]['DIRS'].extend(XPACK_TEMPLATES_DIR)
+    TEMPLATES[0]['OPTIONS']['context_processors'].extend(XPACK_CONTEXT_PROCESSOR)

apps/jumpserver/settings/auth.py

+# -*- coding: utf-8 -*-
+#
+import os
+import ldap
+from django.urls import reverse_lazy
+
+from ..const import CONFIG, DYNAMIC, PROJECT_DIR
+
+# OTP settings
+OTP_ISSUER_NAME = CONFIG.OTP_ISSUER_NAME
+OTP_VALID_WINDOW = CONFIG.OTP_VALID_WINDOW
+
+# Auth LDAP settings
+AUTH_LDAP = DYNAMIC.AUTH_LDAP
+AUTH_LDAP_SERVER_URI = DYNAMIC.AUTH_LDAP_SERVER_URI
+AUTH_LDAP_BIND_DN = DYNAMIC.AUTH_LDAP_BIND_DN
+AUTH_LDAP_BIND_PASSWORD = DYNAMIC.AUTH_LDAP_BIND_PASSWORD
+AUTH_LDAP_SEARCH_OU = DYNAMIC.AUTH_LDAP_SEARCH_OU
+AUTH_LDAP_SEARCH_FILTER = DYNAMIC.AUTH_LDAP_SEARCH_FILTER
+AUTH_LDAP_START_TLS = DYNAMIC.AUTH_LDAP_START_TLS
+AUTH_LDAP_USER_ATTR_MAP = DYNAMIC.AUTH_LDAP_USER_ATTR_MAP
+AUTH_LDAP_GLOBAL_OPTIONS = {
+    ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+    ldap.OPT_REFERRALS: CONFIG.AUTH_LDAP_OPTIONS_OPT_REFERRALS
+}
+LDAP_CERT_FILE = os.path.join(PROJECT_DIR, "data", "certs", "ldap_ca.pem")
+if os.path.isfile(LDAP_CERT_FILE):
+    AUTH_LDAP_GLOBAL_OPTIONS[ldap.OPT_X_TLS_CACERTFILE] = LDAP_CERT_FILE
+# AUTH_LDAP_GROUP_SEARCH_OU = CONFIG.AUTH_LDAP_GROUP_SEARCH_OU
+# AUTH_LDAP_GROUP_SEARCH_FILTER = CONFIG.AUTH_LDAP_GROUP_SEARCH_FILTER
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+#    AUTH_LDAP_GROUP_SEARCH_OU, ldap.SCOPE_SUBTREE, AUTH_LDAP_GROUP_SEARCH_FILTER
+# )
+AUTH_LDAP_CONNECTION_OPTIONS = {
+    ldap.OPT_TIMEOUT: CONFIG.AUTH_LDAP_CONNECT_TIMEOUT
+}
+AUTH_LDAP_CACHE_TIMEOUT = 1
+AUTH_LDAP_ALWAYS_UPDATE_USER = True
+
+AUTH_LDAP_SEARCH_PAGED_SIZE = CONFIG.AUTH_LDAP_SEARCH_PAGED_SIZE
+AUTH_LDAP_SYNC_IS_PERIODIC = CONFIG.AUTH_LDAP_SYNC_IS_PERIODIC
+AUTH_LDAP_SYNC_INTERVAL = CONFIG.AUTH_LDAP_SYNC_INTERVAL
+AUTH_LDAP_SYNC_CRONTAB = CONFIG.AUTH_LDAP_SYNC_CRONTAB
+AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS = CONFIG.AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS
+
+# openid
+# Auth OpenID settings
+BASE_SITE_URL = CONFIG.BASE_SITE_URL
+AUTH_OPENID = CONFIG.AUTH_OPENID
+AUTH_OPENID_SERVER_URL = CONFIG.AUTH_OPENID_SERVER_URL
+AUTH_OPENID_REALM_NAME = CONFIG.AUTH_OPENID_REALM_NAME
+AUTH_OPENID_CLIENT_ID = CONFIG.AUTH_OPENID_CLIENT_ID
+AUTH_OPENID_CLIENT_SECRET = CONFIG.AUTH_OPENID_CLIENT_SECRET
+AUTH_OPENID_IGNORE_SSL_VERIFICATION = CONFIG.AUTH_OPENID_IGNORE_SSL_VERIFICATION
+AUTH_OPENID_SHARE_SESSION = CONFIG.AUTH_OPENID_SHARE_SESSION
+AUTH_OPENID_LOGIN_URL = reverse_lazy("authentication:openid:openid-login")
+AUTH_OPENID_LOGIN_COMPLETE_URL = reverse_lazy("authentication:openid:openid-login-complete")
+
+
+# Radius Auth
+AUTH_RADIUS = CONFIG.AUTH_RADIUS
+AUTH_RADIUS_BACKEND = 'authentication.backends.radius.RadiusBackend'
+RADIUS_SERVER = CONFIG.RADIUS_SERVER
+RADIUS_PORT = CONFIG.RADIUS_PORT
+RADIUS_SECRET = CONFIG.RADIUS_SECRET
+
+
+TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION
+LOGIN_CONFIRM_ENABLE = CONFIG.LOGIN_CONFIRM_ENABLE
+OTP_IN_RADIUS = CONFIG.OTP_IN_RADIUS

apps/jumpserver/settings/custom.py

+# -*- coding: utf-8 -*-
+#
+from ..const import CONFIG, DYNAMIC
+
+# Storage settings
+COMMAND_STORAGE = {
+    'ENGINE': 'terminal.backends.command.db',
+}
+DEFAULT_TERMINAL_COMMAND_STORAGE = {
+    "default": {
+        "TYPE": "server",
+    },
+}
+TERMINAL_COMMAND_STORAGE = DYNAMIC.TERMINAL_COMMAND_STORAGE or {}
+DEFAULT_TERMINAL_REPLAY_STORAGE = {
+    "default": {
+        "TYPE": "server",
+    },
+}
+TERMINAL_REPLAY_STORAGE = DYNAMIC.TERMINAL_REPLAY_STORAGE
+
+# Security settings
+SECURITY_MFA_AUTH = DYNAMIC.SECURITY_MFA_AUTH
+SECURITY_COMMAND_EXECUTION = DYNAMIC.SECURITY_COMMAND_EXECUTION
+SECURITY_LOGIN_LIMIT_COUNT = DYNAMIC.SECURITY_LOGIN_LIMIT_COUNT
+SECURITY_LOGIN_LIMIT_TIME = DYNAMIC.SECURITY_LOGIN_LIMIT_TIME  # Unit: minute
+SECURITY_MAX_IDLE_TIME = DYNAMIC.SECURITY_MAX_IDLE_TIME  # Unit: minute
+SECURITY_PASSWORD_EXPIRATION_TIME = DYNAMIC.SECURITY_PASSWORD_EXPIRATION_TIME # Unit: day
+SECURITY_PASSWORD_MIN_LENGTH = DYNAMIC.SECURITY_PASSWORD_MIN_LENGTH  # Unit: bit
+SECURITY_PASSWORD_UPPER_CASE = DYNAMIC.SECURITY_PASSWORD_UPPER_CASE
+SECURITY_PASSWORD_LOWER_CASE = DYNAMIC.SECURITY_PASSWORD_LOWER_CASE
+SECURITY_PASSWORD_NUMBER = DYNAMIC.SECURITY_PASSWORD_NUMBER
+SECURITY_PASSWORD_SPECIAL_CHAR = DYNAMIC.SECURITY_PASSWORD_SPECIAL_CHAR
+SECURITY_PASSWORD_RULES = [
+    'SECURITY_PASSWORD_MIN_LENGTH',
+    'SECURITY_PASSWORD_UPPER_CASE',
+    'SECURITY_PASSWORD_LOWER_CASE',
+    'SECURITY_PASSWORD_NUMBER',
+    'SECURITY_PASSWORD_SPECIAL_CHAR'
+]
+SECURITY_MFA_VERIFY_TTL = CONFIG.SECURITY_MFA_VERIFY_TTL
+SECURITY_VIEW_AUTH_NEED_MFA = CONFIG.SECURITY_VIEW_AUTH_NEED_MFA
+SECURITY_SERVICE_ACCOUNT_REGISTRATION = DYNAMIC.SECURITY_SERVICE_ACCOUNT_REGISTRATION
+
+# Terminal other setting
+TERMINAL_PASSWORD_AUTH = DYNAMIC.TERMINAL_PASSWORD_AUTH
+TERMINAL_PUBLIC_KEY_AUTH = DYNAMIC.TERMINAL_PUBLIC_KEY_AUTH
+TERMINAL_HEARTBEAT_INTERVAL = DYNAMIC.TERMINAL_HEARTBEAT_INTERVAL
+TERMINAL_ASSET_LIST_SORT_BY = DYNAMIC.TERMINAL_ASSET_LIST_SORT_BY
+TERMINAL_ASSET_LIST_PAGE_SIZE = DYNAMIC.TERMINAL_ASSET_LIST_PAGE_SIZE
+TERMINAL_SESSION_KEEP_DURATION = DYNAMIC.TERMINAL_SESSION_KEEP_DURATION
+TERMINAL_HOST_KEY = DYNAMIC.TERMINAL_HOST_KEY
+TERMINAL_HEADER_TITLE = DYNAMIC.TERMINAL_HEADER_TITLE
+TERMINAL_TELNET_REGEX = DYNAMIC.TERMINAL_TELNET_REGEX
+
+# User or user group permission cache time, default 3600 seconds
+ASSETS_PERM_CACHE_ENABLE = CONFIG.ASSETS_PERM_CACHE_ENABLE
+ASSETS_PERM_CACHE_TIME = CONFIG.ASSETS_PERM_CACHE_TIME
+
+# Asset user auth external backend, default AuthBook backend
+BACKEND_ASSET_USER_AUTH_VAULT = False
+
+DEFAULT_ORG_SHOW_ALL_USERS = CONFIG.DEFAULT_ORG_SHOW_ALL_USERS
+PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
+WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
+FLOWER_URL = CONFIG.FLOWER_URL
+
+# Enable internal period task
+PERIOD_TASK_ENABLED = CONFIG.PERIOD_TASK_ENABLED
+
+# Email custom content
+EMAIL_SUBJECT_PREFIX = DYNAMIC.EMAIL_SUBJECT_PREFIX
+EMAIL_SUFFIX = DYNAMIC.EMAIL_SUFFIX
+EMAIL_CUSTOM_USER_CREATED_SUBJECT = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_SUBJECT
+EMAIL_CUSTOM_USER_CREATED_HONORIFIC = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_HONORIFIC
+EMAIL_CUSTOM_USER_CREATED_BODY = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_BODY
+EMAIL_CUSTOM_USER_CREATED_SIGNATURE = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_SIGNATURE
+
+DISPLAY_PER_PAGE = CONFIG.DISPLAY_PER_PAGE
+DEFAULT_EXPIRED_YEARS = 70
+USER_GUIDE_URL = DYNAMIC.USER_GUIDE_URL
+HTTP_LISTEN_PORT = CONFIG.HTTP_LISTEN_PORT
+WS_LISTEN_PORT = CONFIG.WS_LISTEN_PORT
+LOGIN_LOG_KEEP_DAYS = DYNAMIC.LOGIN_LOG_KEEP_DAYS

apps/jumpserver/settings/libs.py

+# -*- coding: utf-8 -*-
+#
+import os
+from ..const import CONFIG, PROJECT_DIR
+
+REST_FRAMEWORK = {
+    # Use Django's standard `django.contrib.auth` permissions,
+    # or allow read-only access for unauthenticated users.
+    'DEFAULT_PERMISSION_CLASSES': (
+        'common.permissions.IsOrgAdmin',
+    ),
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+        'rest_framework.renderers.BrowsableAPIRenderer',
+        'common.drf.renders.JMSCSVRender',
+    ),
+    'DEFAULT_PARSER_CLASSES': (
+        'rest_framework.parsers.JSONParser',
+        'rest_framework.parsers.FormParser',
+        'rest_framework.parsers.MultiPartParser',
+        'common.drf.parsers.JMSCSVParser',
+        'rest_framework.parsers.FileUploadParser',
+    ),
+    'DEFAULT_AUTHENTICATION_CLASSES': (
+        # 'rest_framework.authentication.BasicAuthentication',
+        'authentication.backends.api.AccessKeyAuthentication',
+        'authentication.backends.api.AccessTokenAuthentication',
+        'authentication.backends.api.PrivateTokenAuthentication',
+        'authentication.backends.api.SignatureAuthentication',
+        'authentication.backends.api.SessionAuthentication',
+    ),
+    'DEFAULT_FILTER_BACKENDS': (
+        'django_filters.rest_framework.DjangoFilterBackend',
+        'rest_framework.filters.SearchFilter',
+        'rest_framework.filters.OrderingFilter',
+    ),
+    'DEFAULT_METADATA_CLASS': 'common.drf.metadata.SimpleMetadataWithFilters',
+    'ORDERING_PARAM': "order",
+    'SEARCH_PARAM': "search",
+    'DATETIME_FORMAT': '%Y-%m-%d %H:%M:%S %z',
+    'DATETIME_INPUT_FORMATS': ['iso-8601', '%Y-%m-%d %H:%M:%S %z'],
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+    # 'PAGE_SIZE': 15
+}
+
+SWAGGER_SETTINGS = {
+    'DEFAULT_AUTO_SCHEMA_CLASS': 'jumpserver.swagger.CustomSwaggerAutoSchema',
+    'USE_SESSION_AUTH': True,
+    'SECURITY_DEFINITIONS': {
+        'Bearer': {
+            'type': 'apiKey',
+            'name': 'Authorization',
+            'in': 'header'
+        }
+    },
+}
+
+
+# Captcha settings, more see https://django-simple-captcha.readthedocs.io/en/latest/advanced.html
+CAPTCHA_IMAGE_SIZE = (80, 33)
+CAPTCHA_FOREGROUND_COLOR = '#001100'
+CAPTCHA_NOISE_FUNCTIONS = ('captcha.helpers.noise_dots',)
+CAPTCHA_TEST_MODE = CONFIG.CAPTCHA_TEST_MODE
+
+# Django bootstrap3 setting, more see http://django-bootstrap3.readthedocs.io/en/latest/settings.html
+BOOTSTRAP3 = {
+    'horizontal_label_class': 'col-md-2',
+    # Field class to use in horizontal forms
+    'horizontal_field_class': 'col-md-9',
+    # Set placeholder attributes to label if no placeholder is provided
+    'set_placeholder': False,
+    'success_css_class': '',
+    'required_css_class': 'required',
+}
+
+
+# Django channels support websocket
+CHANNEL_REDIS = "redis://:{}@{}:{}/{}".format(
+    CONFIG.REDIS_PASSWORD, CONFIG.REDIS_HOST, CONFIG.REDIS_PORT,
+    CONFIG.REDIS_DB_WS,
+)
+
+CHANNEL_LAYERS = {
+    'default': {
+        'BACKEND': 'channels_redis.core.RedisChannelLayer',
+        'CONFIG': {
+            "hosts": [CHANNEL_REDIS],
+        },
+    },
+}
+ASGI_APPLICATION = 'jumpserver.routing.application'
+
+
+# Dump all celery log to here
+CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'celery')
+
+# Celery using redis as broker
+CELERY_BROKER_URL = 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % {
+    'password': CONFIG.REDIS_PASSWORD,
+    'host': CONFIG.REDIS_HOST,
+    'port': CONFIG.REDIS_PORT,
+    'db': CONFIG.REDIS_DB_CELERY,
+}
+CELERY_TASK_SERIALIZER = 'pickle'
+CELERY_RESULT_SERIALIZER = 'pickle'
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+CELERY_ACCEPT_CONTENT = ['json', 'pickle']
+CELERY_RESULT_EXPIRES = 3600
+# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
+# CELERY_WORKER_LOG_FORMAT = '%(message)s'
+# CELERY_WORKER_TASK_LOG_FORMAT = '%(task_id)s %(task_name)s %(message)s'
+CELERY_WORKER_TASK_LOG_FORMAT = '%(message)s'
+# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
+CELERY_WORKER_LOG_FORMAT = '%(message)s'
+CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_WORKER_REDIRECT_STDOUTS = True
+CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
+# CELERY_WORKER_HIJACK_ROOT_LOGGER = True
+# CELERY_WORKER_MAX_TASKS_PER_CHILD = 40
+CELERY_TASK_SOFT_TIME_LIMIT = 3600

apps/jumpserver/settings/logging.py

+# -*- coding: utf-8 -*-
+#
+import os
+from ..const import PROJECT_DIR, CONFIG
+
+LOG_DIR = os.path.join(PROJECT_DIR, 'logs')
+JUMPSERVER_LOG_FILE = os.path.join(LOG_DIR, 'jumpserver.log')
+ANSIBLE_LOG_FILE = os.path.join(LOG_DIR, 'ansible.log')
+GUNICORN_LOG_FILE = os.path.join(LOG_DIR, 'gunicorn.log')
+LOG_LEVEL = CONFIG.LOG_LEVEL
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+        },
+        'main': {
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'format': '%(asctime)s [%(module)s %(levelname)s] %(message)s',
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'syslog': {
+            'format': 'jumpserver: %(message)s'
+        },
+        'msg': {
+            'format': '%(message)s'
+        }
+    },
+    'handlers': {
+        'null': {
+            'level': 'DEBUG',
+            'class': 'logging.NullHandler',
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'main'
+        },
+        'file': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'maxBytes': 1024*1024*100,
+            'backupCount': 7,
+            'formatter': 'main',
+            'filename': JUMPSERVER_LOG_FILE,
+        },
+        'ansible_logs': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'formatter': 'main',
+            'maxBytes': 1024*1024*100,
+            'backupCount': 7,
+            'filename': ANSIBLE_LOG_FILE,
+        },
+        'syslog': {
+            'level': 'INFO',
+            'class': 'logging.NullHandler',
+            'formatter': 'syslog'
+        },
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['null'],
+            'propagate': False,
+            'level': LOG_LEVEL,
+        },
+        'django.request': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'django.server': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'jumpserver': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+        },
+        'ops.ansible_api': {
+            'handlers': ['console', 'ansible_logs'],
+            'level': LOG_LEVEL,
+        },
+        'django_auth_ldap': {
+            'handlers': ['console', 'file'],
+            'level': "INFO",
+        },
+        'jms.audits': {
+            'handlers': ['syslog'],
+            'level': 'INFO'
+        },
+        # 'django.db': {
+        #     'handlers': ['console', 'file'],
+        #     'level': 'DEBUG'
+        # }
+    }
+}
+SYSLOG_ENABLE = CONFIG.SYSLOG_ENABLE
+
+if CONFIG.SYSLOG_ADDR != '' and len(CONFIG.SYSLOG_ADDR.split(':')) == 2:
+    host, port = CONFIG.SYSLOG_ADDR.split(':')
+    LOGGING['handlers']['syslog'].update({
+        'class': 'logging.handlers.SysLogHandler',
+        'facility': CONFIG.SYSLOG_FACILITY,
+        'address': (host, int(port)),
+    })
+
+if not os.path.isdir(LOG_DIR):
+    os.makedirs(LOG_DIR)

apps/jumpserver/urls.py

@@ -7,10 +7,7 @@ from django.conf.urls.static import static
 from django.conf.urls.i18n import i18n_patterns
 from django.views.i18n import JavaScriptCatalog
 
-# from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
 from . import views
-from .celery_flower import celery_flower_view
-from .swagger import get_swagger_view
 
 api_v1 = [
     path('users/', include('users.urls.api_urls', namespace='api-users')),
@@ -44,7 +41,7 @@ app_view_patterns = [
     path('auth/', include('authentication.urls.view_urls'), name='auth'),
     path('applications/', include('applications.urls.views_urls', namespace='applications')),
     path('tickets/', include('tickets.urls.views_urls', namespace='tickets')),
-    re_path(r'flower/(?P<path>.*)', celery_flower_view, name='flower-view'),
+    re_path(r'flower/(?P<path>.*)', views.celery_flower_view, name='flower-view'),
 ]
 
 
@@ -82,19 +79,19 @@ urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \
             + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
 urlpatterns += js_i18n_patterns
 
-handler404 = 'jumpserver.error_views.handler404'
-handler500 = 'jumpserver.error_views.handler500'
+handler404 = 'jumpserver.views.handler404'
+handler500 = 'jumpserver.views.handler500'
 
 if settings.DEBUG:
     urlpatterns += [
         re_path('^swagger(?P<format>\.json|\.yaml)$',
-                get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
-        path('docs/', get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"),
-        path('redoc/', get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'),
+                views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
+        path('docs/', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"),
+        path('redoc/', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'),
 
         re_path('^v2/swagger(?P<format>\.json|\.yaml)$',
-                get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
-        path('docs/v2/', get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"),
-        path('redoc/v2/', get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'),
+                views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
+        path('docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"),
+        path('redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'),
     ]
 

apps/jumpserver/views/__init__.py

+# -*- coding: utf-8 -*-
+#
+from .index import *
+from .other import *
+from .celery_flower import *
+from .swagger import *
+from .error_views import *

apps/jumpserver/celery_flower.py → apps/jumpserver/views/celery_flower.py

@@ -9,6 +9,8 @@ from proxy.views import proxy_view
 
 flower_url = settings.FLOWER_URL
 
+__all__ = ['celery_flower_view']
+
 
 @csrf_exempt
 def celery_flower_view(request, path):

apps/jumpserver/error_views.py → apps/jumpserver/views/error_views.py

@@ -3,6 +3,8 @@
 from django.shortcuts import render
 from django.http import JsonResponse
 
+__all__ = ['handler404', 'handler500']
+
 
 def handler404(request, *args, **argv):
     if request.content_type.find('application/json') > -1:

apps/jumpserver/views.py → apps/jumpserver/views/index.py

 import datetime
-import re
-import time
 
-from django.http import HttpResponseRedirect, JsonResponse
-from django.conf import settings
-from django.views.generic import TemplateView, View
+from django.views.generic import TemplateView
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django.db.models import Count
 from django.shortcuts import redirect
-from rest_framework.views import APIView
-from django.views.decorators.csrf import csrf_exempt
-from django.http import HttpResponse
 
 
 from users.models import User
@@ -19,7 +12,8 @@ from assets.models import Asset
 from terminal.models import Session
 from orgs.utils import current_org
 from common.permissions import PermissionsMixin, IsValidUser
-from common.http import HttpResponseTemporaryRedirect
+
+__all__ = ['IndexView']
 
 
 class IndexView(PermissionsMixin, TemplateView):
@@ -186,58 +180,3 @@ class IndexView(PermissionsMixin, TemplateView):
 
         kwargs.update(context)
         return super(IndexView, self).get_context_data(**kwargs)
-
-
-class LunaView(View):
-    def get(self, request):
-        msg = _("<div>Luna is a separately deployed program, you need to deploy Luna, koko, configure nginx for url distribution,</div> "
-                "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
-        return HttpResponse(msg)
-
-
-class I18NView(View):
-    def get(self, request, lang):
-        referer_url = request.META.get('HTTP_REFERER', '/')
-        response = HttpResponseRedirect(referer_url)
-        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang)
-        return response
-
-
-api_url_pattern = re.compile(r'^/api/(?P<app>\w+)/(?P<version>v\d)/(?P<extra>.*)$')
-
-
-@csrf_exempt
-def redirect_format_api(request, *args, **kwargs):
-    _path, query = request.path, request.GET.urlencode()
-    matched = api_url_pattern.match(_path)
-    if matched:
-        kwargs = matched.groupdict()
-        kwargs["query"] = query
-        _path = '/api/{version}/{app}/{extra}?{query}'.format(**kwargs).rstrip("?")
-        return HttpResponseTemporaryRedirect(_path)
-    else:
-        return JsonResponse({"msg": "Redirect url failed: {}".format(_path)}, status=404)
-
-
-class HealthCheckView(APIView):
-    permission_classes = ()
-
-    def get(self, request):
-        return JsonResponse({"status": 1, "time": int(time.time())})
-
-
-class WsView(APIView):
-    ws_port = settings.CONFIG.HTTP_LISTEN_PORT + 1
-
-    def get(self, request):
-        msg = _("Websocket server run on port: {}, you should proxy it on nginx"
-                .format(self.ws_port))
-        return JsonResponse({"msg": msg})
-
-
-class KokoView(View):
-    def get(self, request):
-        msg = _(
-            "<div>Koko is a separately deployed program, you need to deploy Koko, configure nginx for url distribution,</div> "
-            "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
-        return HttpResponse(msg)

apps/jumpserver/views/other.py

+# -*- coding: utf-8 -*-
+#
+import re
+import time
+
+from django.http import HttpResponseRedirect, JsonResponse
+from django.conf import settings
+from django.views.generic import View
+from django.utils.translation import ugettext_lazy as _
+from rest_framework.views import APIView
+from django.views.decorators.csrf import csrf_exempt
+from django.http import HttpResponse
+
+from common.http import HttpResponseTemporaryRedirect
+
+
+__all__ = [
+    'LunaView', 'I18NView', 'KokoView', 'WsView', 'HealthCheckView',
+    'redirect_format_api'
+]
+
+
+class LunaView(View):
+    def get(self, request):
+        msg = _("<div>Luna is a separately deployed program, you need to deploy Luna, koko, configure nginx for url distribution,</div> "
+                "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
+        return HttpResponse(msg)
+
+
+class I18NView(View):
+    def get(self, request, lang):
+        referer_url = request.META.get('HTTP_REFERER', '/')
+        response = HttpResponseRedirect(referer_url)
+        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang)
+        return response
+
+
+api_url_pattern = re.compile(r'^/api/(?P<app>\w+)/(?P<version>v\d)/(?P<extra>.*)$')
+
+
+@csrf_exempt
+def redirect_format_api(request, *args, **kwargs):
+    _path, query = request.path, request.GET.urlencode()
+    matched = api_url_pattern.match(_path)
+    if matched:
+        kwargs = matched.groupdict()
+        kwargs["query"] = query
+        _path = '/api/{version}/{app}/{extra}?{query}'.format(**kwargs).rstrip("?")
+        return HttpResponseTemporaryRedirect(_path)
+    else:
+        return JsonResponse({"msg": "Redirect url failed: {}".format(_path)}, status=404)
+
+
+class HealthCheckView(APIView):
+    permission_classes = ()
+
+    def get(self, request):
+        return JsonResponse({"status": 1, "time": int(time.time())})
+
+
+class WsView(APIView):
+    ws_port = settings.HTTP_LISTEN_PORT + 1
+
+    def get(self, request):
+        msg = _("Websocket server run on port: {}, you should proxy it on nginx"
+                .format(self.ws_port))
+        return JsonResponse({"msg": msg})
+
+
+class KokoView(View):
+    def get(self, request):
+        msg = _(
+            "<div>Koko is a separately deployed program, you need to deploy Koko, configure nginx for url distribution,</div> "
+            "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
+        return HttpResponse(msg)

apps/jumpserver/swagger.py → apps/jumpserver/views/swagger.py

@@ -50,7 +50,7 @@ class CustomSwaggerAutoSchema(SwaggerAutoSchema):
 
 
 def get_swagger_view(version='v1'):
-    from .urls import api_v1, api_v2
+    from ..urls import api_v1, api_v2
     from django.urls import path, include
     api_v1_patterns = [
         path('api/v1/', include(api_v1))

apps/ops/api/celery.py

@@ -5,17 +5,20 @@ import os
 import re
 
 from django.utils.translation import ugettext as _
+from rest_framework import viewsets
 from celery.result import AsyncResult
 from rest_framework import generics
+from django_celery_beat.models import PeriodicTask
 
-from common.permissions import IsValidUser
+from common.permissions import IsValidUser, IsSuperUser
 from common.api import LogTailApi
 from ..models import CeleryTask
-from ..serializers import CeleryResultSerializer
+from ..serializers import CeleryResultSerializer, CeleryPeriodTaskSerializer
 from ..celery.utils import get_celery_task_log_path
+from common.mixins.api import CommonApiMixin
 
 
-__all__ = ['CeleryTaskLogApi', 'CeleryResultApi']
+__all__ = ['CeleryTaskLogApi', 'CeleryResultApi', 'CeleryPeriodTaskViewSet']
 
 
 class CeleryTaskLogApi(LogTailApi):
@@ -62,3 +65,14 @@ class CeleryResultApi(generics.RetrieveAPIView):
         pk = self.kwargs.get('pk')
         return AsyncResult(pk)
 
+
+class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet):
+    queryset = PeriodicTask.objects.all()
+    serializer_class = CeleryPeriodTaskSerializer
+    permission_classes = (IsSuperUser,)
+    http_method_names = ('get', 'head', 'options', 'patch')
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.exclude(description='')
+        return queryset

apps/ops/celery/decorator.py

@@ -9,51 +9,37 @@ _after_app_shutdown_clean_periodic_tasks = []
 
 def add_register_period_task(task):
     _need_registered_period_tasks.append(task)
-    # key = "__REGISTER_PERIODIC_TASKS"
-    # value = cache.get(key, [])
-    # value.append(name)
-    # cache.set(key, value)
 
 
 def get_register_period_tasks():
-    # key = "__REGISTER_PERIODIC_TASKS"
-    # return cache.get(key, [])
     return _need_registered_period_tasks
 
 
 def add_after_app_shutdown_clean_task(name):
-    # key = "__AFTER_APP_SHUTDOWN_CLEAN_TASKS"
-    # value = cache.get(key, [])
-    # value.append(name)
-    # cache.set(key, value)
     _after_app_shutdown_clean_periodic_tasks.append(name)
 
 
 def get_after_app_shutdown_clean_tasks():
-    # key = "__AFTER_APP_SHUTDOWN_CLEAN_TASKS"
-    # return cache.get(key, [])
     return _after_app_shutdown_clean_periodic_tasks
 
 
 def add_after_app_ready_task(name):
-    # key = "__AFTER_APP_READY_RUN_TASKS"
-    # value = cache.get(key, [])
-    # value.append(name)
-    # cache.set(key, value)
     _after_app_ready_start_tasks.append(name)
 
 
 def get_after_app_ready_tasks():
-    # key = "__AFTER_APP_READY_RUN_TASKS"
-    # return cache.get(key, [])
     return _after_app_ready_start_tasks
 
 
-def register_as_period_task(crontab=None, interval=None):
+def register_as_period_task(
+        crontab=None, interval=None, name=None,
+        description=''):
     """
     Warning: Task must be have not any args and kwargs
     :param crontab:  "* * * * *"
     :param interval:  60*60*60
+    :param description: "
+    :param name: ""
     :return:
     """
     if crontab is None and interval is None:
@@ -65,14 +51,16 @@ def register_as_period_task(crontab=None, interval=None):
 
         # Because when this decorator run, the task was not created,
         # So we can't use func.name
-        name = '{func.__module__}.{func.__name__}'.format(func=func)
+        task = '{func.__module__}.{func.__name__}'.format(func=func)
+        _name = name if name else task
         add_register_period_task({
-           name: {
-               'task': name,
+           _name: {
+               'task': task,
                'interval': interval,
                'crontab': crontab,
                'args': (),
                'enabled': True,
+               'description': description
            }
         })
 

apps/ops/celery/signal_handler.py

 # -*- coding: utf-8 -*-
 #
 import logging
+from django.dispatch import receiver
 
 from django.core.cache import cache
 from celery import subtask
@@ -11,6 +12,7 @@ from kombu.utils.encoding import safe_str
 from django_celery_beat.models import PeriodicTask
 
 from common.utils import get_logger
+from common.signals import django_ready
 from .decorator import get_after_app_ready_tasks, get_after_app_shutdown_clean_tasks
 from .logger import CeleryTaskFileHandler
 

apps/ops/celery/utils.py

@@ -10,6 +10,10 @@ from django_celery_beat.models import (
     PeriodicTask, IntervalSchedule, CrontabSchedule, PeriodicTasks
 )
 
+from common.utils import get_logger
+
+logger = get_logger(__name__)
+
 
 def create_or_update_celery_periodic_tasks(tasks):
     """
@@ -21,6 +25,7 @@ def create_or_update_celery_periodic_tasks(tasks):
             'args': (16, 16),
             'kwargs': {},
             'enabled': False,
+            'description': ''
         },
     }
     :return:
@@ -35,34 +40,30 @@ def create_or_update_celery_periodic_tasks(tasks):
             return None
 
         if isinstance(detail.get("interval"), int):
-            intervals = IntervalSchedule.objects.filter(
-                every=detail["interval"], period=IntervalSchedule.SECONDS
+            kwargs = dict(
+                every=detail['interval'],
+                period=IntervalSchedule.SECONDS,
             )
-            if intervals:
-                interval = intervals[0]
-            else:
-                interval = IntervalSchedule.objects.create(
-                    every=detail['interval'],
-                    period=IntervalSchedule.SECONDS,
-                )
+            # 不能使用 get_or_create，因为可能会有多个
+            interval = IntervalSchedule.objects.filter(**kwargs).first()
+            if interval is None:
+                interval = IntervalSchedule.objects.create(**kwargs)
         elif isinstance(detail.get("crontab"), str):
             try:
                 minute, hour, day, month, week = detail["crontab"].split()
             except ValueError:
-                raise SyntaxError("crontab is not valid")
+                logger.error("crontab is not valid")
+                return
             kwargs = dict(
                 minute=minute, hour=hour, day_of_week=week,
                 day_of_month=day, month_of_year=month, timezone=get_current_timezone()
             )
-            contabs = CrontabSchedule.objects.filter(
-                **kwargs
-            )
-            if contabs:
-                crontab = contabs[0]
-            else:
+            crontab = CrontabSchedule.objects.filter(**kwargs).first()
+            if crontab is None:
                 crontab = CrontabSchedule.objects.create(**kwargs)
         else:
-            raise SyntaxError("Schedule is not valid")
+            logger.error("Schedule is not valid")
+            return
 
         defaults = dict(
             interval=interval,
@@ -71,8 +72,9 @@ def create_or_update_celery_periodic_tasks(tasks):
             task=detail['task'],
             args=json.dumps(detail.get('args', [])),
             kwargs=json.dumps(detail.get('kwargs', {})),
-            enabled=detail.get('enabled', True),
+            description=detail.get('description') or ''
         )
+        print(defaults)
 
         task = PeriodicTask.objects.update_or_create(
             defaults=defaults, name=name,

apps/ops/serializers/__init__.py

+# -*- coding: utf-8 -*-
+#
+
+from .celery import *
+from .adhoc import *

apps/ops/serializers.py → apps/ops/serializers/adhoc.py

@@ -3,17 +3,7 @@ from __future__ import unicode_literals
 from rest_framework import serializers
 from django.shortcuts import reverse
 
-from .models import Task, AdHoc, AdHocRunHistory, CommandExecution
-
-
-class CeleryResultSerializer(serializers.Serializer):
-    id = serializers.UUIDField()
-    result = serializers.JSONField()
-    state = serializers.CharField(max_length=16)
-
-
-class CeleryTaskSerializer(serializers.Serializer):
-    pass
+from ..models import Task, AdHoc, AdHocRunHistory, CommandExecution
 
 
 class TaskSerializer(serializers.ModelSerializer):
@@ -87,3 +77,4 @@ class CommandExecutionSerializer(serializers.ModelSerializer):
     @staticmethod
     def get_log_url(obj):
         return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id})
+

apps/ops/serializers/celery.py

+# ~*~ coding: utf-8 ~*~
+from __future__ import unicode_literals
+from rest_framework import serializers
+
+from django_celery_beat.models import PeriodicTask
+
+__all__ = [
+    'CeleryResultSerializer', 'CeleryTaskSerializer',
+    'CeleryPeriodTaskSerializer'
+]
+
+
+class CeleryResultSerializer(serializers.Serializer):
+    id = serializers.UUIDField()
+    result = serializers.JSONField()
+    state = serializers.CharField(max_length=16)
+
+
+class CeleryTaskSerializer(serializers.Serializer):
+    pass
+
+
+class CeleryPeriodTaskSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = PeriodicTask
+        fields = [
+            'name', 'task', 'enabled', 'description',
+            'last_run_at', 'total_run_count'
+        ]

apps/ops/tasks.py

 # coding: utf-8
 import os
 import subprocess
-import datetime
 import time
 
 from django.conf import settings
 from celery import shared_task, subtask
 from celery.exceptions import SoftTimeLimitExceeded
 from django.utils import timezone
+from django.utils.translation import ugettext_lazy as _
 
-from common.utils import get_logger, get_object_or_none
+from common.utils import get_logger, get_object_or_none, get_disk_usage
 from .celery.decorator import (
     register_as_period_task, after_app_shutdown_clean_periodic,
     after_app_ready_start
 )
 from .celery.utils import create_or_update_celery_periodic_tasks
 from .models import Task, CommandExecution, CeleryTask
+from .utils import send_server_performance_mail
 
 logger = get_logger(__file__)
 
@@ -59,7 +60,7 @@ def run_command_execution(cid, **kwargs):
 
 @shared_task
 @after_app_shutdown_clean_periodic
-@register_as_period_task(interval=3600*24)
+@register_as_period_task(interval=3600*24, description=_("Clean task history period"))
 def clean_tasks_adhoc_period():
     logger.debug("Start clean task adhoc and run history")
     tasks = Task.objects.all()
@@ -72,7 +73,7 @@ def clean_tasks_adhoc_period():
 
 @shared_task
 @after_app_shutdown_clean_periodic
-@register_as_period_task(interval=3600*24)
+@register_as_period_task(interval=3600*24, description=_("Clean celery log period"))
 def clean_celery_tasks_period():
     expire_days = 30
     logger.debug("Start clean celery task history")
@@ -103,6 +104,19 @@ def create_or_update_registered_periodic_tasks():
         create_or_update_celery_periodic_tasks(task)
 
 
+@shared_task
+@register_as_period_task(interval=3600)
+def check_server_performance_period():
+    usages = get_disk_usage()
+    usages = {path: usage for path, usage in usages.items()
+              if not path.startswith('/etc')}
+
+    for path, usage in usages.items():
+        if usage.percent > 80:
+            send_server_performance_mail(path, usage, usages)
+            return
+
+
 @shared_task(queue="ansible")
 def hello(name, callback=None):
     import time

apps/ops/templates/ops/adhoc_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/adhoc_history.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/adhoc_history_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/command_execution_create.html

@@ -11,8 +11,6 @@
           rel="stylesheet">
     <link href="{% static 'css/plugins/codemirror/ambiance.css' %}"
           rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}"
-          rel="stylesheet">
     <script type="text/javascript"
             src="{% static 'js/plugins/ztree/jquery.ztree.all.min.js' %}"></script>
     <script type="text/javascript"
@@ -22,7 +20,6 @@
     <script src="{% static 'js/plugins/xterm/addons/fit/fit.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/codemirror.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/mode/shell/shell.js' %}"></script>
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style type="text/css">
         .xterm .xterm-screen canvas {
             position: absolute;

apps/ops/templates/ops/command_execution_list.html

@@ -6,8 +6,6 @@
 {% block custom_head_css_js %}
     <link href="{% static "css/plugins/footable/footable.core.css" %}" rel="stylesheet">
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         .form-control {
             height: 30px;
@@ -16,6 +14,11 @@
         #search_btn {
             margin-bottom: 0;
         }
+
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
+
     </style>
 {% endblock %}
 

apps/ops/templates/ops/task_adhoc.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/task_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static 'css/plugins/sweetalert/sweetalert.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static 'js/plugins/sweetalert/sweetalert.min.js' %}"></script>
 {% endblock %}
 

apps/ops/templates/ops/task_history.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/urls/api_urls.py

@@ -13,6 +13,7 @@ router.register(r'tasks', api.TaskViewSet, 'task')
 router.register(r'adhoc', api.AdHocViewSet, 'adhoc')
 router.register(r'history', api.AdHocRunHistoryViewSet, 'history')
 router.register(r'command-executions', api.CommandExecutionViewSet, 'command-execution')
+router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task')
 
 urlpatterns = [
     path('tasks/<uuid:pk>/run/', api.TaskRun.as_view(), name='task-run'),

apps/ops/utils.py

 # ~*~ coding: utf-8 ~*~
 from django.utils.translation import ugettext_lazy as _
+
 from common.utils import get_logger, get_object_or_none
+from common.tasks import send_mail_async
 from orgs.utils import set_to_root_org
 from .models import Task, AdHoc
 
@@ -56,4 +58,14 @@ def update_or_create_ansible_task(
     return task, created
 
 
+def send_server_performance_mail(path, usage, usages):
+    from users.models import User
+    subject = _("Disk used more than 80%: {} => {}").format(path, usage.percent)
+    message = subject
+    admins = User.objects.filter(role=User.ROLE_ADMIN)
+    recipient_list = [u.email for u in admins if u.email]
+    logger.info(subject)
+    send_mail_async(subject, message, recipient_list, html_message=message)
+
+
 

apps/ops/views/celery.py

@@ -17,6 +17,6 @@ class CeleryTaskLogView(PermissionsMixin, TemplateView):
         context = super().get_context_data(**kwargs)
         context.update({
             'task_id': self.kwargs.get('pk'),
-            'ws_port': settings.CONFIG.WS_LISTEN_PORT
+            'ws_port': settings.WS_LISTEN_PORT
         })
         return context

apps/ops/views/command.py

@@ -80,7 +80,7 @@ class CommandExecutionStartView(PermissionsMixin, TemplateView):
             'action': _('Command execution'),
             'form': self.get_form(),
             'system_users': system_users,
-            'ws_port': settings.CONFIG.WS_LISTEN_PORT
+            'ws_port': settings.WS_LISTEN_PORT
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/orgs/mixins/api.py

@@ -46,7 +46,11 @@ class OrgModelViewSet(CommonApiMixin, OrgQuerySetMixin, ModelViewSet):
 
 class OrgBulkModelViewSet(CommonApiMixin, OrgQuerySetMixin, BulkModelViewSet):
     def allow_bulk_destroy(self, qs, filtered):
-        if qs.count() <= filtered.count():
+        qs_count = qs.count()
+        filtered_count = filtered.count()
+        if filtered_count == 1:
+            return True
+        if qs_count <= filtered_count:
             return False
         if self.request.query_params.get('spm', ''):
             return True

apps/orgs/models.py

@@ -95,6 +95,14 @@ class Organization(models.Model):
     def get_org_admins(self):
         return self.org_admins
 
+    def org_id(self):
+        if self.is_real():
+            return self.id
+        elif self.is_root():
+            return None
+        else:
+            return ''
+
     @lazyproperty
     def org_auditors(self):
         from users.models import User