Merge pull request #3294 from jumpserver/dev

Dev

Dockerfile

@@ -21,5 +21,6 @@ VOLUME /opt/jumpserver/logs
 ENV LANG=zh_CN.UTF-8
 ENV LC_ALL=zh_CN.UTF-8
 
+EXPOSE 8070
 EXPOSE 8080
 ENTRYPOINT ["./entrypoint.sh"]

apps/assets/templates/assets/_asset_user_list.html

 {% load i18n %}
 <style>
-.btn-group>.btn+.dropdown-toggle {
-    padding-right: 4px;
-    padding-left: 4px;
-}
+    .btn-group>.btn+.dropdown-toggle {
+        padding-right: 4px;
+        padding-left: 4px;
+    }
+
+    table.dataTable tbody tr.selected a {
+        color: rgb(103, 106, 108);;
+    }
+
 </style>
 <table class="table table-striped table-bordered table-hover" id="asset_user_list_table" style="width: 100%">
   <thead>
@@ -137,8 +142,7 @@ $(document).ready(function(){
     }
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -149,4 +153,4 @@ $(document).ready(function(){
 })
 
 
-</script>
\ No newline at end of file
+</script>

apps/assets/templates/assets/admin_user_assets.html

@@ -85,8 +85,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:admin-user-connective' pk=admin_user.id %}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/templates/assets/admin_user_detail.html

@@ -140,8 +140,7 @@ function replaceNodeAssetsAdminUser(nodes) {
 
 jumpserver.nodes_selected = {};
 $(document).ready(function () {
-    var url = "{% url 'api-assets:node-list' %}";
-    nodesSelect2Init(".nodes-select2", url)
+    nodesSelect2Init(".nodes-select2")
     .on('select2:select', function(evt) {
         var data = evt.params.data;
         jumpserver.nodes_selected[data.id] = data.text;

apps/assets/templates/assets/asset_asset_user_list.html

@@ -81,8 +81,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:asset-user-connective' %}" + "?asset_id={{ asset.id }}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -92,4 +91,4 @@ $(document).ready(function () {
     });
 })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/asset_create.html

@@ -110,8 +110,7 @@ $(document).ready(function () {
     $('.select2').select2({
         allowClear: true
     });
-    var url = "{% url 'api-assets:node-list' %}";
-    nodesSelect2Init(".nodes-select2", url);
+    nodesSelect2Init(".nodes-select2");
     $(".labels").select2({
         allowClear: true,
         templateSelection: format

apps/assets/templates/assets/asset_detail.html

@@ -276,8 +276,7 @@ function refreshAssetHardware() {
     var success = function(data) {
         console.log(data);
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -288,8 +287,7 @@ function refreshAssetHardware() {
 
 
 $(document).ready(function () {
-    var url = "{% url 'api-assets:node-list' %}";
-    nodesSelect2Init(".nodes-select2", url)
+    nodesSelect2Init(".nodes-select2")
     .on('select2:select', function(evt) {
         var data = evt.params.data;
         jumpserver.nodes_selected[data.id] = data.text;
@@ -355,8 +353,7 @@ $(document).ready(function () {
 
     var success = function(data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     };
 
     requestApi({

apps/assets/templates/assets/asset_list.html

@@ -523,8 +523,7 @@ $(document).ready(function(){
     function success(data) {
         rMenu.css({"visibility" : "hidden"});
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     }
     requestApi({
         url: the_url,
@@ -538,8 +537,7 @@ $(document).ready(function(){
     function success(data) {
         rMenu.css({"visibility" : "hidden"});
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     }
     requestApi({
         url: the_url,
@@ -552,4 +550,4 @@ $(document).ready(function(){
 
 </script>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/assets/templates/assets/system_user_assets.html

@@ -153,8 +153,7 @@ jumpserver.nodes_selected = {};
 
 $(document).ready(function () {
 	$('.select2').select2()
-    var url = "{% url 'api-assets:node-list' %}";
-    nodesSelect2Init(".nodes-select2", url)
+    nodesSelect2Init(".nodes-select2")
 	.on('select2:select', function(evt) {
             var data = evt.params.data;
             jumpserver.nodes_selected[data.id] = data.text;
@@ -202,8 +201,7 @@ $(document).ready(function () {
     };
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -219,8 +217,7 @@ $(document).ready(function () {
     the_url = the_url.replace("{{ DEFAULT_PK }}", asset_id);
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     var error = function (data) {
         alert(data)
@@ -239,8 +236,7 @@ $(document).ready(function () {
     };
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/templates/assets/system_user_detail.html

@@ -251,8 +251,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -265,8 +264,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/utils.py

 # ~*~ coding: utf-8 ~*~
 #
-from functools import reduce
 from treelib import Tree
+from treelib.exceptions import NodeIDAbsentError
 from collections import defaultdict
 from copy import deepcopy
-import threading
-from django.db.models import Q
 
 from common.utils import get_object_or_none, get_logger, timeit
-from .models import SystemUser, Label, Asset
+from .models import SystemUser, Asset
 
 
 logger = get_logger(__file__)
@@ -31,6 +29,7 @@ class TreeService(Tree):
         super().__init__(*args, **kwargs)
         self.nodes_assets_map = defaultdict(set)
         self.all_nodes_assets_map = {}
+        self._invalid_assets = frozenset()
 
     @classmethod
     @timeit
@@ -47,25 +46,34 @@ class TreeService(Tree):
                 key = node["key"]
                 value = node["value"]
                 parent_key = ":".join(key.split(":")[:-1])
-                tree.create_node(
+                tree.safe_create_node(
                     tag=value, identifier=key,
                     parent=parent_key,
                 )
             tree.init_assets()
         return tree
 
+    @timeit
     def init_assets(self):
         from orgs.utils import tmp_to_root_org
         self.all_nodes_assets_map = {}
         self.nodes_assets_map = defaultdict(set)
-        logger.debug('Init tree assets')
         with tmp_to_root_org():
             queryset = Asset.objects.all().values_list('id', 'nodes__key')
+            invalid_assets = Asset.objects.filter(is_active=False)\
+                .values_list('id', flat=True)
+            self._invalid_assets = frozenset(invalid_assets)
             for asset_id, key in queryset:
                 if not key:
                     continue
                 self.nodes_assets_map[key].add(asset_id)
 
+    def safe_create_node(self, **kwargs):
+        parent = kwargs.get("parent")
+        if not self.contains(parent):
+            kwargs['parent'] = self.root
+        self.create_node(**kwargs)
+
     def all_children_ids(self, nid, with_self=True):
         children_ids = self.expand_tree(nid)
         if not with_self:
@@ -93,6 +101,11 @@ class TreeService(Tree):
         children = self.all_children(nid, with_self=False)
         return ancestors + [self[nid]] + children
 
+    @staticmethod
+    def is_parent(child, parent):
+        parent_id = child.bpointer
+        return parent_id == parent.identifier
+
     def root_node(self):
         return self.get_node(self.root)
 
@@ -108,12 +121,15 @@ class TreeService(Tree):
             parent = self.copy_node(parent)
         return parent
 
+    def set_assets(self, nid, assets):
+        self.nodes_assets_map[nid] = set(assets)
+
     def assets(self, nid):
         assets = self.nodes_assets_map[nid]
         return assets
 
-    def set_assets(self, nid, assets):
-        self.nodes_assets_map[nid] = assets
+    def valid_assets(self, nid):
+        return set(self.assets(nid)) - set(self._invalid_assets)
 
     def all_assets(self, nid):
         assets = self.all_nodes_assets_map.get(nid)
@@ -123,41 +139,55 @@ class TreeService(Tree):
         children = self.children(nid)
         for child in children:
             assets.update(self.all_assets(child.identifier))
+        self.all_nodes_assets_map[nid] = assets
         return assets
 
+    def all_valid_assets(self, nid):
+        return set(self.all_assets(nid)) - set(self._invalid_assets)
+
     def assets_amount(self, nid):
         return len(self.all_assets(nid))
 
+    def valid_assets_amount(self, nid):
+        return len(self.all_valid_assets(nid))
+
     @staticmethod
     def copy_node(node):
         new_node = deepcopy(node)
         new_node.fpointer = None
         return new_node
 
-    def safe_add_ancestors(self, ancestors):
-        # 如果祖先节点为1个，那么添加该节点, 父节点是root node
-        if len(ancestors) == 1:
-            node = ancestors[0]
+    def safe_add_ancestors(self, node, ancestors):
+        # 如果没有祖先节点，那么添加该节点, 父节点是root node
+        if len(ancestors) == 0:
             parent = self.root_node()
         else:
-            node, ancestors = ancestors[0], ancestors[1:]
-            parent_id = ancestors[0].identifier
-            # 如果父节点不存在, 则先添加父节点
-            if not self.contains(parent_id):
-                self.safe_add_ancestors(ancestors)
-            parent = self.get_node(parent_id)
+            parent = ancestors[0]
 
         # 如果当前节点已再树中，则移动当前节点到父节点中
         # 这个是由于 当前节点放到了二级节点中
+        if not self.contains(parent.identifier):
+            # logger.debug('Add parent: {}'.format(parent.identifier))
+            self.safe_add_ancestors(parent, ancestors[1:])
+
         if self.contains(node.identifier):
+            # msg = 'Move node to parent: {} => {}'.format(
+            #     node.identifier, parent.identifier
+            # )
+            # logger.debug(msg)
             self.move_node(node.identifier, parent.identifier)
         else:
+            # logger.debug('Add node: {}'.format(node.identifier))
             self.add_node(node, parent)
+
     #
     # def __getstate__(self):
     #     self.mutex = None
     #     return self.__dict__
     #
-    # def __setstate__(self, state):
-    #     self.__dict__ = state
-    #     self.mutex = threading.Lock()
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+        if '_invalid_assets' not in state:
+            self._invalid_assets = frozenset()
+        # self.mutex = threading.Lock()

apps/authentication/templates/authentication/_access_key_modal.html

@@ -15,7 +15,8 @@
 
 </style>
 <div class="alert alert-info help-message" style="margin-left: 0px; margin-right: 0px;">
-    {% trans 'Using api key sign api header, every requests header difference'%}, <a href="https://tools.ietf.org/html/draft-cavage-http-signatures-08">{% trans 'docs' %} </a>
+    {% trans 'Using api key sign api header, every requests header difference'%}
+    <a href="https://jumpserver.readthedocs.io/zh/master/api_auth.html#access-key" target="_blank">{% trans 'docs' %} </a>
 </div>
 <div class="uc pull-left m-r-0 m-t-10">
     <button class="btn btn-primary btn-sm" id="create-btn" href="#"> {% trans "Create" %} </button>

apps/common/api.py

@@ -83,6 +83,8 @@ class LogTailApi(generics.RetrieveAPIView):
         return Response({"data": data, 'end': end, 'mark': new_mark})
 
 
+
+
 class ResourcesIDCacheApi(APIView):
     def post(self, request, *args, **kwargs):
         spm = str(uuid.uuid4())

apps/common/utils/common.py

@@ -194,7 +194,7 @@ def timeit(func):
         now = time.time()
         result = func(*args, **kwargs)
         using = (time.time() - now) * 1000
-        msg = "Call {} end, using: {:.1f}ms".format(func.__name__, using)
+        msg = "End call {}, using: {:.1f}ms".format(func.__name__, using)
         logger.debug(msg)
         return result
     return wrapper

apps/jumpserver/asgi.py

+import os
+import django
+from channels.routing import get_default_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jumpserver.settings")
+django.setup()
+application = get_default_application()

apps/jumpserver/conf.py

@@ -335,6 +335,7 @@ defaults = {
     'REDIS_DB_CELERY': 3,
     'REDIS_DB_CACHE': 4,
     'REDIS_DB_SESSION': 5,
+    'REDIS_DB_WS': 6,
     'CAPTCHA_TEST_MODE': None,
     'TOKEN_EXPIRATION': 3600 * 24,
     'DISPLAY_PER_PAGE': 25,
@@ -375,6 +376,7 @@ defaults = {
     'RADIUS_SECRET': '',
     'HTTP_BIND_HOST': '0.0.0.0',
     'HTTP_LISTEN_PORT': 8080,
+    'WS_LISTEN_PORT': 8070,
     'LOGIN_LOG_KEEP_DAYS': 90,
     'ASSETS_PERM_CACHE_TIME': 3600*24,
     'SECURITY_MFA_VERIFY_TTL': 3600,
@@ -383,7 +385,9 @@ defaults = {
     'SYSLOG_FACILITY': 'user',
     'PERM_SINGLE_ASSET_TO_UNGROUP_NODE': False,
     'WINDOWS_SSH_DEFAULT_SHELL': 'cmd',
-    'FLOWER_URL': "127.0.0.1:5555"
+    'FLOWER_URL': "127.0.0.1:5555",
+    'AUTH_LDAP_SEARCH_PAGED_SIZE': 1000,
+    'DEFAULT_ORG_SHOW_ALL_USERS': True,
 }
 
 

apps/jumpserver/routing.py

+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+
+from ops.urls.ws_urls import urlpatterns as ops_urlpatterns
+
+urlpatterns = []
+urlpatterns += ops_urlpatterns
+
+application = ProtocolTypeRouter({
+    'websocket': AuthMiddlewareStack(
+        URLRouter(urlpatterns)
+    ),
+})

apps/jumpserver/settings.py

@@ -74,6 +74,7 @@ INSTALLED_APPS = [
     'rest_framework',
     'rest_framework_swagger',
     'drf_yasg',
+    'channels',
     'django_filters',
     'bootstrap3',
     'captcha',
@@ -140,7 +141,8 @@ TEMPLATES = [
     },
 ]
 
-# WSGI_APPLICATION = 'jumpserver.wsgi.applications'
+WSGI_APPLICATION = 'jumpserver.wsgi.application'
+ASGI_APPLICATION = 'jumpserver.routing.application'
 
 LOGIN_REDIRECT_URL = reverse_lazy('index')
 LOGIN_URL = reverse_lazy('authentication:login')
@@ -422,6 +424,7 @@ OTP_VALID_WINDOW = CONFIG.OTP_VALID_WINDOW
 
 # Auth LDAP settings
 AUTH_LDAP = False
+AUTH_LDAP_SEARCH_PAGED_SIZE = CONFIG.AUTH_LDAP_SEARCH_PAGED_SIZE
 AUTH_LDAP_SERVER_URI = 'ldap://localhost:389'
 AUTH_LDAP_BIND_DN = 'cn=admin,dc=jumpserver,dc=org'
 AUTH_LDAP_BIND_PASSWORD = ''
@@ -621,6 +624,24 @@ ASSETS_PERM_CACHE_TIME = CONFIG.ASSETS_PERM_CACHE_TIME
 # Asset user auth external backend, default AuthBook backend
 BACKEND_ASSET_USER_AUTH_VAULT = False
 
+DEFAULT_ORG_SHOW_ALL_USERS = CONFIG.DEFAULT_ORG_SHOW_ALL_USERS
+
 PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
 WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
 FLOWER_URL = CONFIG.FLOWER_URL
+
+
+# Django channels support websocket
+CHANNEL_REDIS = "redis://:{}@{}:{}/{}".format(
+    CONFIG.REDIS_PASSWORD, CONFIG.REDIS_HOST, CONFIG.REDIS_PORT,
+    CONFIG.REDIS_DB_WS,
+)
+
+CHANNEL_LAYERS = {
+    'default': {
+        'BACKEND': 'channels_redis.core.RedisChannelLayer',
+        'CONFIG': {
+            "hosts": [CHANNEL_REDIS],
+        },
+    },
+}

apps/jumpserver/urls.py

@@ -66,6 +66,7 @@ urlpatterns = [
     re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', views.redirect_format_api),
     path('api/health/', views.HealthCheckView.as_view(), name="health"),
     path('luna/', views.LunaView.as_view(), name='luna-view'),
+    re_path('ws/.*', views.WsView.as_view(), name='ws-view'),
     path('i18n/<str:lang>/', views.I18NView.as_view(), name='i18n-switch'),
     path('settings/', include('settings.urls.view_urls', namespace='settings')),
 

apps/jumpserver/views.py

@@ -226,4 +226,11 @@ class HealthCheckView(APIView):
         return JsonResponse({"status": 1, "time": int(time.time())})
 
 
+class WsView(APIView):
+    ws_port = settings.CONFIG.HTTP_LISTEN_PORT + 1
+
+    def get(self, request):
+        msg = _("Websocket server run on port: {}, you should proxy it on nginx"
+                .format(self.ws_port))
+        return JsonResponse({"msg": msg})
 

apps/locale/zh/LC_MESSAGES/django.po

@@ -6135,7 +6135,7 @@ msgstr "管理员"
 
 #: xpack/plugins/orgs/forms.py:42
 msgid "Select auditor"
-msgstr "选择审计员员"
+msgstr "选择审计员"
 
 #: xpack/plugins/orgs/meta.py:8 xpack/plugins/orgs/views.py:26
 #: xpack/plugins/orgs/views.py:43 xpack/plugins/orgs/views.py:60

apps/ops/templates/ops/adhoc_detail.html

@@ -196,8 +196,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 </script>
 {% endblock %}

apps/ops/templates/ops/adhoc_history_detail.html

@@ -145,8 +145,8 @@
 <script>
 $(document).ready(function () {
 }).on('click', '.celery-task-log', function () {
-    var url = '{% url 'ops:celery-task-log' pk=object.pk %}';
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    var taskId = "{{ object.pk }}";
+    showCeleryTaskLog(taskId);
 })
 
 </script>

apps/ops/templates/ops/celery_task_log.html

@@ -20,50 +20,13 @@
 </div>
 
 <script>
-    var rowHeight = 18;
-    var colWidth = 10;
-    var mark = '';
-    var url = "{% url 'api-ops:celery-task-log' pk=task_id %}";
+    var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+    var port = document.location.port ? ":" + document.location.port : "";
+    var url = "/ws/ops/tasks/" + "{{ task_id }}" + "/log/";
+    var wsURL = scheme + "://" + document.location.hostname + port + url;
     var term;
-    var end = false;
-    var error = false;
-    var interval = 200;
-    var success = true;
+    var ws;
 
-    function calWinSize() {
-        var t = $('#marker');
-        {#rowHeight = 1.00 * t.height();#}
-        {#colWidth = 1.00 * t.width() / 6;#}
-    }
-    function resize() {
-       {#var rows = Math.floor(window.innerHeight / rowHeight) - 1;#}
-       {#var cols = Math.floor(window.innerWidth / colWidth) - 2;#}
-       {#term.resize(cols, rows);#}
-    }
-    function requestAndWrite() {
-        if (!end && success) {
-            success = false;
-            $.ajax({
-                url: url + '?mark=' + mark,
-                method: "GET",
-                contentType: "application/json; charset=utf-8"
-            }).done(function(data, textStatue, jqXHR) {
-                success = true;
-                if (jqXHR.status === 203) {
-                    error = true;
-                    term.write('.');
-                    interval = 500;
-                }
-                if (jqXHR.status === 200){
-                    term.write(data.data);
-                    mark = data.mark;
-                    if (data.end){
-                        end = true
-                    }
-                }
-            })
-        }
-    }
     $(document).ready(function () {
         term = new Terminal({
             cursorBlink: false,
@@ -74,18 +37,14 @@
             disableStdin: true
         });
         term.open(document.getElementById('term'));
-        term.resize(90, 32);
-        resize();
-        term.on('data', function (data) {
-            {#term.write(data.replace('\r', '\r\n'))#}
-            term.write(data);
-        });
-        window.onresize = function () {
-            resize()
+        term.resize(120, 30);
+        ws = new WebSocket(wsURL);
+        ws.onmessage = function(e) {
+            var data = JSON.parse(e.data);
+            term.write(data.message);
         };
-        {#$('.terminal').detach().appendTo('#term');#}
-        setInterval(function () {
-            requestAndWrite()
-        }, interval)
+        ws.onerror = function (e) {
+            term.write("Connect websocket server error")
+        }
     });
 </script>

apps/ops/templates/ops/task_adhoc.html

@@ -130,8 +130,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 </script>
 {% endblock %}

apps/ops/templates/ops/task_detail.html

@@ -174,8 +174,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 
 </script>

apps/ops/templates/ops/task_history.html

@@ -155,8 +155,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 
 </script>

apps/ops/templates/ops/task_list.html

@@ -98,8 +98,7 @@ $(document).ready(function () {
     };
     var success = function(data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/ops/urls/ws_urls.py

+from django.urls import path
+
+from .. import ws
+
+app_name = 'ops'
+
+urlpatterns = [
+    path('ws/ops/tasks/<uuid:task_id>/log/', ws.CeleryLogWebsocket, name='task-log-ws'),
+]

apps/ops/ws.py

+import time
+import threading
+
+from .celery.utils import get_celery_task_log_path
+from channels.generic.websocket import JsonWebsocketConsumer
+
+
+class CeleryLogWebsocket(JsonWebsocketConsumer):
+    task = ''
+    task_log_f = None
+    disconnected = False
+
+    def connect(self):
+        task_id = self.scope['url_route']['kwargs']['task_id']
+        log_path = get_celery_task_log_path(task_id)
+        try:
+            self.task_log_f = open(log_path)
+        except OSError:
+            self.send({'message': "Task {} log not found".format(task_id)})
+            self.disconnect(None)
+            return
+
+        self.accept()
+        self.send_log_to_client()
+
+    def disconnect(self, close_code):
+        self.disconnected = True
+        if self.task_log_f and not self.task_log_f.closed:
+            self.task_log_f.close()
+        self.close()
+
+    def send_log_to_client(self):
+        def func():
+            while not self.disconnected:
+                data = self.task_log_f.read(4096)
+                if data:
+                    data = data.replace('\n', '\r\n')
+                    self.send_json({'message': data})
+                time.sleep(0.2)
+        thread = threading.Thread(target=func)
+        thread.start()

apps/orgs/models.py

 import uuid
+from django.conf import settings
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -75,7 +76,10 @@ class Organization(models.Model):
         from users.models import User
         if self.is_real():
             return self.users.all()
-        return User.objects.filter(role=User.ROLE_USER)
+        users = User.objects.filter(role=User.ROLE_USER)
+        if self.is_default() and not settings.DEFAULT_ORG_SHOW_ALL_USERS:
+            users = users.filter(related_user_orgs__isnull=True)
+        return users
 
     def get_org_admins(self):
         from users.models import User
@@ -130,7 +134,7 @@ class Organization(models.Model):
         return admin_orgs
 
     @classmethod
-    def get_user_user_orgs(self, user):
+    def get_user_user_orgs(cls, user):
         user_orgs = []
         if user.is_anonymous:
             return user_orgs

apps/orgs/utils.py

@@ -11,8 +11,17 @@ def get_org_from_request(request):
     oid = request.session.get("oid")
     if not oid:
         oid = request.META.get("HTTP_X_JMS_ORG")
+
+    request_params_oid = request.GET.get("oid")
+    if request_params_oid:
+        oid = request.GET.get("oid")
+
     if not oid:
         oid = Organization.DEFAULT_ID
+    if oid.lower() == "default":
+        oid = Organization.DEFAULT_ID
+    elif oid.lower() == "root":
+        oid = Organization.ROOT_ID
     org = Organization.get_instance(oid)
     return org
 

apps/perms/api/user_permission/mixin.py

@@ -33,7 +33,7 @@ class UserNodeTreeMixin:
         _queryset = []
 
         for node in nodes:
-            assets_amount = self.tree.assets_amount(node.key)
+            assets_amount = self.tree.valid_assets_amount(node.key)
             if assets_amount == 0 and node.key != Node.empty_key:
                 continue
             node.assets_amount = assets_amount

apps/perms/forms/asset_permission.py

@@ -38,20 +38,21 @@ class AssetPermissionForm(OrgModelForm):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        users_field = self.fields.get('users')
-        users_field.queryset = current_org.get_org_members(exclude=('Auditor',))
 
         if self.data:
             return
         # 前端渲染优化, 防止过多资产
+        users_field = self.fields.get('users')
         assets_field = self.fields['assets']
         nodes_field = self.fields['nodes']
         if self.instance:
             assets_field.queryset = self.instance.assets.all()
             nodes_field.queryset = self.instance.nodes.all()
+            users_field.queryset = self.instance.users.all()
         else:
             assets_field.queryset = Asset.objects.none()
             nodes_field.queryset = Node.objects.none()
+            users_field.queryset = []
 
     def set_nodes_initial(self, nodes):
         field = self.fields['nodes']
@@ -70,7 +71,7 @@ class AssetPermissionForm(OrgModelForm):
         )
         widgets = {
             'users': forms.SelectMultiple(
-                attrs={'class': 'select2', 'data-placeholder': _("User")}
+                attrs={'class': 'users-select2', 'data-placeholder': _("User")}
             ),
             'user_groups': forms.SelectMultiple(
                 attrs={'class': 'select2', 'data-placeholder': _("User group")}

apps/perms/forms/remote_app_permission.py

@@ -17,9 +17,12 @@ __all__ = [
 class RemoteAppPermissionCreateUpdateForm(OrgModelForm):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+
         users_field = self.fields.get('users')
-        if hasattr(users_field, 'queryset'):
-            users_field.queryset = current_org.get_org_members(exclude=('Auditor',))
+        if self.instance:
+            users_field.queryset = self.instance.users.all()
+        else:
+            users_field.queryset = []
 
     class Meta:
         model = RemoteAppPermission
@@ -28,7 +31,7 @@ class RemoteAppPermissionCreateUpdateForm(OrgModelForm):
         )
         widgets = {
             'users': forms.SelectMultiple(
-                attrs={'class': 'select2', 'data-placeholder': _('User')}
+                attrs={'class': 'users-select2', 'data-placeholder': _('User')}
             ),
             'user_groups': forms.SelectMultiple(
                 attrs={'class': 'select2', 'data-placeholder': _('User group')}

apps/perms/templates/perms/asset_permission_asset.html

@@ -204,8 +204,7 @@ $(document).ready(function () {
     $('.select2').select2();
     table = initAssetTable();
 
-    var nodeListUrl = "{% url 'api-assets:node-list' %}";
-    nodesSelect2Init(".nodes-select2", nodeListUrl);
+    nodesSelect2Init(".nodes-select2");
 
     $("#id_assets").parent().find(".select2-selection").on('click', function (e) {
         if ($(e.target).attr('class') !== 'select2-selection__choice__remove'){

apps/perms/templates/perms/asset_permission_create_update.html

@@ -116,8 +116,8 @@ $(document).ready(function () {
     $('.select2').select2({
         closeOnSelect: false
     });
-    var url = "{% url 'api-assets:node-list' %}";
-    nodesSelect2Init(".nodes-select2", url);
+    nodesSelect2Init(".nodes-select2");
+    usersSelect2Init(".users-select2");
 
     $('#date_start').daterangepicker(dateOptions);
     $('#date_expired').daterangepicker(dateOptions);
@@ -154,4 +154,4 @@ $(document).ready(function () {
     formSubmit(props);
  })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/perms/templates/perms/remote_app_permission_create_update.html

@@ -114,6 +114,7 @@ $(document).ready(function () {
     $('.select2').select2({
         closeOnSelect: false
     });
+    usersSelect2Init('.users-select2');
     $('#date_start').daterangepicker(dateOptions);
     $('#date_expired').daterangepicker(dateOptions);
 })
@@ -141,4 +142,4 @@ $(document).ready(function () {
     formSubmit(props);
  })
 </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

apps/perms/utils/asset_permission.py

 # coding: utf-8
 
-import time
 import pickle
 from collections import defaultdict
 from functools import reduce
-from hashlib import md5
-import json
 
 from django.core.cache import cache
 from django.db.models import Q
@@ -94,6 +91,7 @@ class AssetPermissionUtilCacheMixin:
         user_tree = pickle.loads(data)
         return user_tree
 
+    @timeit
     def get_user_tree_from_cache_if_need(self):
         if not self.user_tree_cache_enable:
             return None
@@ -278,9 +276,9 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
             )
             if not ancestors:
                 continue
-            parent_id = ancestors[0].identifier
-            user_tree.safe_add_ancestors(ancestors)
-            user_tree.move_node(child.identifier, parent_id)
+            user_tree.safe_add_ancestors(child, ancestors)
+            # parent_id = ancestors[0].identifier
+            # user_tree.move_node(child.identifier, parent_id)
 
     @staticmethod
     def add_empty_node_if_need(user_tree):
@@ -315,6 +313,7 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
             self.set_user_tree_to_local(user_tree)
             return user_tree
         user_tree = TreeService()
+        user_tree._invalid_assets = self.full_tree._invalid_assets
         full_tree_root = self.full_tree.root_node()
         user_tree.create_node(
             tag=full_tree_root.tag,

apps/settings/utils.py

@@ -26,6 +26,8 @@ class LDAPUtil:
                  password=None, use_ssl=None, search_ougroup=None,
                  search_filter=None, attr_map=None, auth_ldap=None):
         # config
+        self.paged_size = settings.AUTH_LDAP_SEARCH_PAGED_SIZE
+
         if use_settings_config:
             self._load_config_from_settings()
         else:
@@ -79,23 +81,45 @@ class LDAPUtil:
             user_item[attr] = value
         return user_item
 
+    def _search_user_items_ou(self, search_ou, cookie=None):
+        ok = self.connection.search(
+            search_ou, self.search_filter % ({"user": "*"}),
+            attributes=list(self.attr_map.values()),
+            paged_size=self.paged_size, paged_cookie=cookie
+        )
+        if not ok:
+            error = _("Search no entry matched in ou {}".format(search_ou))
+            raise LDAPOUGroupException(error)
+
+        user_items = []
+        for entry in self.connection.entries:
+            user_item = self._ldap_entry_to_user_item(entry)
+            user = self.get_user_by_username(user_item['username'])
+            user_item['existing'] = bool(user)
+            if user_item in user_items:
+                continue
+            user_items.append(user_item)
+        return user_items
+
+    def _cookie(self):
+        if self.paged_size is None:
+            cookie = None
+        else:
+            cookie = self.connection.result['controls']['1.2.840.113556.1.4.319']['value']['cookie']
+        return cookie
+
     def search_user_items(self):
         user_items = []
+        logger.info("Search user items")
         for search_ou in str(self.search_ougroup).split("|"):
-            ok = self.connection.search(
-                search_ou, self.search_filter % ({"user": "*"}),
-                attributes=list(self.attr_map.values())
-            )
-            if not ok:
-                error = _("Search no entry matched in ou {}".format(search_ou))
-                raise LDAPOUGroupException(error)
-            for entry in self.connection.entries:
-                user_item = self._ldap_entry_to_user_item(entry)
-                user = self.get_user_by_username(user_item['username'])
-                user_item['existing'] = bool(user)
-                if user_item in user_items:
-                    continue
-                user_items.append(user_item)
+            logger.info("Search user search ou: {}".format(search_ou))
+            _user_items = self._search_user_items_ou(search_ou)
+            user_items.extend(_user_items)
+            while self._cookie():
+                logger.info("Page Search user search ou: {}".format(search_ou))
+                _user_items = self._search_user_items_ou(search_ou, self._cookie())
+                user_items.extend(_user_items)
+        logger.info("Search user items end")
         return user_items
 
     def search_filter_user_items(self, username_list):

apps/static/css/jumpserver.css

@@ -88,7 +88,7 @@ table.dataTable tbody td.selected a,
 table.dataTable tbody tr.selected td i.text-navy,
 table.dataTable tbody th.selected td i.text-navy,
 table.dataTable tbody td.selected td i.text-navy {
-    color: white !important;
+    color: white;
 }
 
 .m-0 {
@@ -473,4 +473,4 @@ span.select2-selection__placeholder {
 
 .p-r-5 {
     padding-right: 5px;
-}
\ No newline at end of file
+}

apps/static/js/jumpserver.js

@@ -1177,6 +1177,9 @@ function readFile(ref) {
 }
 
 function nodesSelect2Init(selector, url) {
+    if (!url) {
+        url = '/api/v1/assets/nodes/'
+    }
     return $(selector).select2({
         closeOnSelect: false,
         ajax: {
@@ -1201,3 +1204,37 @@ function nodesSelect2Init(selector, url) {
     })
 }
 
+function usersSelect2Init(selector, url) {
+    if (!url) {
+        url = '/api/v1/users/users/'
+    }
+    return $(selector).select2({
+        closeOnSelect: false,
+        ajax: {
+            url: url,
+            data: function (params) {
+                var page = params.page || 1;
+                var query = {
+                    search: params.term,
+                    offset: (page - 1) * 10,
+                    limit: 10
+                };
+                return query
+            },
+            processResults: function (data) {
+                var results = $.map(data.results, function (v, i) {
+                    var display = v.name + '(' + v.username +')';
+                    return {id: v.id, text: display}
+                });
+                var more = !!data.next;
+                return {results: results, pagination: {"more": more}}
+            }
+        },
+    })
+
+}
+
+function showCeleryTaskLog(taskId) {
+    var url = '/ops/celery/task/taskId/log/'.replace('taskId', taskId);
+    window.open(url, '', 'width=900,height=600')
+}

apps/users/api/user.py

@@ -12,7 +12,7 @@ from rest_framework_bulk import BulkModelViewSet
 
 from common.permissions import (
     IsOrgAdmin, IsCurrentUserOrReadOnly, IsOrgAdminOrAppUser,
-    CanUpdateDeleteUser,
+    CanUpdateDeleteUser, IsSuperUser
 )
 from common.mixins import CommonApiMixin
 from common.utils import get_logger
@@ -52,12 +52,15 @@ class UserViewSet(CommonApiMixin, BulkModelViewSet):
         self.send_created_signal(users)
 
     def get_queryset(self):
-        queryset = current_org.get_org_members().prefetch_related('groups')
+        queryset = current_org.get_org_members()\
+            .prefetch_related('groups')
         return queryset
 
     def get_permissions(self):
         if self.action in ["retrieve", "list"]:
             self.permission_classes = (IsOrgAdminOrAppUser,)
+        if self.request.query_params.get('all'):
+            self.permission_classes = (IsSuperUser,)
         return super().get_permissions()
 
     def perform_bulk_destroy(self, objects):
@@ -176,4 +179,4 @@ class UserResetOTPApi(generics.RetrieveAPIView):
             user.otp_secret_key = ''
             user.save()
             logout(request)
-        return Response({"msg": "success"})
\ No newline at end of file
+        return Response({"msg": "success"})

apps/users/forms.py

@@ -308,11 +308,11 @@ class UserBulkUpdateForm(OrgModelForm):
 
 class UserGroupForm(OrgModelForm):
     users = forms.ModelMultipleChoiceField(
-        queryset=User.objects.all(),
+        queryset=User.objects.none(),
         label=_("User"),
         widget=forms.SelectMultiple(
             attrs={
-                'class': 'select2',
+                'class': 'users-select2',
                 'data-placeholder': _('Select users')
             }
         ),
@@ -329,8 +329,10 @@ class UserGroupForm(OrgModelForm):
         if 'initial' not in kwargs:
             return
         users_field = self.fields.get('users')
-        if hasattr(users_field, 'queryset'):
-            users_field.queryset = current_org.get_org_members(exclude=('Auditor',))
+        if instance:
+            users_field.queryset = instance.users.all()
+        else:
+            users_field.queryset = User.objects.none()
 
     def save(self, commit=True):
         group = super().save(commit=commit)

apps/users/templates/users/_granted_assets.html

@@ -103,6 +103,7 @@ function onSelected(event, treeNode) {
 
 
 function initTree(refresh) {
+    var asyncUrl = setUrlParam(treeUrl, 'cache_policy', '1');
     var setting = {
         view: {
             dblClickExpand: false,
@@ -115,7 +116,7 @@ function initTree(refresh) {
         },
         async: {
 			enable: true,
-			url: treeUrl,
+			url: asyncUrl,
 			autoParam: ["id=key", "name=n", "level=lv"],
             type: 'get'
 		},

apps/users/templates/users/user_group_create_update.html

@@ -46,6 +46,7 @@ $(document).ready(function () {
     $('.select2').select2({
         closeOnSelect: false
     });
+    usersSelect2Init('.users-select2')
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();

config_example.yml

@@ -44,6 +44,7 @@ DB_NAME: jumpserver
 # 运行时绑定端口
 HTTP_BIND_HOST: 0.0.0.0
 HTTP_LISTEN_PORT: 8080
+WS_LISTEN_PORT: 8070
 
 # Use Redis as broker for celery and web socket
 # Redis配置

jms

@@ -47,6 +47,7 @@ LOG_DIR = os.path.join(BASE_DIR, 'logs')
 TMP_DIR = os.path.join(BASE_DIR, 'tmp')
 HTTP_HOST = CONFIG.HTTP_BIND_HOST or '127.0.0.1'
 HTTP_PORT = CONFIG.HTTP_LISTEN_PORT or 8080
+WS_PORT = CONFIG.WS_LISTEN_PORT or 8082
 DEBUG = CONFIG.DEBUG or False
 LOG_LEVEL = CONFIG.LOG_LEVEL or 'INFO'
 
@@ -201,12 +202,15 @@ def is_running(s, unlink=True):
 
 def parse_service(s):
     all_services = [
-        'gunicorn', 'celery_ansible', 'celery_default', 'beat', 'flower'
+        'gunicorn', 'celery_ansible', 'celery_default',
+        'beat', 'flower', 'daphne',
     ]
     if s == 'all':
         return all_services
     elif s == "web":
-        return ['gunicorn', 'flower']
+        return ['gunicorn', 'flower', 'daphne']
+    elif s == 'ws':
+        return ['daphne']
     elif s == "task":
         return ["celery_ansible", "celery_default", "beat"]
     elif s == 'gunicorn':
@@ -225,10 +229,8 @@ def parse_service(s):
 def get_start_gunicorn_kwargs():
     print("\n- Start Gunicorn WSGI HTTP Server")
     prepare()
-    service = 'gunicorn'
     bind = '{}:{}'.format(HTTP_HOST, HTTP_PORT)
     log_format = '%(h)s %(t)s "%(r)s" %(s)s %(b)s '
-    pid_file = get_pid_file_path(service)
 
     cmd = [
         'gunicorn', 'jumpserver.wsgi',
@@ -238,7 +240,6 @@ def get_start_gunicorn_kwargs():
         '-w', str(WORKERS),
         '--max-requests', '4096',
         '--access-logformat', log_format,
-        '-p', pid_file,
         '--access-logfile', '-'
     ]
 
@@ -247,6 +248,16 @@ def get_start_gunicorn_kwargs():
     return {'cmd': cmd, 'cwd': APPS_DIR}
 
 
+def get_start_daphne_kwargs():
+    print("\n- Start Daphne ASGI WS Server")
+    cmd = [
+        'daphne', 'jumpserver.asgi:application',
+        '-b', HTTP_HOST,
+        '-p', str(WS_PORT),
+    ]
+    return {'cmd': cmd, 'cwd': APPS_DIR}
+
+
 def get_start_celery_ansible_kwargs():
     print("\n- Start Celery as Distributed Task Queue: Ansible")
     return get_start_worker_kwargs('ansible', 4)
@@ -362,6 +373,7 @@ def start_service(s):
         "celery_default": get_start_celery_default_kwargs,
         "beat": get_start_beat_kwargs,
         "flower": get_start_flower_kwargs,
+        "daphne": get_start_daphne_kwargs,
     }
 
     kwargs = services_kwargs.get(s)()

requirements/requirements.txt

@@ -59,7 +59,7 @@ pytz==2018.3
 PyYAML==5.1
 redis==2.10.6
 requests==2.22.0
-jms-storage==0.0.23
+jms-storage==0.0.25
 s3transfer==0.1.13
 simplejson==3.13.2
 six==1.11.0
@@ -86,3 +86,6 @@ httpsig==1.3.0
 treelib==1.5.3
 django-proxy==1.2.1
 flower==0.9.3
+channels-redis==2.4.0
+channels==2.3.0
+daphne==2.3.0

requirements/rpm_requirements.txt

-libtiff-devel libjpeg-devel libzip-devel freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel sshpass openldap-devel mariadb-devel mysql-devel libffi-devel openssh-clients telnet openldap-clients 
+gcc krb5-devel libtiff-devel libjpeg-devel libzip-devel freetype-devel lcms2-devel libwebp-devel tcl-devel tk-devel sshpass openldap-devel mariadb-devel mysql-devel libffi-devel openssh-clients telnet openldap-clients