Support ws (#3291)

* [Update] add ws support

* [Update] 修改log使用websocket

* [Update] 修复 资产用户 右侧动作菜单，字体颜色

* [Update] 修改Dockerfile

* [Bugfix] 修复settings中WSG_APPLICATION

Dockerfile

@@ -22,4 +22,5 @@ ENV LANG=zh_CN.UTF-8
 ENV LC_ALL=zh_CN.UTF-8
 
 EXPOSE 8080
+EXPOSE 8081
 ENTRYPOINT ["./entrypoint.sh"]

apps/assets/templates/assets/_asset_user_list.html

 {% load i18n %}
 <style>
-.btn-group>.btn+.dropdown-toggle {
+    .btn-group>.btn+.dropdown-toggle {
         padding-right: 4px;
         padding-left: 4px;
-}
+    }
+
+    table.dataTable tbody tr.selected a {
+        color: rgb(103, 106, 108);;
+    }
+
 </style>
 <table class="table table-striped table-bordered table-hover" id="asset_user_list_table" style="width: 100%">
   <thead>
@@ -137,8 +142,7 @@ $(document).ready(function(){
     }
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/templates/assets/admin_user_assets.html

@@ -85,8 +85,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:admin-user-connective' pk=admin_user.id %}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/templates/assets/asset_asset_user_list.html

@@ -81,8 +81,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:asset-user-connective' %}" + "?asset_id={{ asset.id }}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/templates/assets/asset_detail.html

@@ -276,8 +276,7 @@ function refreshAssetHardware() {
     var success = function(data) {
         console.log(data);
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -355,8 +354,7 @@ $(document).ready(function () {
 
     var success = function(data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     };
 
     requestApi({

apps/assets/templates/assets/asset_list.html

@@ -523,8 +523,7 @@ $(document).ready(function(){
     function success(data) {
         rMenu.css({"visibility" : "hidden"});
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     }
     requestApi({
         url: the_url,
@@ -538,8 +537,7 @@ $(document).ready(function(){
     function success(data) {
         rMenu.css({"visibility" : "hidden"});
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     }
     requestApi({
         url: the_url,

apps/assets/templates/assets/system_user_assets.html

@@ -202,8 +202,7 @@ $(document).ready(function () {
     };
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -219,8 +218,7 @@ $(document).ready(function () {
     the_url = the_url.replace("{{ DEFAULT_PK }}", asset_id);
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     var error = function (data) {
         alert(data)
@@ -239,8 +237,7 @@ $(document).ready(function () {
     };
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/assets/templates/assets/system_user_detail.html

@@ -251,8 +251,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,
@@ -265,8 +264,7 @@ $(document).ready(function () {
     var the_url = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
     var success = function (data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/common/api.py

@@ -83,6 +83,8 @@ class LogTailApi(generics.RetrieveAPIView):
         return Response({"data": data, 'end': end, 'mark': new_mark})
 
 
+
+
 class ResourcesIDCacheApi(APIView):
     def post(self, request, *args, **kwargs):
         spm = str(uuid.uuid4())

apps/jumpserver/asgi.py

+import os
+import django
+from channels.routing import get_default_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jumpserver.settings")
+django.setup()
+application = get_default_application()

apps/jumpserver/conf.py

@@ -335,6 +335,7 @@ defaults = {
     'REDIS_DB_CELERY': 3,
     'REDIS_DB_CACHE': 4,
     'REDIS_DB_SESSION': 5,
+    'REDIS_DB_WS': 6,
     'CAPTCHA_TEST_MODE': None,
     'TOKEN_EXPIRATION': 3600 * 24,
     'DISPLAY_PER_PAGE': 25,

apps/jumpserver/routing.py

+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+
+from ops.urls.ws_urls import urlpatterns as ops_urlpatterns
+
+urlpatterns = []
+urlpatterns += ops_urlpatterns
+
+application = ProtocolTypeRouter({
+    'websocket': AuthMiddlewareStack(
+        URLRouter(urlpatterns)
+    ),
+})

apps/jumpserver/settings.py

@@ -74,6 +74,7 @@ INSTALLED_APPS = [
     'rest_framework',
     'rest_framework_swagger',
     'drf_yasg',
+    'channels',
     'django_filters',
     'bootstrap3',
     'captcha',
@@ -140,7 +141,8 @@ TEMPLATES = [
     },
 ]
 
-# WSGI_APPLICATION = 'jumpserver.wsgi.applications'
+WSGI_APPLICATION = 'jumpserver.wsgi.application'
+ASGI_APPLICATION = 'jumpserver.routing.application'
 
 LOGIN_REDIRECT_URL = reverse_lazy('index')
 LOGIN_URL = reverse_lazy('authentication:login')
@@ -624,3 +626,19 @@ BACKEND_ASSET_USER_AUTH_VAULT = False
 PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
 WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
 FLOWER_URL = CONFIG.FLOWER_URL
+
+
+# Django channels support websocket
+CHANNEL_REDIS = "redis://:{}@{}:{}/{}".format(
+    CONFIG.REDIS_PASSWORD, CONFIG.REDIS_HOST, CONFIG.REDIS_PORT,
+    CONFIG.REDIS_DB_WS,
+)
+
+CHANNEL_LAYERS = {
+    'default': {
+        'BACKEND': 'channels_redis.core.RedisChannelLayer',
+        'CONFIG': {
+            "hosts": [CHANNEL_REDIS],
+        },
+    },
+}

apps/jumpserver/urls.py

@@ -66,6 +66,7 @@ urlpatterns = [
     re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', views.redirect_format_api),
     path('api/health/', views.HealthCheckView.as_view(), name="health"),
     path('luna/', views.LunaView.as_view(), name='luna-view'),
+    re_path('ws/.*', views.WsView.as_view(), name='ws-view'),
     path('i18n/<str:lang>/', views.I18NView.as_view(), name='i18n-switch'),
     path('settings/', include('settings.urls.view_urls', namespace='settings')),
 

apps/jumpserver/views.py

@@ -226,4 +226,11 @@ class HealthCheckView(APIView):
         return JsonResponse({"status": 1, "time": int(time.time())})
 
 
+class WsView(APIView):
+    ws_port = settings.CONFIG.HTTP_LISTEN_PORT + 1
+
+    def get(self, request):
+        msg = _("Websocket server run on port: {}, you should proxy it on nginx"
+                .format(self.ws_port))
+        return JsonResponse({"msg": msg})
 

apps/ops/templates/ops/adhoc_detail.html

@@ -196,8 +196,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 </script>
 {% endblock %}

apps/ops/templates/ops/adhoc_history_detail.html

@@ -145,8 +145,8 @@
 <script>
 $(document).ready(function () {
 }).on('click', '.celery-task-log', function () {
-    var url = '{% url 'ops:celery-task-log' pk=object.pk %}';
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    var taskId = "{{ object.pk }}";
+    showCeleryTaskLog(taskId);
 })
 
 </script>

apps/ops/templates/ops/celery_task_log.html

@@ -20,50 +20,13 @@
 </div>
 
 <script>
-    var rowHeight = 18;
-    var colWidth = 10;
-    var mark = '';
-    var url = "{% url 'api-ops:celery-task-log' pk=task_id %}";
+    var scheme = document.location.protocol === "https:" ? "wss" : "ws";
+    var port = document.location.port ? ":" + document.location.port : "";
+    var url = "/ws/ops/tasks/" + "{{ task_id }}" + "/log/";
+    var wsURL = scheme + "://" + document.location.hostname + port + url;
     var term;
-    var end = false;
-    var error = false;
-    var interval = 200;
-    var success = true;
+    var ws;
 
-    function calWinSize() {
-        var t = $('#marker');
-        {#rowHeight = 1.00 * t.height();#}
-        {#colWidth = 1.00 * t.width() / 6;#}
-    }
-    function resize() {
-       {#var rows = Math.floor(window.innerHeight / rowHeight) - 1;#}
-       {#var cols = Math.floor(window.innerWidth / colWidth) - 2;#}
-       {#term.resize(cols, rows);#}
-    }
-    function requestAndWrite() {
-        if (!end && success) {
-            success = false;
-            $.ajax({
-                url: url + '?mark=' + mark,
-                method: "GET",
-                contentType: "application/json; charset=utf-8"
-            }).done(function(data, textStatue, jqXHR) {
-                success = true;
-                if (jqXHR.status === 203) {
-                    error = true;
-                    term.write('.');
-                    interval = 500;
-                }
-                if (jqXHR.status === 200){
-                    term.write(data.data);
-                    mark = data.mark;
-                    if (data.end){
-                        end = true
-                    }
-                }
-            })
-        }
-    }
     $(document).ready(function () {
         term = new Terminal({
             cursorBlink: false,
@@ -74,18 +37,14 @@
             disableStdin: true
         });
         term.open(document.getElementById('term'));
-        term.resize(90, 32);
-        resize();
-        term.on('data', function (data) {
-            {#term.write(data.replace('\r', '\r\n'))#}
-            term.write(data);
-        });
-        window.onresize = function () {
-            resize()
+        term.resize(120, 30);
+        ws = new WebSocket(wsURL);
+        ws.onmessage = function(e) {
+            var data = JSON.parse(e.data);
+            term.write(data.message);
         };
-        {#$('.terminal').detach().appendTo('#term');#}
-        setInterval(function () {
-            requestAndWrite()
-        }, interval)
+        ws.onerror = function (e) {
+            term.write("Connect websocket server error")
+        }
     });
 </script>

apps/ops/templates/ops/task_adhoc.html

@@ -130,8 +130,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 </script>
 {% endblock %}

apps/ops/templates/ops/task_detail.html

@@ -174,8 +174,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 
 </script>

apps/ops/templates/ops/task_history.html

@@ -155,8 +155,7 @@ $(document).ready(function () {
         alert("没有运行历史");
         return
     }
-    var url = '{% url 'ops:celery-task-log' pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', history_pk);
-    window.open(url, '', 'width=800,height=600,left=400,top=400')
+    showCeleryTaskLog(history_pk);
 })
 
 </script>

apps/ops/templates/ops/task_list.html

@@ -98,8 +98,7 @@ $(document).ready(function () {
     };
     var success = function(data) {
         var task_id = data.task;
-        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=600,left=400,top=400')
+        showCeleryTaskLog(task_id);
     };
     requestApi({
         url: the_url,

apps/ops/urls/ws_urls.py

+from django.urls import path
+
+from .. import ws
+
+app_name = 'ops'
+
+urlpatterns = [
+    path('ws/ops/tasks/<uuid:task_id>/log/', ws.CeleryLogWebsocket, name='task-log-ws'),
+]

apps/ops/ws.py

+import time
+import threading
+
+from .celery.utils import get_celery_task_log_path
+from channels.generic.websocket import JsonWebsocketConsumer
+
+
+class CeleryLogWebsocket(JsonWebsocketConsumer):
+    task = ''
+    task_log_f = None
+    disconnected = False
+
+    def connect(self):
+        task_id = self.scope['url_route']['kwargs']['task_id']
+        log_path = get_celery_task_log_path(task_id)
+        try:
+            self.task_log_f = open(log_path)
+        except OSError:
+            self.send({'message': "Task {} log not found".format(task_id)})
+            self.disconnect(None)
+            return
+
+        self.accept()
+        self.send_log_to_client()
+
+    def disconnect(self, close_code):
+        self.disconnected = True
+        if self.task_log_f and not self.task_log_f.closed:
+            self.task_log_f.close()
+        self.close()
+
+    def send_log_to_client(self):
+        def func():
+            while not self.disconnected:
+                data = self.task_log_f.read(4096)
+                if data:
+                    data = data.replace('\n', '\r\n')
+                    self.send_json({'message': data})
+                time.sleep(0.2)
+        thread = threading.Thread(target=func)
+        thread.start()

apps/static/css/jumpserver.css

@@ -88,7 +88,7 @@ table.dataTable tbody td.selected a,
 table.dataTable tbody tr.selected td i.text-navy,
 table.dataTable tbody th.selected td i.text-navy,
 table.dataTable tbody td.selected td i.text-navy {
-    color: white !important;
+    color: white;
 }
 
 .m-0 {

apps/static/js/jumpserver.js

@@ -1201,3 +1201,7 @@ function nodesSelect2Init(selector, url) {
     })
 }
 
+function showCeleryTaskLog(taskId) {
+    var url = '/ops/celery/task/taskId/log/'.replace('taskId', taskId);
+    window.open(url, '', 'width=900,height=600')
+}

jms

@@ -47,6 +47,7 @@ LOG_DIR = os.path.join(BASE_DIR, 'logs')
 TMP_DIR = os.path.join(BASE_DIR, 'tmp')
 HTTP_HOST = CONFIG.HTTP_BIND_HOST or '127.0.0.1'
 HTTP_PORT = CONFIG.HTTP_LISTEN_PORT or 8080
+WS_PORT = HTTP_PORT + 1
 DEBUG = CONFIG.DEBUG or False
 LOG_LEVEL = CONFIG.LOG_LEVEL or 'INFO'
 
@@ -201,12 +202,15 @@ def is_running(s, unlink=True):
 
 def parse_service(s):
     all_services = [
-        'gunicorn', 'celery_ansible', 'celery_default', 'beat', 'flower'
+        'gunicorn', 'celery_ansible', 'celery_default',
+        'beat', 'flower', 'daphne',
     ]
     if s == 'all':
         return all_services
     elif s == "web":
-        return ['gunicorn', 'flower']
+        return ['gunicorn', 'flower', 'daphne']
+    elif s == 'ws':
+        return ['daphne']
     elif s == "task":
         return ["celery_ansible", "celery_default", "beat"]
     elif s == 'gunicorn':
@@ -225,10 +229,8 @@ def parse_service(s):
 def get_start_gunicorn_kwargs():
     print("\n- Start Gunicorn WSGI HTTP Server")
     prepare()
-    service = 'gunicorn'
     bind = '{}:{}'.format(HTTP_HOST, HTTP_PORT)
     log_format = '%(h)s %(t)s "%(r)s" %(s)s %(b)s '
-    pid_file = get_pid_file_path(service)
 
     cmd = [
         'gunicorn', 'jumpserver.wsgi',
@@ -238,7 +240,6 @@ def get_start_gunicorn_kwargs():
         '-w', str(WORKERS),
         '--max-requests', '4096',
         '--access-logformat', log_format,
-        '-p', pid_file,
         '--access-logfile', '-'
     ]
 
@@ -247,6 +248,16 @@ def get_start_gunicorn_kwargs():
     return {'cmd': cmd, 'cwd': APPS_DIR}
 
 
+def get_start_daphne_kwargs():
+    print("\n- Start Daphne ASGI WS Server")
+    cmd = [
+        'daphne', 'jumpserver.asgi:application',
+        '-b', HTTP_HOST,
+        '-p', str(WS_PORT),
+    ]
+    return {'cmd': cmd, 'cwd': APPS_DIR}
+
+
 def get_start_celery_ansible_kwargs():
     print("\n- Start Celery as Distributed Task Queue: Ansible")
     return get_start_worker_kwargs('ansible', 4)
@@ -362,6 +373,7 @@ def start_service(s):
         "celery_default": get_start_celery_default_kwargs,
         "beat": get_start_beat_kwargs,
         "flower": get_start_flower_kwargs,
+        "daphne": get_start_daphne_kwargs,
     }
 
     kwargs = services_kwargs.get(s)()

requirements/requirements.txt

@@ -86,3 +86,6 @@ httpsig==1.3.0
 treelib==1.5.3
 django-proxy==1.2.1
 flower==0.9.3
+channels-redis==2.4.0
+channels==2.3.0
+daphne==2.3.0