Merge pull request #201 from jumpserver/dev

Dev

coco/app.py

@@ -14,14 +14,16 @@ from .conf import config
 from .sshd import SSHServer
 from .httpd import HttpServer
 from .tasks import TaskHandler
-from .utils import get_logger, ugettext as _, ignore_error
+from .utils import (
+    get_logger, ugettext as _, ignore_error,
+)
 from .service import app_service
 from .recorder import get_replay_recorder
 from .session import Session
 from .models import Connection
 
 
-__version__ = '1.4.8'
+__version__ = '1.4.9'
 
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 logger = get_logger(__file__)
@@ -89,20 +91,8 @@ class Coco:
     # @ignore_error
     def heartbeat(self):
         sessions = list(Session.sessions.keys())
-        # p = psutil.Process(os.getpid())
-        # cpu_used = p.cpu_percent(interval=1.0)
-        # memory_used = int(p.memory_info().rss / 1024 / 1024)
-        # connections = len(p.connections())
-        # threads = p.num_threads()
-        # session_online = len(sessions)
         data = {
-            # "cpu_used": cpu_used,
-            # "memory_used": memory_used,
-            # "connections": connections,
-            # "threads": threads,
-            # "boot_time": p.create_time(),
-            # "session_online": session_online,
-            "sessions": sessions,
+            'sessions': sessions,
         }
         tasks = app_service.terminal_heartbeat(data)
 
@@ -201,6 +191,7 @@ class Coco:
                             check_session_idle_too_long(s)
                 except Exception as e:
                     logger.error("Unexpected error occur: {}".format(e))
+                    logger.error(e, exc_info=True)
                 time.sleep(interval)
         thread = threading.Thread(target=func)
         thread.start()

coco/conf.py

@@ -341,7 +341,7 @@ defaults = {
     'TELNET_REGEX': '',
     'PASSWORD_AUTH': True,
     'PUBLIC_KEY_AUTH': True,
-    'SSH_TIMEOUT': 10,
+    'SSH_TIMEOUT': 15,
     'ALLOW_SSH_USER': [],
     'BLOCK_SSH_USER': [],
     'HEARTBEAT_INTERVAL': 20,

coco/connection.py

@@ -25,13 +25,13 @@ AUTO_LOGIN = 'auto'
 
 class SSHConnection:
     @staticmethod
-    def get_system_user_auth(system_user):
+    def get_system_user_auth(system_user, asset):
         """
         获取系统用户的认证信息，密码或秘钥
         :return: system user have full info
         """
         password, private_key = \
-            app_service.get_system_user_auth_info(system_user)
+            app_service.get_system_user_auth_info(system_user, asset)
         system_user.password = password
         system_user.private_key = private_key
 
@@ -42,7 +42,7 @@ class SSHConnection:
         error = ''
 
         if not system_user.password and not system_user.private_key:
-            self.get_system_user_auth(system_user)
+            self.get_system_user_auth(system_user, asset)
 
         if asset.domain:
             sock = self.get_proxy_sock_v2(asset)
@@ -68,7 +68,7 @@ class SSHConnection:
                     look_for_keys=False, sock=sock, allow_agent=False,
                 )
             transport = ssh.get_transport()
-            transport.set_keepalive(300)
+            transport.set_keepalive(20)
         except Exception as e:
             password_short = "None"
             key_fingerprint = "None"
@@ -132,7 +132,9 @@ class SSHConnection:
             except:
                 continue
             try:
-                sock = ssh.get_transport().open_channel(
+                transport = ssh.get_transport()
+                transport.set_keep_alive(20)
+                sock = transport.open_channel(
                     'direct-tcpip', (asset.ip, asset.port), ('127.0.0.1', 0)
                 )
                 break

coco/httpd/elfinder/connector.py

@@ -110,9 +110,7 @@ class ElFinderConnector:
             func()
         except Exception as e:
             self.response['error'] = '%s' % e
-            logger.error("Error occur ------------------------------")
-            logger.exception(e)
-            logger.error("Error end ------------------------------")
+            logger.error(e, exc_info=True)
 
     def get_request_data(self):
         data_source = {}

coco/httpd/ws.py

@@ -87,6 +87,7 @@ class ProxyNamespace(BaseNamespace):
                 forwarder.proxy()
             except Exception as e:
                 logger.error("Unexpected error occur: {}".format(e))
+                logger.error(e, exc_info=True)
             self.logout(client_id, connection)
         self.socketio.start_background_task(proxy)
 
@@ -175,7 +176,7 @@ class ProxyNamespace(BaseNamespace):
             try:
                 self.on_logout(client_id)
             except Exception as e:
-                logger.warn(e)
+                logger.error(e, exc_info=True)
         Connection.remove_connection(connection.id)
 
     def logout(self, client_id, connection):

coco/interactive.py

@@ -135,7 +135,6 @@ class InteractiveServer:
             self.display_banner()
         elif opt in ['r', 'R']:
             self.refresh_assets_nodes()
-            self.display_banner()
         elif opt in ['h', 'H']:
             self.display_banner()
         else:
@@ -165,8 +164,9 @@ class InteractiveServer:
             self.display_assets_paging(assets)
 
     def refresh_assets_nodes(self):
-        self.get_user_assets_and_update_async()
-        self.get_user_nodes_async()
+        self.get_user_assets_and_update(cache_policy='2')
+        self.get_user_nodes(cache_policy='2')
+        self.client.send_unicode(_("Refresh done"))
 
     def wait_until_assets_load(self):
         while self.assets is None and \
@@ -319,9 +319,7 @@ class InteractiveServer:
     #
 
     def load_user_assets_from_cache(self):
-        assets = self.__class__._user_assets_cached.get(
-            self.client.user.id
-        )
+        assets = self.__class__._user_assets_cached.get(self.client.user.id)
         self.assets = assets
         if assets:
             self.total_asset_count = len(assets)
@@ -330,8 +328,8 @@ class InteractiveServer:
         thread = threading.Thread(target=self.get_user_assets_and_update)
         thread.start()
 
-    def get_user_assets_and_update(self):
-        assets = app_service.get_user_assets(self.client.user)
+    def get_user_assets_and_update(self, cache_policy='1'):
+        assets = app_service.get_user_assets(self.client.user, cache_policy=cache_policy)
         assets = self.filter_system_users(assets)
         self.__class__._user_assets_cached[self.client.user.id] = assets
         self.load_user_assets_from_cache()
@@ -344,8 +342,8 @@ class InteractiveServer:
         thread = threading.Thread(target=self.get_user_nodes)
         thread.start()
 
-    def get_user_nodes(self):
-        nodes = app_service.get_user_asset_groups(self.client.user)
+    def get_user_nodes(self, cache_policy='1'):
+        nodes = app_service.get_user_asset_groups(self.client.user, cache_policy=cache_policy)
         nodes = sorted(nodes, key=lambda node: node.key)
         self.nodes = self.filter_system_users_of_assets_under_nodes(nodes)
         self._construct_node_tree()

coco/models.py

@@ -326,7 +326,7 @@ class BaseServer(object):
             self.s_parse_input_output_filter(data)
             data = self.s_filter_cmd_filter(data)
         except Exception as e:
-            logger.exception(e)
+            logger.error(e, exc_info=True)
         return self.chan.send(data)
 
     def recv(self, size):

coco/proxy.py

@@ -34,7 +34,7 @@ class ProxyServer:
         :return: system user have full info
         """
         password, private_key = \
-            app_service.get_system_user_auth_info(self.system_user)
+            app_service.get_system_user_auth_info(self.system_user, self.asset)
         if self.system_user.login_mode == MANUAL_LOGIN \
                 or (not password and not private_key):
             prompt = "{}'s password: ".format(self.system_user.username)

coco/sftp.py

@@ -35,6 +35,7 @@ def convert_error(func):
             error = e
             response = SFTP_EOF
         except Exception as e:
+            logger.error(e, exc_info=True)
             error = e
             response = SFTP_FAILURE
         finally:
@@ -108,7 +109,7 @@ class SFTPServer(paramiko.SFTPServerInterface):
         self._sftp = {}
 
     def get_host_sftp(self, host, su):
-        asset = self.hosts.get(host)['asset']
+        asset = self.hosts.get(host, {}).get('asset')
         system_user = self.get_host_system_users(host, only_name=False).get(su)
 
         if not asset or not system_user:

coco/sshd.py

@@ -102,6 +102,7 @@ class SSHServer:
             logger.warning("Handle connection EOF Error: {}".format(e))
         except Exception as e:
             logger.error("Unexpect error occur on handle connection: {}".format(e))
+            logger.error(e, exc_info=True)
         finally:
             Connection.remove_connection(connection.id)
             sock.close()

coco/struct.py

@@ -29,12 +29,24 @@ class SizedList(list):
     def __init__(self, maxsize=0):
         self.maxsize = maxsize
         self.size = 0
+        self.end_with_ascii = False
         super(list, self).__init__()
 
+    def is_full(self):
+        if self.maxsize == 0:
+            return False
+        if not self.end_with_ascii:
+            return False
+        if self.size >= self.maxsize:
+            return True
+        else:
+            return False
+
     def append(self, b):
-        if self.maxsize == 0 or self.size < self.maxsize:
+        if not self.is_full():
             super(SizedList, self).append(b)
             self.size += len(b)
+            self.end_with_ascii = b[-1] <= 126
 
     def clean(self):
         self.size = 0

coco/utils.py

@@ -4,11 +4,13 @@
 
 from __future__ import unicode_literals
 
+import time
 import logging
 import re
 import os
 import gettext
 import gzip
+import psutil
 from io import StringIO
 from binascii import hexlify
 from werkzeug.local import Local, LocalProxy
@@ -461,7 +463,7 @@ def ignore_error(func):
             resp = func(*args, **kwargs)
             return resp
         except Exception as e:
-            logger.error("Error occur: {} {}".format(func.__name__, e))
+            logger.error(e, exc_info=True)
     return wrapper
 
 
@@ -472,4 +474,66 @@ def gzip_file(src_path, dst_path, unlink_ori=True):
         os.unlink(src_path)
 
 
+def get_cpu_info():
+    cpus = ['cpu%s' % i for i in range(psutil.cpu_count())]
+    percents = psutil.cpu_percent(interval=0.5, percpu=True)
+    return dict(zip(cpus, percents))
+
+
+def get_memory_info():
+    mem = psutil.virtual_memory()
+    return {
+        'total': mem.total,
+        'avail': mem.available,
+        'used': mem.used,
+        'percent': mem.percent,
+    }
+
+
+def get_disk_info():
+    partitions = psutil.disk_partitions()
+    info = {}
+    for partition in partitions:
+        usage = psutil.disk_usage(partition.mountpoint)
+        info[partition.device] = {
+            'mountpoint': partition.mountpoint,
+            'device': partition.device,
+            'total': usage.total,
+            'used': usage.used,
+            'free': usage.free,
+            'percent': usage.percent
+        }
+    return info
+
+
+def get_net_info():
+    counter = psutil.net_io_counters()
+    return {
+        'bytes_sent': counter.bytes_sent,
+        'bytes_recv': counter.bytes_recv,
+    }
+
+
+def get_coco_monitor_data():
+    p = psutil.Process(os.getpid())
+    cpu_used = p.cpu_percent(interval=0.5)
+    memory_used = p.memory_info().rss
+    connections = len(p.connections())
+    return {
+        'p_cpu': cpu_used,
+        'p_memory': memory_used,
+        'p_conns': connections
+    }
+
+
+def get_monitor_data():
+    return {
+        'timestamp': int(time.time()),
+        'cpu': get_cpu_info(),
+        'memory': get_memory_info(),
+        'disk': get_disk_info(),
+        'net': get_net_info(),
+    }
+
+
 ugettext = LocalProxy(partial(_find, 'LANGUAGE_CODE'))

cocod

@@ -25,8 +25,7 @@ sys.path.insert(0, BASE_DIR)
 dirs = ('logs', 'keys')
 for d in dirs:
     d2 = os.path.join('data', d)
-    if not os.path.isdir(d2):
-        os.makedirs(d2)
+    os.makedirs(d2, exist_ok=True)
 
 try:
     from coco import Coco

locale/en/LC_MESSAGES/coco.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2019-02-21 20:00+0800\n"
+"POT-Creation-Date: 2019-03-06 14:51+0800\n"
 "PO-Revision-Date: 2018-08-10 10:42+0800\n"
 "Last-Translator: BaiJiangjie <bugatti_it@163.com>\n"
 "Language-Team: Language locale/en/LC\n"
@@ -16,7 +16,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: coco/app.py:180
+#: coco/app.py:182
 msgid "Connect idle more than {} minutes, disconnect"
 msgstr ""
 
@@ -83,10 +83,14 @@ msgstr ""
 msgid "{T}0) Enter {green}q{end} exit.{R}"
 msgstr ""
 
-#: coco/interactive.py:159
+#: coco/interactive.py:158
 msgid "Terminal does not support login rdp, please use web terminal to access"
 msgstr ""
 
+#: coco/interactive.py:169
+msgid "Refresh done"
+msgstr ""
+
 #: coco/interactive.py:211
 msgid "No Assets"
 msgstr ""
@@ -131,27 +135,27 @@ msgstr ""
 msgid "BACK: b/q"
 msgstr ""
 
-#: coco/interactive.py:373
+#: coco/interactive.py:371
 msgid "No Nodes"
 msgstr ""
 
-#: coco/interactive.py:377
+#: coco/interactive.py:375
 msgid "Node: [ ID.Name(Asset amount) ]"
 msgstr ""
 
-#: coco/interactive.py:379
+#: coco/interactive.py:377
 msgid "Tips: Enter g+NodeID to display the host under the node, such as g1"
 msgstr ""
 
-#: coco/interactive.py:387
+#: coco/interactive.py:385
 msgid "There is no matched node, please re-enter"
 msgstr ""
 
-#: coco/interactive.py:417
+#: coco/interactive.py:415
 msgid "Select a login:: "
 msgstr ""
 
-#: coco/interactive.py:440
+#: coco/interactive.py:438
 msgid "No system user"
 msgstr ""
 

locale/zh_CN/LC_MESSAGES/coco.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2019-02-21 20:00+0800\n"
+"POT-Creation-Date: 2019-03-06 14:51+0800\n"
 "PO-Revision-Date: 2018-08-10 10:42+0800\n"
 "Last-Translator: BaiJiangjie <bugatti_it@163.com>\n"
 "Language-Team: Language locale/zh\n"
@@ -16,7 +16,7 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: coco/app.py:180
+#: coco/app.py:182
 msgid "Connect idle more than {} minutes, disconnect"
 msgstr "空闲时间超过 {} 分钟，断开连接"
 
@@ -25,6 +25,7 @@ msgid "Welcome to use Jumpserver open source fortress system"
 msgstr "欢迎使用Jumpserver开源跳板机系统"
 
 #: coco/interactive.py:87
+#, python-brace-format
 msgid ""
 "\n"
 "{T}{T}{title} {user}, {header_title} {end}{R}{R}"
@@ -89,10 +90,14 @@ msgstr "{T}0) 输入 {green}r{end} 刷新最新的机器和节点信息.{R}"
 msgid "{T}0) Enter {green}q{end} exit.{R}"
 msgstr "{T}0) 输入 {green}q{end} 退出.{R}"
 
-#: coco/interactive.py:159
+#: coco/interactive.py:158
 msgid "Terminal does not support login rdp, please use web terminal to access"
 msgstr "终端不支持登录windows, 请使用web terminal访问"
 
+#: coco/interactive.py:169
+msgid "Refresh done"
+msgstr "刷新完成"
+
 #: coco/interactive.py:211
 msgid "No Assets"
 msgstr "没有资产"
@@ -137,27 +142,27 @@ msgstr "下一页: Enter|N/n"
 msgid "BACK: b/q"
 msgstr "返回: B/b"
 
-#: coco/interactive.py:373
+#: coco/interactive.py:371
 msgid "No Nodes"
 msgstr "没有节点"
 
-#: coco/interactive.py:377
+#: coco/interactive.py:375
 msgid "Node: [ ID.Name(Asset amount) ]"
 msgstr "节点: [ ID.名称(资产数量) ]"
 
-#: coco/interactive.py:379
+#: coco/interactive.py:377
 msgid "Tips: Enter g+NodeID to display the host under the node, such as g1"
 msgstr "提示: 输入 g+节点ID 显示节点下主机. 如: g1"
 
-#: coco/interactive.py:387
+#: coco/interactive.py:385
 msgid "There is no matched node, please re-enter"
 msgstr "没有匹配分组，请重新输入"
 
-#: coco/interactive.py:417
+#: coco/interactive.py:415
 msgid "Select a login:: "
 msgstr "选择一个登录:"
 
-#: coco/interactive.py:440
+#: coco/interactive.py:438
 msgid "No system user"
 msgstr "没有系统用户"
 

requirements/requirements.txt

@@ -4,7 +4,7 @@ boto3==1.6.5
 botocore==1.9.5
 cachetools==2.0.1
 certifi==2018.1.18
-cffi==1.11.2
+cffi==1.11.5
 chardet==3.0.4
 click==6.7
 crcmod==1.7
@@ -19,13 +19,13 @@ itsdangerous==0.24
 Jinja2==2.10
 jmespath==0.9.3
 jms-storage==0.0.22
-jumpserver-python-sdk==0.0.56
+jumpserver-python-sdk==0.0.57
 MarkupSafe==1.0
 oss2==2.4.0
 paramiko==2.4.1
 psutil==5.4.1
 pyasn1==0.4.2
-pycparser==2.18
+pycparser==2.19
 PyNaCl==1.2.1
 pyte==0.8.0
 python-dateutil==2.6.1