Unverified Commit da40d0f8 authored by 老广's avatar 老广 Committed by GitHub

Merge pull request #177 from jumpserver/dev

Dev
parents a0bbd8dc a24bb091
......@@ -5,7 +5,7 @@ env/
.access_key
*.log
logs/*
conf.py
host_rsa_key
sessions/*
coco.pid
config.yml
......@@ -12,10 +12,9 @@ ENV LANG=zh_CN.UTF-8
ENV LC_ALL=zh_CN.UTF-8
COPY . /opt/coco
VOLUME /opt/coco/logs
VOLUME /opt/coco/keys
VOLUME /opt/coco/data
RUN cp conf_docker.py conf.py
RUN echo > config.yml
EXPOSE 2222
CMD python run_server.py
......@@ -8,8 +8,12 @@ import time
import threading
import json
import signal
import copy
from collections import defaultdict
from .config import config
import psutil
from .conf import config
from .sshd import SSHServer
from .httpd import HttpServer
from .tasks import TaskHandler
......@@ -59,10 +63,13 @@ class Coco:
@ignore_error
def load_extra_conf_from_server():
configs = app_service.load_config_from_server()
config.update(configs)
tmp = copy.deepcopy(configs)
tmp['HOST_KEY'] = tmp['HOST_KEY'][32:50] + '...'
logger.debug("Loading config from server: {}".format(
json.dumps(configs)
json.dumps(tmp)
))
config.update(configs)
def keep_load_extra_conf(self):
def func():
......@@ -79,10 +86,26 @@ class Coco:
self.monitor_sessions()
self.monitor_sessions_replay()
@ignore_error
# @ignore_error
def heartbeat(self):
_sessions = [s.to_json() for s in Session.sessions.values() if s]
tasks = app_service.terminal_heartbeat(_sessions)
sessions = list(Session.sessions.keys())
# p = psutil.Process(os.getpid())
# cpu_used = p.cpu_percent(interval=1.0)
# memory_used = int(p.memory_info().rss / 1024 / 1024)
# connections = len(p.connections())
# threads = p.num_threads()
# session_online = len(sessions)
data = {
# "cpu_used": cpu_used,
# "memory_used": memory_used,
# "connections": connections,
# "threads": threads,
# "boot_time": p.create_time(),
# "session_online": session_online,
"sessions": sessions,
}
tasks = app_service.terminal_heartbeat(data)
if tasks:
self.handle_task(tasks)
if tasks is False:
......@@ -103,7 +126,7 @@ class Coco:
while not self.stop_evt.is_set():
try:
self.heartbeat()
except Exception as e:
except IndexError as e:
logger.error("Unexpected error occur: {}".format(e))
time.sleep(config["HEARTBEAT_INTERVAL"])
thread = threading.Thread(target=func)
......@@ -112,23 +135,38 @@ class Coco:
def monitor_sessions_replay(self):
interval = 10
log_dir = os.path.join(config['LOG_DIR'])
max_try = 5
upload_failed = defaultdict(int)
def func():
while not self.stop_evt.is_set():
active_sessions = [sid for sid in Session.sessions]
for filename in os.listdir(log_dir):
suffix = filename.split('.')[-1]
if suffix != 'gz':
continue
session_id = filename.split('.')[0]
full_path = os.path.join(log_dir, filename)
if len(session_id) != 36:
continue
full_path = os.path.join(log_dir, filename)
stat = os.stat(full_path)
# 是否是一天前的,因为现在多个coco共享了日志目录,
# 不能单纯判断session是否关闭
if stat.st_mtime > time.time() - 24*60*60:
continue
# 失败次数过多
if session_id in upload_failed \
and upload_failed[session_id] >= max_try:
continue
recorder = get_replay_recorder()
if session_id not in active_sessions:
recorder.file_path = full_path
ok = recorder.upload_replay(session_id, 1)
if not ok and os.path.getsize(full_path) == 0:
if ok:
upload_failed.pop(session_id, None)
elif not ok and os.path.getsize(full_path) == 0:
os.unlink(full_path)
else:
upload_failed[session_id] += 1
time.sleep(1)
time.sleep(interval)
thread = threading.Thread(target=func)
......
......@@ -14,10 +14,12 @@
"""
import os
import sys
import types
import errno
import json
import socket
import yaml
from werkzeug.utils import import_string
......@@ -206,6 +208,21 @@ class Config(dict):
raise
return self.from_mapping(obj)
def from_yaml(self, filename, silent=False):
if self.root_path:
filename = os.path.join(self.root_path, filename)
try:
with open(filename) as f:
obj = yaml.load(f)
except IOError as e:
if silent and e.errno in (errno.ENOENT, errno.EISDIR):
return False
e.strerror = 'Unable to load configuration file (%s)' % e.strerror
raise
if obj:
return self.from_mapping(obj)
return True
def from_mapping(self, *mapping, **kwargs):
"""Updates the config like :meth:`update` ignoring items with non-upper
keys.
......@@ -279,21 +296,31 @@ class Config(dict):
return value
value = os.environ.get(item, None)
if value is not None:
if value.isdigit():
value = int(value)
return value
return self.defaults.get(item)
def __getattr__(self, item):
return self.__getitem__(item)
def __setattr__(self, key, value):
return self.__setitem__(key, value)
def __repr__(self):
return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self))
access_key_path = os.path.abspath(os.path.join(root_path, 'keys', '.access_key'))
default_config = {
access_key_path = os.path.abspath(
os.path.join(root_path, 'data', 'keys', '.access_key')
)
host_key_path = os.path.abspath(
os.path.join(root_path, 'data', 'keys', 'host_rsa_key')
)
defaults = {
'NAME': socket.gethostname(),
'CORE_HOST': 'http://127.0.0.1:8080',
'BOOTSTRAP_TOKEN': os.environ.get("BOOTSTRAP_TOKEN") or 'PleaseChangeMe',
'BOOTSTRAP_TOKEN': '',
'ROOT_PATH': root_path,
'DEBUG': True,
'BIND_HOST': '0.0.0.0',
......@@ -301,17 +328,17 @@ default_config = {
'HTTPD_PORT': 5000,
'COCO_ACCESS_KEY': '',
'ACCESS_KEY_FILE': access_key_path,
'HOST_KEY_FILE': host_key_path,
'SECRET_KEY': 'SDK29K03%MM0ksf&#2',
'LOG_LEVEL': 'DEBUG',
'LOG_DIR': os.path.join(root_path, 'logs'),
'SESSION_DIR': os.path.join(root_path, 'sessions'),
'LOG_LEVEL': 'INFO',
'LOG_DIR': os.path.join(root_path, 'data', 'logs'),
'ASSET_LIST_SORT_BY': 'hostname', # hostname, ip
'PASSWORD_AUTH': True,
'PUBLIC_KEY_AUTH': True,
'SSH_TIMEOUT': 10,
'ALLOW_SSH_USER': [],
'BLOCK_SSH_USER': [],
'HEARTBEAT_INTERVAL': 5,
'HEARTBEAT_INTERVAL': 20,
'MAX_CONNECTIONS': 500, # Not use now
'ADMINS': '',
'COMMAND_STORAGE': {'TYPE': 'server'}, # server
......@@ -319,16 +346,57 @@ default_config = {
'LANGUAGE_CODE': 'zh',
'SECURITY_MAX_IDLE_TIME': 60,
'ASSET_LIST_PAGE_SIZE': 'auto',
'SFTP_ROOT': '/tmp',
'SFTP_SHOW_HIDDEN_FILE': False
}
config = Config(root_path, default_config)
config.from_pyfile('conf.py')
try:
from conf import config as _conf
config.from_object(_conf)
except ImportError:
def load_from_object(config):
try:
from conf import config as c
config.from_object(c)
return True
except ImportError:
pass
return False
def load_from_yml(config):
for i in ['config.yml', 'config.yaml']:
if not os.path.isfile(os.path.join(config.root_path, i)):
continue
loaded = config.from_yaml(i)
if loaded:
return True
return False
def load_user_config():
sys.path.insert(0, root_path)
config = Config(root_path, defaults)
loaded = load_from_object(config)
if not loaded:
loaded = load_from_yml(config)
if not loaded:
msg = """
Error: No config file found.
You can run `cp config_example.yml config.yml`, and edit it.
"""
raise ImportError(msg)
return config
config = load_user_config()
old_host_key_path = os.path.join(root_path, 'keys', 'host_rsa_key')
old_access_key_path = os.path.join(root_path, 'keys', '.access_key')
if os.path.isfile(old_host_key_path) and not os.path.isfile(config.HOST_KEY_FILE):
config.HOST_KEY_FILE = old_host_key_path
if os.path.isfile(old_access_key_path) and not os.path.isfile(config.ACCESS_KEY_FILE):
config.ACCESS_KEY_FILE = old_access_key_path
if not config['NAME']:
config['NAME'] = default_config['NAME']
......@@ -14,7 +14,7 @@ except ImportError:
import paramiko
from .service import app_service
from .config import config
from .conf import config
from .utils import get_logger, get_private_key_fingerprint
logger = get_logger(__file__)
......
......@@ -6,7 +6,7 @@ from flask_socketio import SocketIO
from flask import Flask
from coco.utils import get_logger
from coco.config import config
from coco.conf import config
from coco.httpd.ws import ProxyNamespace, ElfinderNamespace
logger = get_logger(__file__)
......
......@@ -10,7 +10,7 @@ from ..models import Connection, WSProxy
from ..proxy import ProxyServer
from ..utils import get_logger
from ..service import app_service
from ..config import config
from ..conf import config
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
logger = get_logger(__file__)
......
......@@ -11,7 +11,7 @@ import time
from treelib import Tree
from . import char
from .config import config
from .conf import config
from .utils import wrap_with_line_feed as wr, wrap_with_title as title, \
wrap_with_warning as warning, is_obj_attr_has, is_obj_attr_eq, \
sort_assets, ugettext as _, get_logger, net_input, format_with_zh, \
......
......@@ -7,7 +7,7 @@ import threading
from collections import Iterable
from .utils import get_logger
from .config import config
from .conf import config
from .service import app_service
logger = get_logger(__file__)
......
......@@ -3,15 +3,19 @@
#
import os
import socket
import logging
from logging.config import dictConfig
from .config import config as app_config
from .conf import config as app_config
def create_logger():
level = app_config['LOG_LEVEL']
log_dir = app_config['LOG_DIR']
log_path = os.path.join(log_dir, 'coco.log')
filename = 'coco-{}.log'.format(socket.gethostname())
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_path = os.path.join(log_dir, filename)
main_setting = {
'handlers': ['console', 'file'],
'level': level,
......
......@@ -9,7 +9,7 @@ from .session import Session
from .models import Server, TelnetServer
from .connection import SSHConnection, TelnetConnection
from .service import app_service
from .config import config
from .conf import config
from .utils import wrap_with_line_feed as wr, wrap_with_warning as warning, \
get_logger, net_input, ugettext as _, ignore_error
......@@ -72,6 +72,12 @@ class ProxyServer:
self.server.close()
return
session = Session.new_session(self.client, self.server)
if not session:
msg = _("Connect with api server failed")
logger.error(msg)
self.client.send_unicode(msg)
self.server.close()
try:
session.bridge()
finally:
......
......@@ -2,7 +2,6 @@
# -*- coding: utf-8 -*-
#
import abc
import threading
import time
import os
......@@ -12,8 +11,8 @@ from copy import deepcopy
import jms_storage
from .config import config
from .utils import get_logger, Singleton
from .conf import config
from .utils import get_logger
from .struct import MemoryQueue
from .service import app_service
......
......@@ -2,11 +2,10 @@
#
from jms.service import AppService
from .config import config
from .conf import config
inited = False
app_service = AppService(config)
if not inited:
......
......@@ -48,7 +48,14 @@ class Session:
session.set_command_recorder(command_recorder)
session.set_replay_recorder(replay_recorder)
cls.sessions[session.id] = session
app_service.create_session(session.to_json())
_session = None
for i in range(5):
_session = app_service.create_session(session.to_json())
if _session:
break
time.sleep(0.2)
if _session is None:
return None
return session
@classmethod
......
......@@ -8,7 +8,7 @@ from paramiko.sftp import SFTP_PERMISSION_DENIED, SFTP_NO_SUCH_FILE, \
SFTP_FAILURE, SFTP_EOF, SFTP_CONNECTION_LOST
from coco.utils import get_logger
from .config import config
from .conf import config
from .service import app_service
from .connection import SSHConnection
......@@ -45,7 +45,8 @@ def convert_error(func):
class SFTPServer(paramiko.SFTPServerInterface):
root = '/tmp' # Home or /tmp or other path, must exist on all server
# Home or /tmp or other path, must exist on all server
root = config.SFTP_ROOT
def __init__(self, server, **kwargs):
"""
......@@ -234,6 +235,9 @@ class SFTPServer(paramiko.SFTPServerInterface):
else:
client, rpath = self.get_sftp_client_rpath(request)
output = client.listdir_attr(rpath)
show_hidden_file = config['SFTP_SHOW_HIDDEN_FILE']
if not show_hidden_file:
output = [attr for attr in output if not attr.filename.startswith('.')]
return output
@convert_error
......@@ -291,6 +295,7 @@ class SFTPServer(paramiko.SFTPServerInterface):
try:
client, rpath = self.get_sftp_client_rpath(path)
f = client.open(rpath, mode, bufsize=4096)
f.prefetch()
obj = paramiko.SFTPHandle(flags)
obj.filename = rpath
obj.readfile = f
......
......@@ -14,7 +14,7 @@ from coco.interface import SSHInterface
from coco.interactive import InteractiveServer
from coco.models import Connection
from coco.sftp import SFTPServer
from coco.config import config
from coco.conf import config
logger = get_logger(__file__)
BACKLOG = 5
......@@ -29,8 +29,12 @@ class SSHServer:
@property
def host_key(self):
host_key_path = os.path.join(config['ROOT_PATH'], 'keys', 'host_rsa_key')
host_key_path = config['HOST_KEY_FILE']
if not os.path.isfile(host_key_path):
if config.HOST_KEY:
with open(host_key_path, 'w') as f:
f.write(config.HOST_KEY)
else:
self.gen_host_key(host_key_path)
return paramiko.RSAKey(filename=host_key_path)
......
......@@ -17,7 +17,7 @@ import paramiko
import pyte
from . import char
from .config import config
from .conf import config
BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
......
......@@ -19,15 +19,15 @@ import signal
dirs = ('logs', 'keys')
for d in dirs:
if not os.path.isdir(d):
os.makedirs(d)
d2 = os.path.join('data', d)
if not os.path.isdir(d2):
os.makedirs(d2)
from coco import Coco
try:
from conf import config
from coco.conf import config
except ImportError:
print("Please prepare config file `cp conf_example.py conf.py`")
sys.exit(1)
......
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
import os
BASE_DIR = os.path.dirname(__file__)
class Config:
"""
Coco config file
"""
# 默认的名字
NAME = os.environ.get("NAME") or None
# Jumpserver项目的url, api请求注册会使用
CORE_HOST = os.environ.get("CORE_HOST") or 'http://core:8080'
# 启动时绑定的ip, 默认 0.0.0.0
BIND_HOST = '0.0.0.0'
# 监听的SSH端口号, 默认2222
SSHD_PORT = 2222
# 监听的HTTP/WS端口号,默认5000
HTTPD_PORT = 5000
# 项目使用的ACCESS KEY, 默认会注册,并保存到 ACCESS_KEY_STORE中,
# 如果有需求, 可以写到配置文件中, 格式 access_key_id:access_key_secret
ACCESS_KEY = os.environ.get("ACCESS_KEY") or None
# ACCESS KEY 保存的地址, 默认注册后会保存到该文件中
# ACCESS_KEY_STORE = os.path.join(BASE_DIR, 'keys', '.access_key')
# 加密密钥
SECRET_KEY = os.environ.get("SECRET_KEY") or 'SKdfm239LSKdfj())_23jK*^2'
# 设置日志级别 ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL', 'CRITICAL']
LOG_LEVEL = os.environ.get("LOG_LEVEL") or 'DEBUG'
# 日志存放的目录
LOG_DIR = os.environ.get("LOG_DIR") or os.path.join(BASE_DIR, 'logs')
# Session录像存放目录
SESSION_DIR = os.environ.get("SESSION_DIR") or os.path.join(BASE_DIR, 'sessions')
# 资产显示排序方式, ['ip', 'hostname']
ASSET_LIST_SORT_BY = os.environ.get("SESSION_DIR") or 'ip'
# 登录是否支持密码认证
SSH_PASSWORD_AUTH = bool(os.environ.get("SSH_PASSWORD_AUTH")) if os.environ.get("SSH_PASSWORD_AUTH") else True
# 登录是否支持秘钥认证
SSH_PUBLIC_KEY_AUTH = bool(os.environ.get("SSH_PUBLIC_KEY_AUTH")) if os.environ.get("SSH_PUBLIC_KEY_AUTH") else True
# 和Jumpserver 保持心跳时间间隔
HEARTBEAT_INTERVAL = int(os.environ.get("HEARTBEAT_INTERVAL")) if os.environ.get("HEARTBEAT_INTERVAL") else 5
# Admin的名字,出问题会提示给用户
ADMINS = os.environ.get("ADMINS") or ''
COMMAND_STORAGE = {
"TYPE": "server"
}
class ConfigDocker(Config):
pass
config = ConfigDocker()
\ No newline at end of file
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
import os
BASE_DIR = os.path.dirname(__file__)
class Config:
"""
Coco config file, coco also load config from server update setting below
"""
# 项目名称, 会用来向Jumpserver注册, 识别而已, 不能重复
# NAME = "localhost"
# Jumpserver项目的url, api请求注册会使用
# CORE_HOST = os.environ.get("CORE_HOST") or 'http://127.0.0.1:8080'
# Bootstrap Token, 预共享秘钥, 用来注册coco使用的service account和terminal
# 请和jumpserver 配置文件中保持一致,注册完成后可以删除
# BOOTSTRAP_TOKEN = "PleaseChangeMe"
# 启动时绑定的ip, 默认 0.0.0.0
# BIND_HOST = '0.0.0.0'
# 监听的SSH端口号, 默认2222
# SSHD_PORT = 2222
# 监听的HTTP/WS端口号,默认5000
# HTTPD_PORT = 5000
# 项目使用的ACCESS KEY, 默认会注册,并保存到 ACCESS_KEY_STORE中,
# 如果有需求, 可以写到配置文件中, 格式 access_key_id:access_key_secret
# ACCESS_KEY = None
# ACCESS KEY 保存的地址, 默认注册后会保存到该文件中
# ACCESS_KEY_STORE = os.path.join(BASE_DIR, 'keys', '.access_key')
# 加密密钥
# SECRET_KEY = None
# 设置日志级别 ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL', 'CRITICAL']
# LOG_LEVEL = 'INFO'
# 日志存放的目录
# LOG_DIR = os.path.join(BASE_DIR, 'logs')
# Session录像存放目录
# SESSION_DIR = os.path.join(BASE_DIR, 'sessions')
# 资产显示排序方式, ['ip', 'hostname']
# ASSET_LIST_SORT_BY = 'ip'
# 登录是否支持密码认证
# PASSWORD_AUTH = True
# 登录是否支持秘钥认证
# PUBLIC_KEY_AUTH = True
# SSH白名单
# ALLOW_SSH_USER = 'all' # ['test', 'test2']
# SSH黑名单, 如果用户同时在白名单和黑名单,黑名单优先生效
# BLOCK_SSH_USER = []
# 和Jumpserver 保持心跳时间间隔
# HEARTBEAT_INTERVAL = 5
# Admin的名字,出问题会提示给用户
# ADMINS = ''
COMMAND_STORAGE = {
"TYPE": "server"
}
REPLAY_STORAGE = {
"TYPE": "server"
}
# SSH连接超时时间 (default 15 seconds)
# SSH_TIMEOUT = 15
# 语言 = en
LANGUAGE_CODE = 'zh'
config = Config()
# 项目名称, 会用来向Jumpserver注册, 识别而已, 不能重复
# NAME: {{ Hostname }}
# Jumpserver项目的url, api请求注册会使用
CORE_HOST: http://127.0.0.1:8080
# Bootstrap Token, 预共享秘钥, 用来注册coco使用的service account和terminal
# 请和jumpserver 配置文件中保持一致,注册完成后可以删除
BOOTSTRAP_TOKEN: <PleasgeChangeSameWithJumpserver>
# 启动时绑定的ip, 默认 0.0.0.0
# BIND_HOST: 0.0.0.0
# 监听的SSH端口号, 默认2222
# SSHD_PORT: 2222
# 监听的HTTP/WS端口号,默认5000
# HTTPD_PORT: 5000
# 项目使用的ACCESS KEY, 默认会注册,并保存到 ACCESS_KEY_STORE中,
# 如果有需求, 可以写到配置文件中, 格式 access_key_id:access_key_secret
# ACCESS_KEY: null
# ACCESS KEY 保存的地址, 默认注册后会保存到该文件中
# ACCESS_KEY_FILE: data/keys/.access_key
# 加密密钥
# SECRET_KEY: null
# 设置日志级别 ['DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL', 'CRITICAL']
# LOG_LEVEL: INFO
# 日志存放的目录
# LOG_DIR: logs
# SSH白名单
# ALLOW_SSH_USER: 'all'
# SSH黑名单, 如果用户同时在白名单和黑名单,黑名单优先生效
# BLOCK_SSH_USER:
# -
# 和Jumpserver 保持心跳时间间隔
# HEARTBEAT_INTERVAL: 5
# Admin的名字,出问题会提示给用户
# ADMINS: ''
# SSH连接超时时间 (default 15 seconds)
# SSH_TIMEOUT: 15
# 语言 = en
# LANGUAGE_CODE: zh
......@@ -19,7 +19,7 @@ itsdangerous==0.24
Jinja2==2.10
jmespath==0.9.3
jms-storage==0.0.20
jumpserver-python-sdk==0.0.54
jumpserver-python-sdk==0.0.56
MarkupSafe==1.0
oss2==2.4.0
paramiko==2.4.1
......@@ -43,4 +43,4 @@ wcwidth==0.1.7
eventlet==0.24.1
Werkzeug==0.14.1
treelib==1.5.3
pyyaml==3.13
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment