Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
C
coco
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ops
coco
Commits
983a9d8c
Commit
983a9d8c
authored
Mar 12, 2018
by
ibuler
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'dev' of bitbucket.org:jumpserver/coco into dev
parents
a21aa4e7
8ff95546
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
57 additions
and
90 deletions
+57
-90
app.py
coco/app.py
+9
-5
recorder.py
coco/recorder.py
+32
-79
requirements.txt
requirements/requirements.txt
+16
-6
No files found.
coco/app.py
View file @
983a9d8c
...
@@ -16,7 +16,7 @@ from .sshd import SSHServer
...
@@ -16,7 +16,7 @@ from .sshd import SSHServer
from
.httpd
import
HttpServer
from
.httpd
import
HttpServer
from
.logger
import
create_logger
from
.logger
import
create_logger
from
.tasks
import
TaskHandler
from
.tasks
import
TaskHandler
from
.recorder
import
get_command_recorder_class
,
get_replay_recorder_class
from
.recorder
import
get_command_recorder_class
,
ServerReplayRecorder
from
.utils
import
get_logger
from
.utils
import
get_logger
...
@@ -109,7 +109,7 @@ class Coco:
...
@@ -109,7 +109,7 @@ class Coco:
self
.
config
.
update
(
configs
)
self
.
config
.
update
(
configs
)
def
get_recorder_class
(
self
):
def
get_recorder_class
(
self
):
self
.
replay_recorder_class
=
get_replay_recorder_class
(
self
.
config
)
self
.
replay_recorder_class
=
ServerReplayRecorder
self
.
command_recorder_class
=
get_command_recorder_class
(
self
.
config
)
self
.
command_recorder_class
=
get_command_recorder_class
(
self
.
config
)
def
new_command_recorder
(
self
):
def
new_command_recorder
(
self
):
...
@@ -233,6 +233,9 @@ class Coco:
...
@@ -233,6 +233,9 @@ class Coco:
def
remove_session
(
self
,
session
):
def
remove_session
(
self
,
session
):
with
self
.
lock
:
with
self
.
lock
:
logger
.
info
(
"Remove session: {}"
.
format
(
session
))
try
:
self
.
sessions
.
remove
(
session
)
logger
.
info
(
"Remove session: {}"
.
format
(
session
))
self
.
service
.
finish_session
(
session
.
to_json
())
self
.
sessions
.
remove
(
session
)
self
.
service
.
finish_session
(
session
.
to_json
())
except
ValueError
:
logger
.
warning
(
"Remove session: {} fail, maybe already removed"
.
format
(
session
))
\ No newline at end of file
coco/recorder.py
View file @
983a9d8c
...
@@ -9,9 +9,8 @@ import os
...
@@ -9,9 +9,8 @@ import os
import
gzip
import
gzip
import
json
import
json
import
shutil
import
shutil
import
boto3
# AWS S3 sdk
from
jms_es_sdk
import
ESStor
e
import
jms_storag
e
from
.utils
import
get_logger
from
.utils
import
get_logger
from
.alignment
import
MemoryQueue
from
.alignment
import
MemoryQueue
...
@@ -125,14 +124,19 @@ class ServerReplayRecorder(ReplayRecorder):
...
@@ -125,14 +124,19 @@ class ServerReplayRecorder(ReplayRecorder):
with
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay'
),
'rb'
)
as
f_in
,
\
with
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay'
),
'rb'
)
as
f_in
,
\
gzip
.
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
'wb'
)
as
f_out
:
gzip
.
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
'wb'
)
as
f_out
:
shutil
.
copyfileobj
(
f_in
,
f_out
)
shutil
.
copyfileobj
(
f_in
,
f_out
)
if
self
.
push_to_server
(
session_id
):
if
self
.
upload_replay
(
session_id
):
logger
.
info
(
"Succeed to push {}'s {}"
.
format
(
session_id
,
"record"
))
logger
.
info
(
"Succeed to push {}'s {}"
.
format
(
session_id
,
"record"
))
else
:
else
:
logger
.
error
(
"Failed to push {}'s {}"
.
format
(
session_id
,
"record"
))
logger
.
error
(
"Failed to push {}'s {}"
.
format
(
session_id
,
"record"
))
self
.
push_to_server
(
session_id
)
self
.
upload_replay
(
session_id
)
def
push_to_server
(
self
,
session_id
):
def
upload_replay
(
self
,
session_id
):
if
self
.
upload_replay
(
3
,
session_id
):
configs
=
self
.
app
.
service
.
load_config_from_server
()
logger
.
debug
(
"upload_replay print config: {}"
.
format
(
configs
))
self
.
client
=
jms_storage
.
init
(
configs
[
"REPLAY_STORAGE"
])
if
not
self
.
client
:
self
.
client
=
jms_storage
.
jms
(
self
.
app
.
service
)
if
self
.
push_storage
(
3
,
session_id
):
if
self
.
finish_replay
(
3
,
session_id
):
if
self
.
finish_replay
(
3
,
session_id
):
return
True
return
True
else
:
else
:
...
@@ -140,21 +144,27 @@ class ServerReplayRecorder(ReplayRecorder):
...
@@ -140,21 +144,27 @@ class ServerReplayRecorder(ReplayRecorder):
else
:
else
:
return
False
return
False
def
push_local
(
self
,
session_id
):
def
push_to_storage
(
self
,
session_id
):
return
self
.
app
.
service
.
push_session_replay
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
return
self
.
client
.
upload_file
(
session_id
)
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
time
.
strftime
(
'
%
Y-
%
m-
%
d'
,
time
.
localtime
(
self
.
starttime
))
+
'/'
+
session_id
+
'.replay.gz'
)
def
upload_replay
(
self
,
times
,
session_id
):
def
push_storage
(
self
,
times
,
session_id
):
if
times
>
0
:
if
times
>
0
:
if
self
.
push_
local
(
session_id
):
if
self
.
push_
to_storage
(
session_id
):
logger
.
info
(
"success push session: {}'s replay log "
.
format
(
session_id
))
logger
.
info
(
"success push session: {}'s replay log
to storage
"
.
format
(
session_id
))
return
True
return
True
else
:
else
:
logger
.
error
(
"failed report session {}'s replay log, try {} times"
.
format
(
session_id
,
times
))
logger
.
error
(
return
self
.
upload_replay
(
times
-
1
,
session_id
)
"failed report session {}'s replay log to storage, try {} times"
.
format
(
session_id
,
times
))
return
self
.
push_storage
(
times
-
1
,
session_id
)
else
:
else
:
logger
.
error
(
"failed report session {}'s replay log"
.
format
(
session_id
))
logger
.
error
(
"failed report session {}'s replay log storage, try to push to local"
.
format
(
session_id
))
return
False
if
self
.
client
.
type
()
==
'jms'
:
return
False
else
:
self
.
client
=
jms_storage
.
jms
(
self
.
app
.
service
)
return
self
.
push_storage
(
3
,
session_id
)
def
finish_replay
(
self
,
times
,
session_id
):
def
finish_replay
(
self
,
times
,
session_id
):
if
times
>
0
:
if
times
>
0
:
...
@@ -232,7 +242,7 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
...
@@ -232,7 +242,7 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
self
.
stop_evt
=
threading
.
Event
()
self
.
stop_evt
=
threading
.
Event
()
self
.
push_to_es_async
()
self
.
push_to_es_async
()
self
.
__class__
.
no
+=
1
self
.
__class__
.
no
+=
1
self
.
store
=
ESStore
(
app
.
config
[
"COMMAND_STORAGE"
]
.
get
(
"HOSTS"
,
self
.
default_hosts
))
self
.
store
=
jms_storage
.
ESStore
(
app
.
config
[
"COMMAND_STORAGE"
]
.
get
(
"HOSTS"
,
self
.
default_hosts
))
if
not
self
.
store
.
ping
():
if
not
self
.
store
.
ping
():
raise
AssertionError
(
"ESCommand storage init error"
)
raise
AssertionError
(
"ESCommand storage init error"
)
...
@@ -273,58 +283,6 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
...
@@ -273,58 +283,6 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
print
(
"{} has been gc"
.
format
(
self
))
print
(
"{} has been gc"
.
format
(
self
))
class
S3ReplayRecorder
(
ServerReplayRecorder
):
def
__init__
(
self
,
app
):
super
()
.
__init__
(
app
)
self
.
bucket
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"BUCKET"
,
"jumpserver"
)
self
.
REGION
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"REGION"
,
None
)
self
.
ACCESS_KEY
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"ACCESS_KEY"
,
None
)
self
.
SECRET_KEY
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"SECRET_KEY"
,
None
)
if
self
.
ACCESS_KEY
and
self
.
REGION
and
self
.
SECRET_KEY
:
self
.
s3
=
boto3
.
client
(
's3'
,
region_name
=
self
.
REGION
,
aws_access_key_id
=
self
.
ACCESS_KEY
,
aws_secret_access_key
=
self
.
SECRET_KEY
)
else
:
self
.
s3
=
boto3
.
client
(
's3'
)
def
push_to_s3
(
self
,
session_id
):
logger
.
debug
(
"push to s3"
)
try
:
self
.
s3
.
upload_file
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
self
.
bucket
,
time
.
strftime
(
'
%
Y-
%
m-
%
d'
,
time
.
localtime
(
self
.
starttime
))
+
'/'
+
session_id
+
'.replay.gz'
)
return
True
except
:
return
False
def
upload_replay
(
self
,
times
,
session_id
):
if
times
>
0
:
if
self
.
push_to_s3
(
session_id
):
logger
.
info
(
"success push session: {}'s replay log to S3 "
.
format
(
session_id
))
return
True
else
:
logger
.
error
(
"failed report session {}'s replay log to S3, try {} times"
.
format
(
session_id
,
times
))
return
self
.
upload_replay
(
times
-
1
,
session_id
)
else
:
logger
.
error
(
"failed report session {}'s replay log S3, try to push to local"
.
format
(
session_id
))
return
self
.
upload_replay_to_local
(
3
,
session_id
)
def
upload_replay_to_local
(
self
,
times
,
session_id
):
if
times
>
0
:
if
self
.
push_local
(
session_id
):
logger
.
info
(
"success push session: {}'s replay log "
.
format
(
session_id
))
return
True
else
:
logger
.
error
(
"failed report session {}'s replay log, try {} times"
.
format
(
session_id
,
times
))
return
self
.
upload_replay_to_local
(
times
-
1
,
session_id
)
else
:
logger
.
error
(
"failed report session {}'s replay log"
.
format
(
session_id
))
return
False
def
get_command_recorder_class
(
config
):
def
get_command_recorder_class
(
config
):
command_storage
=
config
[
"COMMAND_STORAGE"
]
command_storage
=
config
[
"COMMAND_STORAGE"
]
storage_type
=
command_storage
.
get
(
'TYPE'
)
storage_type
=
command_storage
.
get
(
'TYPE'
)
...
@@ -334,12 +292,7 @@ def get_command_recorder_class(config):
...
@@ -334,12 +292,7 @@ def get_command_recorder_class(config):
else
:
else
:
return
ServerCommandRecorder
return
ServerCommandRecorder
#
def
get_replay_recorder_class
(
config
):
# def get_replay_recorder_class(config):
replay_storage
=
config
[
"REPLAY_STORAGE"
]
# ServerReplayRecorder.client = jms_storage.init(config["REPLAY_STORAGE"])
logger
.
debug
(
replay_storage
)
# return ServerReplayRecorder
storage_type
=
replay_storage
.
get
(
'TYPE'
)
if
storage_type
==
"s3"
:
return
S3ReplayRecorder
else
:
return
ServerReplayRecorder
requirements/requirements.txt
View file @
983a9d8c
asn1crypto==0.23.0
asn1crypto==0.23.0
bcrypt==3.1.4
bcrypt==3.1.4
boto3==1.5.18
boto3==1.6.5
botocore==1.8.32
botocore==1.9.5
certifi==2017.11.5
cachetools==2.0.1
certifi==2018.1.18
cffi==1.11.2
cffi==1.11.2
chardet==3.0.4
chardet==3.0.4
click==6.7
click==6.7
crcmod==1.7
cryptography==2.1.4
cryptography==2.1.4
docutils==0.14
dotmap==1.2.20
elasticsearch==6.1.1
Flask==0.12.2
Flask==0.12.2
Flask-SocketIO==2.9.2
Flask-SocketIO==2.9.2
idna==2.6
idna==2.6
itsdangerous==0.24
itsdangerous==0.24
Jinja2==2.10
Jinja2==2.10
jmespath==0.9.3
jms-es-sdk==0.5.2
jms-storage==0.0.11
jumpserver-python-sdk==0.0.32
MarkupSafe==1.0
MarkupSafe==1.0
oss2==2.4.0
paramiko==2.4.0
paramiko==2.4.0
psutil==5.4.1
psutil==5.4.1
pyasn1==0.4.2
pyasn1==0.4.2
pycparser==2.18
pycparser==2.18
PyNaCl==1.2.0
PyNaCl==1.2.0
pyte==0.7.0
pyte==0.7.0
python-dateutil==2.6.1
python-engineio==2.0.1
python-engineio==2.0.1
python-gssapi==0.6.4
python-gssapi==0.6.4
python-socketio==1.8.3
python-socketio==1.8.3
pytz==2017.3
pytz==2017.3
requests==2.18.4
requests==2.18.4
s3transfer==0.1.13
simplejson==3.13.2
simplejson==3.13.2
six==1.11.0
six==1.11.0
tornado==4.5.2
tornado==4.5.2
urllib3==1.22
urllib3==1.22
wcwidth==0.1.7
wcwidth==0.1.7
werkzeug==0.12.2
Werkzeug==0.12.2
jumpserver-python-sdk==0.0.32
jms-es-sdk
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment