Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
C
coco
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ops
coco
Commits
60bd34d8
Unverified
Commit
60bd34d8
authored
Mar 08, 2018
by
liuzheng712
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: jms_storage 0.0.9 test
parent
35f530f3
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
43 additions
and
82 deletions
+43
-82
recorder.py
coco/recorder.py
+27
-76
requirements.txt
requirements/requirements.txt
+16
-6
No files found.
coco/recorder.py
View file @
60bd34d8
...
@@ -9,9 +9,8 @@ import os
...
@@ -9,9 +9,8 @@ import os
import
gzip
import
gzip
import
json
import
json
import
shutil
import
shutil
import
boto3
# AWS S3 sdk
from
jms_es_sdk
import
ESStor
e
import
jms_storag
e
from
.utils
import
get_logger
from
.utils
import
get_logger
from
.alignment
import
MemoryQueue
from
.alignment
import
MemoryQueue
...
@@ -93,6 +92,7 @@ class ServerReplayRecorder(ReplayRecorder):
...
@@ -93,6 +92,7 @@ class ServerReplayRecorder(ReplayRecorder):
def
__init__
(
self
,
app
):
def
__init__
(
self
,
app
):
super
()
.
__init__
(
app
)
super
()
.
__init__
(
app
)
self
.
file
=
None
self
.
file
=
None
self
.
client
=
None
def
record
(
self
,
data
):
def
record
(
self
,
data
):
"""
"""
...
@@ -125,14 +125,16 @@ class ServerReplayRecorder(ReplayRecorder):
...
@@ -125,14 +125,16 @@ class ServerReplayRecorder(ReplayRecorder):
with
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay'
),
'rb'
)
as
f_in
,
\
with
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay'
),
'rb'
)
as
f_in
,
\
gzip
.
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
'wb'
)
as
f_out
:
gzip
.
open
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
'wb'
)
as
f_out
:
shutil
.
copyfileobj
(
f_in
,
f_out
)
shutil
.
copyfileobj
(
f_in
,
f_out
)
if
self
.
push_to_server
(
session_id
):
if
self
.
upload_replay
(
session_id
):
logger
.
info
(
"Succeed to push {}'s {}"
.
format
(
session_id
,
"record"
))
logger
.
info
(
"Succeed to push {}'s {}"
.
format
(
session_id
,
"record"
))
else
:
else
:
logger
.
error
(
"Failed to push {}'s {}"
.
format
(
session_id
,
"record"
))
logger
.
error
(
"Failed to push {}'s {}"
.
format
(
session_id
,
"record"
))
self
.
push_to_server
(
session_id
)
self
.
upload_replay
(
session_id
)
def
push_to_server
(
self
,
session_id
):
def
upload_replay
(
self
,
session_id
):
if
self
.
upload_replay
(
3
,
session_id
):
if
not
self
.
client
:
self
.
client
=
jms_storage
.
jms
(
self
.
app
.
service
)
if
self
.
push_storage
(
3
,
session_id
):
if
self
.
finish_replay
(
3
,
session_id
):
if
self
.
finish_replay
(
3
,
session_id
):
return
True
return
True
else
:
else
:
...
@@ -140,21 +142,27 @@ class ServerReplayRecorder(ReplayRecorder):
...
@@ -140,21 +142,27 @@ class ServerReplayRecorder(ReplayRecorder):
else
:
else
:
return
False
return
False
def
push_local
(
self
,
session_id
):
def
push_to_storage
(
self
,
session_id
):
return
self
.
app
.
service
.
push_session_replay
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
return
self
.
client
.
upload_file
(
session_id
)
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
time
.
strftime
(
'
%
Y-
%
m-
%
d'
,
time
.
localtime
(
self
.
starttime
))
+
'/'
+
session_id
+
'.replay.gz'
)
def
upload_replay
(
self
,
times
,
session_id
):
def
push_storage
(
self
,
times
,
session_id
):
if
times
>
0
:
if
times
>
0
:
if
self
.
push_
local
(
session_id
):
if
self
.
push_
to_storage
(
session_id
):
logger
.
info
(
"success push session: {}'s replay log "
.
format
(
session_id
))
logger
.
info
(
"success push session: {}'s replay log
to storage
"
.
format
(
session_id
))
return
True
return
True
else
:
else
:
logger
.
error
(
"failed report session {}'s replay log, try {} times"
.
format
(
session_id
,
times
))
logger
.
error
(
return
self
.
upload_replay
(
times
-
1
,
session_id
)
"failed report session {}'s replay log to storage, try {} times"
.
format
(
session_id
,
times
))
return
self
.
push_storage
(
times
-
1
,
session_id
)
else
:
else
:
logger
.
error
(
"failed report session {}'s replay log"
.
format
(
session_id
))
logger
.
error
(
"failed report session {}'s replay log storage, try to push to local"
.
format
(
session_id
))
return
False
if
self
.
client
.
type
()
==
'jms'
:
return
False
else
:
self
.
client
=
jms_storage
.
jms
(
self
.
app
.
service
)
return
self
.
push_storage
(
3
,
session_id
)
def
finish_replay
(
self
,
times
,
session_id
):
def
finish_replay
(
self
,
times
,
session_id
):
if
times
>
0
:
if
times
>
0
:
...
@@ -232,7 +240,7 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
...
@@ -232,7 +240,7 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
self
.
stop_evt
=
threading
.
Event
()
self
.
stop_evt
=
threading
.
Event
()
self
.
push_to_es_async
()
self
.
push_to_es_async
()
self
.
__class__
.
no
+=
1
self
.
__class__
.
no
+=
1
self
.
store
=
ESStore
(
app
.
config
[
"COMMAND_STORAGE"
]
.
get
(
"HOSTS"
,
self
.
default_hosts
))
self
.
store
=
jms_storage
.
ESStore
(
app
.
config
[
"COMMAND_STORAGE"
]
.
get
(
"HOSTS"
,
self
.
default_hosts
))
if
not
self
.
store
.
ping
():
if
not
self
.
store
.
ping
():
raise
AssertionError
(
"ESCommand storage init error"
)
raise
AssertionError
(
"ESCommand storage init error"
)
...
@@ -273,58 +281,6 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
...
@@ -273,58 +281,6 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
print
(
"{} has been gc"
.
format
(
self
))
print
(
"{} has been gc"
.
format
(
self
))
class
S3ReplayRecorder
(
ServerReplayRecorder
):
def
__init__
(
self
,
app
):
super
()
.
__init__
(
app
)
self
.
bucket
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"BUCKET"
,
"jumpserver"
)
self
.
REGION
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"REGION"
,
None
)
self
.
ACCESS_KEY
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"ACCESS_KEY"
,
None
)
self
.
SECRET_KEY
=
app
.
config
[
"REPLAY_STORAGE"
]
.
get
(
"SECRET_KEY"
,
None
)
if
self
.
ACCESS_KEY
and
self
.
REGION
and
self
.
SECRET_KEY
:
self
.
s3
=
boto3
.
client
(
's3'
,
region_name
=
self
.
REGION
,
aws_access_key_id
=
self
.
ACCESS_KEY
,
aws_secret_access_key
=
self
.
SECRET_KEY
)
else
:
self
.
s3
=
boto3
.
client
(
's3'
)
def
push_to_s3
(
self
,
session_id
):
logger
.
debug
(
"push to s3"
)
try
:
self
.
s3
.
upload_file
(
os
.
path
.
join
(
self
.
app
.
config
[
'LOG_DIR'
],
session_id
+
'.replay.gz'
),
self
.
bucket
,
time
.
strftime
(
'
%
Y-
%
m-
%
d'
,
time
.
localtime
(
self
.
starttime
))
+
'/'
+
session_id
+
'.replay.gz'
)
return
True
except
:
return
False
def
upload_replay
(
self
,
times
,
session_id
):
if
times
>
0
:
if
self
.
push_to_s3
(
session_id
):
logger
.
info
(
"success push session: {}'s replay log to S3 "
.
format
(
session_id
))
return
True
else
:
logger
.
error
(
"failed report session {}'s replay log to S3, try {} times"
.
format
(
session_id
,
times
))
return
self
.
upload_replay
(
times
-
1
,
session_id
)
else
:
logger
.
error
(
"failed report session {}'s replay log S3, try to push to local"
.
format
(
session_id
))
return
self
.
upload_replay_to_local
(
3
,
session_id
)
def
upload_replay_to_local
(
self
,
times
,
session_id
):
if
times
>
0
:
if
self
.
push_local
(
session_id
):
logger
.
info
(
"success push session: {}'s replay log "
.
format
(
session_id
))
return
True
else
:
logger
.
error
(
"failed report session {}'s replay log, try {} times"
.
format
(
session_id
,
times
))
return
self
.
upload_replay_to_local
(
times
-
1
,
session_id
)
else
:
logger
.
error
(
"failed report session {}'s replay log"
.
format
(
session_id
))
return
False
def
get_command_recorder_class
(
config
):
def
get_command_recorder_class
(
config
):
command_storage
=
config
[
"COMMAND_STORAGE"
]
command_storage
=
config
[
"COMMAND_STORAGE"
]
storage_type
=
command_storage
.
get
(
'TYPE'
)
storage_type
=
command_storage
.
get
(
'TYPE'
)
...
@@ -336,10 +292,5 @@ def get_command_recorder_class(config):
...
@@ -336,10 +292,5 @@ def get_command_recorder_class(config):
def
get_replay_recorder_class
(
config
):
def
get_replay_recorder_class
(
config
):
replay_storage
=
config
[
"REPLAY_STORAGE"
]
ServerReplayRecorder
.
client
=
jms_storage
.
init
(
config
[
"REPLAY_STORAGE"
])
logger
.
debug
(
replay_storage
)
return
ServerReplayRecorder
storage_type
=
replay_storage
.
get
(
'TYPE'
)
if
storage_type
==
"s3"
:
return
S3ReplayRecorder
else
:
return
ServerReplayRecorder
requirements/requirements.txt
View file @
60bd34d8
asn1crypto==0.23.0
asn1crypto==0.23.0
bcrypt==3.1.4
bcrypt==3.1.4
boto3==1.5.18
boto3==1.6.5
botocore==1.8.32
botocore==1.9.5
certifi==2017.11.5
cachetools==2.0.1
certifi==2018.1.18
cffi==1.11.2
cffi==1.11.2
chardet==3.0.4
chardet==3.0.4
click==6.7
click==6.7
crcmod==1.7
cryptography==2.1.4
cryptography==2.1.4
docutils==0.14
dotmap==1.2.20
elasticsearch==6.1.1
Flask==0.12.2
Flask==0.12.2
Flask-SocketIO==2.9.2
Flask-SocketIO==2.9.2
idna==2.6
idna==2.6
itsdangerous==0.24
itsdangerous==0.24
Jinja2==2.10
Jinja2==2.10
jmespath==0.9.3
jms-es-sdk==0.5.2
jms-storage==0.0.9
jumpserver-python-sdk==0.0.31
MarkupSafe==1.0
MarkupSafe==1.0
oss2==2.4.0
paramiko==2.4.0
paramiko==2.4.0
psutil==5.4.1
psutil==5.4.1
pyasn1==0.4.2
pyasn1==0.4.2
pycparser==2.18
pycparser==2.18
PyNaCl==1.2.0
PyNaCl==1.2.0
pyte==0.7.0
pyte==0.7.0
python-dateutil==2.6.1
python-engineio==2.0.1
python-engineio==2.0.1
python-gssapi==0.6.4
python-gssapi==0.6.4
python-socketio==1.8.3
python-socketio==1.8.3
pytz==2017.3
pytz==2017.3
requests==2.18.4
requests==2.18.4
s3transfer==0.1.13
simplejson==3.13.2
simplejson==3.13.2
six==1.11.0
six==1.11.0
tornado==4.5.2
tornado==4.5.2
urllib3==1.22
urllib3==1.22
wcwidth==0.1.7
wcwidth==0.1.7
werkzeug==0.12.2
Werkzeug==0.12.2
jumpserver-python-sdk==0.0.32
jms-es-sdk
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment