Session task (#2196)

* [Bugfix] 修复错误

* [Update] 增加会话定期清理

apps/common/forms.py

@@ -15,8 +15,6 @@ class BaseForm(forms.Form):
         super().__init__(*args, **kwargs)
         for name, field in self.fields.items():
             value = getattr(settings, name, None)
-            # django_value = getattr(settings, name) if hasattr(settings, name) else None
-
             if value is None:  # and django_value is None:
                 continue
 
@@ -24,8 +22,6 @@ class BaseForm(forms.Form):
                 if isinstance(value, dict):
                     value = json.dumps(value)
                 initial_value = value
-            # elif django_value is False or django_value:
-            #     initial_value = django_value
             else:
                 initial_value = ''
             field.initial = initial_value
@@ -157,6 +153,11 @@ class TerminalSettingForm(BaseForm):
     TERMINAL_ASSET_LIST_PAGE_SIZE = forms.ChoiceField(
         choices=PAGE_SIZE_CHOICES, initial='auto', label=_("List page size"),
     )
+    TERMINAL_SESSION_KEEP_DURATION = forms.IntegerField(
+        label=_("Session keep duration"),
+        help_text=_("Units: days, Session, record, command will be delete "
+                    "if more than duration, only in database")
+    )
 
 
 class TerminalCommandStorage(BaseForm):

apps/common/signals_handler.py

@@ -26,21 +26,20 @@ def refresh_settings_on_changed(sender, instance=None, **kwargs):
 def refresh_all_settings_on_django_ready(sender, **kwargs):
     logger.debug("Receive django ready signal")
     logger.debug("  - fresh all settings")
-    CACHE_KEY_PREFIX = '_SETTING_'
+    cache_key_prefix = '_SETTING_'
 
     def monkey_patch_getattr(self, name):
-        key = CACHE_KEY_PREFIX + name
+        key = cache_key_prefix + name
         cached = cache.get(key)
         if cached is not None:
             return cached
         if self._wrapped is empty:
             self._setup(name)
         val = getattr(self._wrapped, name)
-        # self.__dict__[name] = val  # Never set it
         return val
 
     def monkey_patch_setattr(self, name, value):
-        key = CACHE_KEY_PREFIX + name
+        key = cache_key_prefix + name
         cache.set(key, value, None)
         if name == '_wrapped':
             self.__dict__.clear()
@@ -51,7 +50,7 @@ def refresh_all_settings_on_django_ready(sender, **kwargs):
     def monkey_patch_delattr(self, name):
         super(LazySettings, self).__delattr__(name)
         self.__dict__.pop(name, None)
-        key = CACHE_KEY_PREFIX + name
+        key = cache_key_prefix + name
         cache.delete(key)
 
     try:

apps/jumpserver/conf.py

@@ -318,6 +318,7 @@ defaults = {
     'TERMINAL_HEARTBEAT_INTERVAL': 5,
     'TERMINAL_ASSET_LIST_SORT_BY': 'hostname',
     'TERMINAL_ASSET_LIST_PAGE_SIZE': 'auto',
+    'TERMINAL_SESSION_KEEP_DURATION': 9999,
 }
 
 

apps/jumpserver/settings.py

@@ -467,6 +467,7 @@ DEFAULT_TERMINAL_REPLAY_STORAGE = {
 TERMINAL_REPLAY_STORAGE = {
 }
 
+
 SECURITY_MFA_AUTH = False
 SECURITY_LOGIN_LIMIT_COUNT = 7
 SECURITY_LOGIN_LIMIT_TIME = 30  # Unit: minute
@@ -490,6 +491,7 @@ TERMINAL_PUBLIC_KEY_AUTH = CONFIG.TERMINAL_PUBLIC_KEY_AUTH
 TERMINAL_HEARTBEAT_INTERVAL = CONFIG.TERMINAL_HEARTBEAT_INTERVAL
 TERMINAL_ASSET_LIST_SORT_BY = CONFIG.TERMINAL_ASSET_LIST_SORT_BY
 TERMINAL_ASSET_LIST_PAGE_SIZE = CONFIG.TERMINAL_ASSET_LIST_PAGE_SIZE
+TERMINAL_SESSION_KEEP_DURATION = CONFIG.TERMINAL_SESSION_KEEP_DURATION
 
 # Django bootstrap3 setting, more see http://django-bootstrap3.readthedocs.io/en/latest/settings.html
 BOOTSTRAP3 = {

apps/locale/zh/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Jumpserver 0.3.3\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2018-12-17 20:06+0800\n"
+"POT-Creation-Date: 2018-12-18 10:13+0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: ibuler <ibuler@qq.com>\n"
 "Language-Team: Jumpserver team<ibuler@qq.com>\n"
@@ -1976,47 +1976,57 @@ msgstr "资产列表排序"
 
 #: common/forms.py:158
 msgid "List page size"
-msgstr "资产列表页面大小"
+msgstr "资产分页每页数量"
 
-#: common/forms.py:170
+#: common/forms.py:161
+msgid "Session keep duration"
+msgstr "会话保留时长"
+
+#: common/forms.py:162
+msgid ""
+"Units: days, Session, record, command will be delete if more than duration, "
+"only in database"
+msgstr "单位：天。 会话、录像、命令记录超过该时长将会被删除(仅影响数据库存储, oss等不受影响)"
+
+#: common/forms.py:175
 msgid "MFA Secondary certification"
 msgstr "MFA 二次认证"
 
-#: common/forms.py:172
+#: common/forms.py:177
 msgid ""
 "After opening, the user login must use MFA secondary authentication (valid "
 "for all users, including administrators)"
 msgstr "开启后，用户登录必须使用MFA二次认证（对所有用户有效，包括管理员）"
 
-#: common/forms.py:179
+#: common/forms.py:184
 msgid "Limit the number of login failures"
 msgstr "限制登录失败次数"
 
-#: common/forms.py:184
+#: common/forms.py:189
 msgid "No logon interval"
 msgstr "禁止登录时间间隔"
 
-#: common/forms.py:186
+#: common/forms.py:191
 msgid ""
 "Tip: (unit/minute) if the user has failed to log in for a limited number of "
 "times, no login is allowed during this time interval."
 msgstr ""
 "提示：（单位：分）当用户登录失败次数达到限制后，那么在此时间间隔内禁止登录"
 
-#: common/forms.py:193
+#: common/forms.py:198
 msgid "Connection max idle time"
 msgstr "SSH最大空闲时间"
 
-#: common/forms.py:195
+#: common/forms.py:200
 msgid ""
 "If idle time more than it, disconnect connection(only ssh now) Unit: minute"
 msgstr "提示：（单位：分）如果超过该配置没有操作，连接会被断开（仅ssh）"
 
-#: common/forms.py:201
+#: common/forms.py:206
 msgid "Password expiration time"
 msgstr "密码过期时间"
 
-#: common/forms.py:204
+#: common/forms.py:209
 msgid ""
 "Tip: (unit: day) If the user does not update the password during the time, "
 "the user password will expire failure;The password expiration reminder mail "
@@ -2026,45 +2036,45 @@ msgstr ""
 "提示：（单位：天）如果用户在此期间没有更新密码，用户密码将过期失效； 密码过期"
 "提醒邮件将在密码过期前5天内由系统（每天）自动发送给用户"
 
-#: common/forms.py:213
+#: common/forms.py:218
 msgid "Password minimum length"
 msgstr "密码最小长度 "
 
-#: common/forms.py:219
+#: common/forms.py:224
 msgid "Must contain capital letters"
 msgstr "必须包含大写字母"
 
-#: common/forms.py:221
+#: common/forms.py:226
 msgid ""
 "After opening, the user password changes and resets must contain uppercase "
 "letters"
 msgstr "开启后，用户密码修改、重置必须包含大写字母"
 
-#: common/forms.py:227
+#: common/forms.py:232
 msgid "Must contain lowercase letters"
 msgstr "必须包含小写字母"
 
-#: common/forms.py:228
+#: common/forms.py:233
 msgid ""
 "After opening, the user password changes and resets must contain lowercase "
 "letters"
 msgstr "开启后，用户密码修改、重置必须包含小写字母"
 
-#: common/forms.py:234
+#: common/forms.py:239
 msgid "Must contain numeric characters"
 msgstr "必须包含数字字符"
 
-#: common/forms.py:235
+#: common/forms.py:240
 msgid ""
 "After opening, the user password changes and resets must contain numeric "
 "characters"
 msgstr "开启后，用户密码修改、重置必须包含数字字符"
 
-#: common/forms.py:241
+#: common/forms.py:246
 msgid "Must contain special characters"
 msgstr "必须包含特殊字符"
 
-#: common/forms.py:242
+#: common/forms.py:247
 msgid ""
 "After opening, the user password changes and resets must contain special "
 "characters"

apps/terminal/api/v1/session.py

@@ -94,44 +94,15 @@ class SessionReplayViewSet(viewsets.ViewSet):
     serializer_class = serializers.ReplaySerializer
     permission_classes = (IsOrgAdminOrAppUser,)
     session = None
-    upload_to = 'replay'  # 仅添加到本地存储中
-
-    def get_session_path(self, version=2):
-        """
-        获取session日志的文件路径
-        :param version: 原来后缀是 .gz，为了统一新版本改为 .replay.gz
-        :return:
-        """
-        suffix = '.replay.gz'
-        if version == 1:
-            suffix = '.gz'
-        date = self.session.date_start.strftime('%Y-%m-%d')
-        return os.path.join(date, str(self.session.id) + suffix)
-
-    def get_local_path(self, version=2):
-        session_path = self.get_session_path(version=version)
-        if version == 2:
-            local_path = os.path.join(self.upload_to, session_path)
-        else:
-            local_path = session_path
-        return local_path
-
-    def save_to_storage(self, f):
-        local_path = self.get_local_path()
-        try:
-            name = default_storage.save(local_path, f)
-            return name, None
-        except OSError as e:
-            return None, e
 
     def create(self, request, *args, **kwargs):
         session_id = kwargs.get('pk')
-        self.session = get_object_or_404(Session, id=session_id)
+        session = get_object_or_404(Session, id=session_id)
         serializer = self.serializer_class(data=request.data)
 
         if serializer.is_valid():
             file = serializer.validated_data['file']
-            name, err = self.save_to_storage(file)
+            name, err = session.save_to_storage(file)
             if not name:
                 msg = "Failed save replay `{}`: {}".format(session_id, err)
                 logger.error(msg)
@@ -145,7 +116,7 @@ class SessionReplayViewSet(viewsets.ViewSet):
 
     def retrieve(self, request, *args, **kwargs):
         session_id = kwargs.get('pk')
-        self.session = get_object_or_404(Session, id=session_id)
+        session = get_object_or_404(Session, id=session_id)
 
         data = {
             'type': 'guacamole' if self.session.protocol == 'rdp' else 'json',
@@ -153,9 +124,9 @@ class SessionReplayViewSet(viewsets.ViewSet):
         }
 
         # 新版本和老版本的文件后缀不同
-        session_path = self.get_session_path()  # 存在外部存储上的路径
-        local_path = self.get_local_path()
-        local_path_v1 = self.get_local_path(version=1)
+        session_path = session.get_rel_replay_path()  # 存在外部存储上的路径
+        local_path = session.get_local_path()
+        local_path_v1 = session.get_local_path(version=1)
 
         # 去default storage中查找
         for _local_path in (local_path, local_path_v1, session_path):

apps/terminal/models.py

 from __future__ import unicode_literals
 
+import os
 import uuid
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
 from django.conf import settings
+from django.core.files.storage import default_storage
 
 from users.models import User
 from orgs.mixins import OrgModelMixin
@@ -148,6 +150,36 @@ class Session(OrgModelMixin):
     date_start = models.DateTimeField(verbose_name=_("Date start"), db_index=True, default=timezone.now)
     date_end = models.DateTimeField(verbose_name=_("Date end"), null=True)
 
+    upload_to = 'replay'
+
+    def get_rel_replay_path(self, version=2):
+        """
+        获取session日志的文件路径
+        :param version: 原来后缀是 .gz，为了统一新版本改为 .replay.gz
+        :return:
+        """
+        suffix = '.replay.gz'
+        if version == 1:
+            suffix = '.gz'
+        date = self.date_start.strftime('%Y-%m-%d')
+        return os.path.join(date, str(self.id) + suffix)
+
+    def get_local_path(self, version=2):
+        rel_path = self.get_rel_replay_path(version=version)
+        if version == 2:
+            local_path = os.path.join(self.upload_to, rel_path)
+        else:
+            local_path = rel_path
+        return local_path
+
+    def save_to_storage(self, f):
+        local_path = self.get_local_path()
+        try:
+            name = default_storage.save(local_path, f)
+            return name, None
+        except OSError as e:
+            return None, e
+
     class Meta:
         db_table = "terminal_session"
         ordering = ["-date_start"]

apps/terminal/tasks.py

@@ -4,15 +4,20 @@
 import datetime
 
 from celery import shared_task
+from celery.utils.log import get_task_logger
 from django.utils import timezone
+from django.conf import settings
+from django.core.files.storage import default_storage
+
 
 from ops.celery.utils import register_as_period_task, after_app_ready_start, \
     after_app_shutdown_clean
-from .models import Status, Session
+from .models import Status, Session, Command
 
 
 CACHE_REFRESH_INTERVAL = 10
 RUNNING = False
+logger = get_task_logger(__name__)
 
 
 @shared_task
@@ -34,3 +39,28 @@ def clean_orphan_session():
         if not session.terminal or not session.terminal.is_active:
             session.is_finished = True
             session.save()
+
+
+@shared_task
+@register_as_period_task(interval=3600*24)
+@after_app_ready_start
+@after_app_shutdown_clean
+def clean_expired_session_period():
+    logger.info("Start clean expired session record, commands and replay")
+    days = settings.TERMINAL_SESSION_KEEP_DURATION
+    dt = timezone.now() - timezone.timedelta(days=days)
+    expired_sessions = Session.objects.filter(date_start__lt=dt)
+    for session in expired_sessions:
+        logger.info("Clean session: {}".format(session.id))
+        Command.objects.filter(session=str(session.id)).delete()
+        # 删除录像文件
+        session_path = session.get_rel_replay_path()
+        local_path = session.get_local_path()
+        local_path_v1 = session.get_local_path(version=1)
+
+        # 去default storage中查找
+        for _local_path in (local_path, local_path_v1, session_path):
+            if default_storage.exists(_local_path):
+                default_storage.delete(_local_path)
+        # 删除session记录
+        session.delete()