[Update] 修改swagger

apps/assets/api/__init__.py

@@ -6,3 +6,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/api/admin_user.py

@@ -19,7 +19,7 @@ from rest_framework import generics
 from rest_framework.response import Response
 from orgs.mixins.api import OrgBulkModelViewSet
 
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from common.utils import get_logger
 from ..hands import IsOrgAdmin
 from ..models import AdminUser, Asset

apps/assets/api/asset.py

@@ -5,9 +5,7 @@ import random
 
 from rest_framework import generics
 from rest_framework.response import Response
-from django.utils.translation import ugettext_lazy as _
 from django.shortcuts import get_object_or_404
-from django.db.models import Q
 
 from common.utils import get_logger, get_object_or_none
 from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
@@ -16,7 +14,7 @@ from ..models import Asset, AdminUser, Node
 from .. import serializers
 from ..tasks import update_asset_hardware_info_manual, \
     test_asset_connectivity_manual
-from ..utils import LabelFilter
+from ..filters import AssetByNodeFilterBackend, LabelFilterBackend
 
 
 logger = get_logger(__file__)
@@ -27,7 +25,7 @@ __all__ = [
 ]
 
 
-class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
+class AssetViewSet(OrgBulkModelViewSet):
     """
     API endpoint that allows Asset to be viewed or edited.
     """
@@ -37,7 +35,7 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
     queryset = Asset.objects.all()
     serializer_class = serializers.AssetSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
-    success_message = _("%(hostname)s was %(action)s successfully")
+    extra_filter_backends = [AssetByNodeFilterBackend, LabelFilterBackend]
 
     def set_assets_node(self, assets):
         if not isinstance(assets, list):
@@ -54,30 +52,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
         assets = serializer.save()
         self.set_assets_node(assets)
 
-    def filter_node(self, queryset):
-        node_id = self.request.query_params.get("node_id")
-        if not node_id:
-            return queryset
-
-        node = get_object_or_404(Node, id=node_id)
-        show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true')
-
-        # 当前节点是顶层节点, 并且仅显示直接资产
-        if node.is_org_root() and show_current_asset:
-            queryset = queryset.filter(
-                Q(nodes=node_id) | Q(nodes__isnull=True)
-            ).distinct()
-        # 当前节点是顶层节点，显示所有资产
-        elif node.is_org_root() and not show_current_asset:
-            return queryset
-        # 当前节点不是鼎城节点，只显示直接资产
-        elif not node.is_org_root() and show_current_asset:
-            queryset = queryset.filter(nodes=node)
-        else:
-            children = node.get_all_children(with_self=True)
-            queryset = queryset.filter(nodes__in=children).distinct()
-        return queryset
-
     def filter_admin_user_id(self, queryset):
         admin_user_id = self.request.query_params.get('admin_user_id')
         if not admin_user_id:
@@ -88,7 +62,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
 
     def filter_queryset(self, queryset):
         queryset = super().filter_queryset(queryset)
-        queryset = self.filter_node(queryset)
         queryset = self.filter_admin_user_id(queryset)
         return queryset
 

apps/assets/api/asset_user.py

@@ -10,7 +10,7 @@ from django.http import Http404
 
 from common.permissions import IsOrgAdminOrAppUser, NeedMFAVerify
 from common.utils import get_object_or_none, get_logger
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from ..backends import AssetUserManager
 from ..models import Asset, Node, SystemUser, AdminUser
 from .. import serializers
@@ -52,7 +52,7 @@ class AssetUserSearchBackend(filters.BaseFilterBackend):
         return _queryset
 
 
-class AssetUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class AssetUserViewSet(CommonApiMixin, BulkModelViewSet):
     serializer_class = serializers.AssetUserSerializer
     permission_classes = [IsOrgAdminOrAppUser]
     http_method_names = ['get', 'post']

apps/assets/api/gathered_user.py

+# -*- coding: utf-8 -*-
+#
+
+from orgs.mixins.api import OrgModelViewSet
+from assets.models import GatheredUser
+from common.permissions import IsOrgAdmin
+
+from ..serializers import GatheredUserSerializer
+
+
+__all__ = ['GatheredUserViewSet']
+
+
+class GatheredUserViewSet(OrgModelViewSet):
+    queryset = GatheredUser.objects.all()
+    serializer_class = GatheredUserSerializer
+    permission_classes = [IsOrgAdmin]
+
+    filter_fields = ['asset', 'username', 'present']
+    search_fields = ['username', 'asset__ip', 'asset__hostname']
+
+

apps/assets/filters.py

+# -*- coding: utf-8 -*-
+#
+
+import coreapi
+from rest_framework import filters
+from django.db.models import Q
+
+from common.utils import dict_get_any, is_uuid, get_object_or_none
+from .models import Node, Label
+
+
+class AssetByNodeFilterBackend(filters.BaseFilterBackend):
+    fields = ['node', 'all']
+
+    # def filter_node(self, queryset):
+    #     node_id = self.request.query_params.get("node_id")
+    #     if not node_id:
+    #         return queryset
+    #
+    #     node = get_object_or_404(Node, id=node_id)
+    #     show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true')
+    #
+    #     # 当前节点是顶层节点, 并且仅显示直接资产
+    #     if node.is_org_root() and show_current_asset:
+    #         queryset = queryset.filter(
+    #             Q(nodes=node_id) | Q(nodes__isnull=True)
+    #         ).distinct()
+    #     # 当前节点是顶层节点，显示所有资产
+    #     elif node.is_org_root() and not show_current_asset:
+    #         return queryset
+    #     # 当前节点不是鼎城节点，只显示直接资产
+    #     elif not node.is_org_root() and show_current_asset:
+    #         queryset = queryset.filter(nodes=node)
+    #     else:
+    #         children = node.get_all_children(with_self=True)
+    #         queryset = queryset.filter(nodes__in=children).distinct()
+    #     return queryset
+
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name=field, location='query', required=False,
+                type='string', example='', description=''
+            )
+            for field in self.fields
+        ]
+
+    def filter_queryset(self, request, queryset, view):
+        node_id = dict_get_any(request.query_params, ['node', 'node_id'])
+        if not node_id:
+            return queryset
+        query_all_arg = request.query_params.get('all')
+        show_current_asset_arg = request.query_params.get('show_current_asset')
+
+        query_all = query_all_arg == '1'
+        if show_current_asset_arg is not None:
+            query_all = show_current_asset_arg != '1'
+
+        if is_uuid(node_id):
+            node = get_object_or_none(Node, id=node_id)
+        else:
+            node = get_object_or_none(Node, key=node_id)
+
+        if not node:
+            return queryset.none()
+
+        if query_all:
+            pattern = node.get_all_children_pattern(with_self=True)
+        else:
+            pattern = node.get_children_key_pattern(with_self=True)
+        return queryset.filter(nodes__key__regex=pattern)
+
+
+class LabelFilterBackend(filters.BaseFilterBackend):
+    sep = '#'
+    query_arg = 'label'
+
+    def get_schema_fields(self, view):
+        example = self.sep.join(['os', 'linux'])
+        return [
+            coreapi.Field(
+                name=self.query_arg, location='query', required=False,
+                type='string', example=example, description=''
+            )
+        ]
+
+    def get_query_labels(self, request):
+        labels_query = request.query_params.getlist(self.query_arg)
+        if not labels_query:
+            return None
+
+        q = None
+        for kv in labels_query:
+            if self.sep not in kv:
+                continue
+            key, value = kv.strip().split(self.sep)[:2]
+            if not all([key, value]):
+                continue
+            if q:
+                q |= Q(name=key, value=value)
+            else:
+                q = Q(name=key, value=value)
+        if not q:
+            return []
+        labels = Label.objects.filter(q, is_active=True)\
+            .values_list('id', flat=True)
+        return labels
+
+    def filter_queryset(self, request, queryset, view):
+        labels = self.get_query_labels(request)
+        if labels is None:
+            return queryset
+        if len(labels) == 0:
+            return queryset.none()
+        for label in labels:
+            queryset = queryset.filter(labels=label)
+        return queryset
+
+

apps/assets/migrations/0039_authbook_is_active.py

+# Generated by Django 2.1.7 on 2019-09-17 12:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0038_auto_20190911_1634'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='authbook',
+            name='is_active',
+            field=models.BooleanField(default=True, verbose_name='Is active'),
+        ),
+    ]

apps/assets/migrations/0040_auto_20190917_2056.py

+# Generated by Django 2.1.7 on 2019-09-17 12:56
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0039_authbook_is_active'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adminuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='authbook',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='gateway',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='systemuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+    ]

apps/assets/migrations/0041_gathereduser.py

+# Generated by Django 2.1.7 on 2019-09-18 04:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0040_auto_20190917_2056'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GatheredUser',
+            fields=[
+                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('username', models.CharField(blank=True, db_index=True, max_length=32, verbose_name='Username')),
+                ('present', models.BooleanField(default=True)),
+                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.Asset')),
+            ],
+            options={'ordering': ['asset'], 'verbose_name': 'GatherUser'},
+        ),
+    ]

apps/assets/models/__init__.py

@@ -9,3 +9,4 @@ from .cmd_filter import *
 from .authbook import *
 from .utils import *
 from .authbook import *
+from .gathered_user import *

apps/assets/models/authbook.py

@@ -13,7 +13,7 @@ __all__ = ['AuthBook']
 class AuthBookQuerySet(models.QuerySet):
 
     def latest_version(self):
-        return self.filter(is_latest=True)
+        return self.filter(is_latest=True).filter(is_active=True)
 
 
 class AuthBookManager(OrgManager):
@@ -24,6 +24,7 @@ class AuthBook(AssetUser):
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
     is_latest = models.BooleanField(default=False, verbose_name=_('Latest version'))
     version = models.IntegerField(default=1, verbose_name=_('Version'))
+    is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
 
     objects = AuthBookManager.from_queryset(AuthBookQuerySet)()
     backend = "db"
@@ -34,25 +35,25 @@ class AuthBook(AssetUser):
     class Meta:
         verbose_name = _('AuthBook')
 
-    def _set_latest(self):
-        self._remove_pre_obj_latest()
+    def set_to_latest(self):
+        self.remove_pre_latest()
         self.is_latest = True
         self.save()
 
-    def _get_pre_obj(self):
+    def get_pre_latest(self):
         pre_obj = self.__class__.objects.filter(
             username=self.username, asset=self.asset
         ).latest_version().first()
         return pre_obj
 
-    def _remove_pre_obj_latest(self):
-        pre_obj = self._get_pre_obj()
+    def remove_pre_latest(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             pre_obj.is_latest = False
             pre_obj.save()
 
-    def _set_version(self):
-        pre_obj = self._get_pre_obj()
+    def set_version(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             self.version = pre_obj.version + 1
         else:
@@ -60,8 +61,8 @@ class AuthBook(AssetUser):
         self.save()
 
     def set_version_and_latest(self):
-        self._set_version()
-        self._set_latest()
+        self.set_version()
+        self.set_to_latest()
 
     def get_related_assets(self):
         return [self.asset]

apps/assets/models/base.py

@@ -26,7 +26,7 @@ logger = get_logger(__file__)
 class AssetUser(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
-    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric])
+    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric], db_index=True)
     password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
     private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
     public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))

apps/assets/models/gathered_user.py

+# -*- coding: utf-8 -*-
+#
+import uuid
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.models import OrgModelMixin
+
+__all__ = ['GatheredUser']
+
+
+class GatheredUser(OrgModelMixin):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE)
+    username = models.CharField(max_length=32, blank=True, db_index=True,
+                                verbose_name=_('Username'))
+    present = models.BooleanField(default=True)
+    date_created = models.DateTimeField(auto_now_add=True,
+                                        verbose_name=_("Date created"))
+    date_updated = models.DateTimeField(auto_now=True,
+                                        verbose_name=_("Date updated"))
+
+    @property
+    def hostname(self):
+        return self.asset.hostname
+
+    @property
+    def ip(self):
+        return self.asset.ip
+
+    class Meta:
+        verbose_name = _('GatherUser')
+        ordering = ['asset']
+
+    def __str__(self):
+        return '{}: {}'.format(self.asset.hostname, self.username)
+
+
+

apps/assets/models/node.py

@@ -116,16 +116,24 @@ class FamilyMixin:
     def all_children(self):
         return self.get_all_children(with_self=False)
 
-    def get_children(self, with_self=False):
+    def get_children_key_pattern(self, with_self=False):
         pattern = r'^{0}:[0-9]+$'.format(self.key)
         if with_self:
             pattern += r'|^{0}$'.format(self.key)
+        return pattern
+
+    def get_children(self, with_self=False):
+        pattern = self.get_children_key_pattern(with_self=with_self)
         return Node.objects.filter(key__regex=pattern)
 
-    def get_all_children(self, with_self=False):
+    def get_all_children_pattern(self, with_self=False):
         pattern = r'^{0}:'.format(self.key)
         if with_self:
             pattern += r'|^{0}$'.format(self.key)
+        return pattern
+
+    def get_all_children(self, with_self=False):
+        pattern = self.get_all_children_pattern(with_self=with_self)
         children = Node.objects.filter(key__regex=pattern)
         return children
 

apps/assets/serializers/__init__.py

@@ -9,3 +9,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/serializers/asset_user.py

@@ -53,6 +53,7 @@ class AssetUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         if not validated_data.get("name") and validated_data.get("username"):
             validated_data["name"] = validated_data["username"]
         instance = AssetUserManager.create(**validated_data)
+        instance.set_version_and_latest()
         return instance
 
 

apps/assets/serializers/gathered_user.py

+# -*- coding: utf-8 -*-
+#
+
+from ..models import GatheredUser
+
+from orgs.mixins.serializers import OrgResourceModelSerializerMixin
+
+
+class GatheredUserSerializer(OrgResourceModelSerializerMixin):
+    class Meta:
+        model = GatheredUser
+        fields = [
+            'id', 'asset', 'hostname', 'ip', 'username',
+            'present', 'date_created', 'date_updated'
+        ]
+        read_only_fields = fields

apps/assets/signals_handler.py

@@ -9,7 +9,7 @@ from django.dispatch import receiver
 
 from common.utils import get_logger
 from common.decorator import on_transaction_commit
-from .models import Asset, SystemUser, Node, AuthBook
+from .models import Asset, SystemUser, Node
 from .tasks import (
     update_assets_hardware_info_util,
     test_asset_connectivity_util,
@@ -190,10 +190,3 @@ def on_asset_nodes_remove(sender, instance=None, action='', model=None,
 def on_node_update_or_created(sender, **kwargs):
     # 刷新节点
     Node.refresh_nodes()
-
-
-@receiver(post_save, sender=AuthBook)
-def on_auth_book_created(sender, instance=None, created=False, **kwargs):
-    if created:
-        logger.debug('Receive create auth book object signal.')
-        instance.set_version_and_latest()

apps/assets/tasks/gather_asset_hardware_info.py

@@ -103,7 +103,7 @@ def update_assets_hardware_info_util(assets, task_name=None):
     )
     result = task.run()
     set_assets_hardware_info(assets, result)
-    return result
+    return True
 
 
 @shared_task(queue="ansible")

apps/assets/tasks/gather_asset_users.py

 # ~*~ coding: utf-8 ~*~
 
+import re
 from collections import defaultdict
 from celery import shared_task
+
 from django.utils.translation import ugettext as _
 
+from orgs.utils import tmp_to_org
+from common.utils import get_logger
+from ..models import GatheredUser, Node
+from .utils import clean_hosts
 from . import const
 
+__all__ = ['gather_asset_users', 'gather_nodes_asset_users']
+logger = get_logger(__name__)
+space = re.compile('\s+')
+ignore_login_shell = re.compile(r'nologin$|sync$|shutdown$|halt$')
+
+
+def parse_linux_result_to_users(result):
+    task_result = {}
+    for task_name, raw in result.items():
+        res = raw.get('ansible_facts', {}).get('getent_passwd')
+        if res:
+            task_result = res
+            break
+    if not task_result or not isinstance(task_result, dict):
+        return []
+    users = []
+    for username, attr in task_result.items():
+        if ignore_login_shell.search(attr[-1]):
+            continue
+        users.append(username)
+    return users
+
+
+def parse_windows_result_to_users(result):
+    task_result = []
+    for task_name, raw in result.items():
+        res = raw.get('stdout_lines', {})
+        if res:
+            task_result = res
+            break
+    if not task_result:
+        return []
+
+    users = []
+
+    for i in range(4):
+        task_result.pop(0)
+    for i in range(2):
+        task_result.pop()
+
+    for line in task_result:
+        user = space.split(line)
+        if user[0]:
+            users.append(user[0])
+    return users
+
+
+def add_asset_users(assets, results):
+    assets_map = {a.hostname: a for a in assets}
+    parser_map = {
+        'linux': parse_linux_result_to_users,
+        'windows': parse_windows_result_to_users
+    }
+
+    assets_users_map = {}
+
+    for platform, platform_results in results.items():
+        for hostname, res in platform_results.items():
+            parse = parser_map.get(platform)
+            users = parse(res)
+            logger.debug('Gathered host users: {} {}'.format(hostname, users))
+            asset = assets_map.get(hostname)
+            if not asset:
+                continue
+            assets_users_map[asset] = users
+
+    for asset, users in assets_users_map.items():
+        with tmp_to_org(asset.org_id):
+            GatheredUser.objects.filter(asset=asset, present=True)\
+                .update(present=False)
+            for username in users:
+                defaults = {'asset': asset, 'username': username, 'present': True}
+                GatheredUser.objects.update_or_create(
+                    defaults=defaults, asset=asset, username=username,
+                )
+
 
 @shared_task(queue="ansible")
-def gather_asset_all_users(assets, task_name=None):
+def gather_asset_users(assets, task_name=None):
     from ops.utils import update_or_create_ansible_task
     if task_name is None:
         task_name = _("Gather assets users")
+    assets = clean_hosts(assets)
+    if not assets:
+        return
     hosts_category = {
         'linux': {
             'hosts': [],
@@ -38,5 +123,12 @@ def gather_asset_all_users(assets, task_name=None):
         )
         raw, summary = task.run()
         results[k].update(raw['ok'])
-    return results
+    add_asset_users(assets, results)
 
+
+@shared_task(queue="ansible")
+def gather_nodes_asset_users(nodes_key):
+    assets = Node.get_nodes_all_assets(nodes_key)
+    assets_groups_by_100 = [assets[i:i+100] for i in range(0, len(assets), 100)]
+    for _assets in assets_groups_by_100:
+        gather_asset_users(_assets)

apps/assets/templates/assets/asset_list.html

@@ -85,7 +85,7 @@
                   <button data-toggle="dropdown" class="btn btn-default btn-sm dropdown-toggle">{% trans 'Label' %} <span class="caret"></span></button>
                   <ul class="dropdown-menu labels">
                       {% for label in labels %}
-                          <li><a style="font-weight: bolder">{{ label.name }}:{{ label.value }}</a></li>
+                          <li><a style="font-weight: bolder">{{ label.name }}#{{ label.value }}</a></li>
                       {% endfor %}
                   </ul>
                </div>
@@ -171,9 +171,13 @@ function initTable() {
         ],
         ajax_url: '{% url "api-assets:asset-list" %}',
         columns: [
-            {data: "id"}, {data: "hostname" }, {data: "ip" },
+            {data: "id"}, {data: "hostname"}, {data: "ip"},
             {data: "cpu_cores", orderable: false},
-            {data: "connectivity", orderable: false}, {data: "id", orderable: false }
+            {
+                data: "connectivity",
+                orderable: false,
+                width: '60px'
+            }, {data: "id", orderable: false}
         ],
         op_html: $('#actions').html()
     };
@@ -271,7 +275,7 @@ $(document).ready(function(){
     setAssetModalOptions(modalOption);
 })
 .on('click', '.labels li', function () {
-    var val = $(this).text();
+    var val = 'label:' + $(this).text();
     $("#asset_list_table_filter input").val(val);
     asset_table.search(val).draw();
  })

apps/assets/urls/api_urls.py

@@ -21,6 +21,7 @@ router.register(r'gateways', api.GatewayViewSet, 'gateway')
 router.register(r'cmd-filters', api.CommandFilterViewSet, 'cmd-filter')
 router.register(r'asset-users', api.AssetUserViewSet, 'asset-user')
 router.register(r'asset-users-info', api.AssetUserExportViewSet, 'asset-user-info')
+router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 
 cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
 cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')

apps/assets/utils.py

@@ -24,37 +24,6 @@ def get_system_user_by_id(id):
     return system_user
 
 
-class LabelFilterMixin:
-    def get_filter_labels_ids(self):
-        query_params = self.request.query_params
-        query_keys = query_params.keys()
-        all_label_keys = Label.objects.values_list('name', flat=True)
-        valid_keys = set(all_label_keys) & set(query_keys)
-
-        if not valid_keys:
-            return []
-
-        labels_query = [
-            {"name": key, "value": query_params[key]}
-            for key in valid_keys
-        ]
-        args = [Q(**kwargs) for kwargs in labels_query]
-        args = reduce(lambda x, y: x | y, args)
-        labels_id = Label.objects.filter(args).values_list('id', flat=True)
-        return labels_id
-
-
-class LabelFilter(LabelFilterMixin):
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        labels_ids = self.get_filter_labels_ids()
-        if not labels_ids:
-            return queryset
-        for labels_id in labels_ids:
-            queryset = queryset.filter(labels=labels_id)
-        return queryset
-
-
 class TreeService(Tree):
     tag_sep = ' / '
     cache_key = '_NODE_FULL_TREE'

apps/common/filters.py

 # -*- coding: utf-8 -*-
 #
+import coreapi
 from rest_framework import filters
 from rest_framework.fields import DateTimeField
 from rest_framework.serializers import ValidationError
+from django.core.cache import cache
 import logging
 
-__all__ = ["DatetimeRangeFilter"]
+from . import const
+
+__all__ = ["DatetimeRangeFilter", "IDSpmFilter", "CustomFilter"]
 
 
 class DatetimeRangeFilter(filters.BaseFilterBackend):
@@ -40,3 +44,50 @@ class DatetimeRangeFilter(filters.BaseFilterBackend):
         if kwargs:
             queryset = queryset.filter(**kwargs)
         return queryset
+
+
+class IDSpmFilter(filters.BaseFilterBackend):
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name='spm', location='query', required=False,
+                type='string', example='',
+                description='Pre post objects id get spm id, then using filter'
+            )
+        ]
+
+    def filter_queryset(self, request, queryset, view):
+        spm = request.query_params.get('spm')
+        if not spm:
+            return queryset
+        cache_key = const.KEY_CACHE_RESOURCES_ID.format(spm)
+        resources_id = cache.get(cache_key)
+        if not resources_id or not isinstance(resources_id, list):
+            queryset = queryset.none()
+            return queryset
+        queryset = queryset.filter(id__in=resources_id)
+        return queryset
+
+
+class CustomFilter(filters.BaseFilterBackend):
+    custom_filter_fields = []  # ["node", "asset"]
+
+    def get_schema_fields(self, view):
+        fields = []
+        defaults = dict(
+            location='query', required=False,
+            type='string', example='',
+            description=''
+        )
+        for field in self.custom_filter_fields:
+            if isinstance(field, str):
+                defaults['name'] = field
+            elif isinstance(field, dict):
+                defaults.update(field)
+            else:
+                continue
+            fields.append(coreapi.Field(**defaults))
+        return fields
+
+    def filter_queryset(self, request, queryset, view):
+        return queryset

apps/common/mixins/api.py

 # -*- coding: utf-8 -*-
 #
 from django.http import JsonResponse
-from django.core.cache import cache
-from django.utils.translation import ugettext_lazy as _
-from django.contrib import messages
+from rest_framework.settings import api_settings
 
-from ..const import KEY_CACHE_RESOURCES_ID
+from ..filters import IDSpmFilter, CustomFilter
 
 __all__ = [
-    "JSONResponseMixin", "IDInCacheFilterMixin", "IDExportFilterMixin",
-    "IDInFilterMixin", "ApiMessageMixin"
+    "JSONResponseMixin", "CommonApiMixin",
+    "IDSpmFilterMixin", "CommonApiMixin",
 ]
 
 
@@ -20,69 +18,31 @@ class JSONResponseMixin(object):
         return JsonResponse(context)
 
 
-class IDInFilterMixin(object):
-    def filter_queryset(self, queryset):
-        queryset = super(IDInFilterMixin, self).filter_queryset(queryset)
-        id_list = self.request.query_params.get('id__in')
-        if id_list:
-            import json
-            try:
-                ids = json.loads(id_list)
-            except Exception as e:
-                return queryset
-            if isinstance(ids, list):
-                queryset = queryset.filter(id__in=ids)
-        return queryset
+class IDSpmFilterMixin:
+    def get_filter_backends(self):
+        backends = super().get_filter_backends()
+        backends.append(IDSpmFilter)
+        return backends
 
 
-class IDInCacheFilterMixin(object):
-
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        spm = self.request.query_params.get('spm')
-        if not spm:
-            return queryset
-        cache_key = KEY_CACHE_RESOURCES_ID.format(spm)
-        resources_id = cache.get(cache_key)
-        if not resources_id or not isinstance(resources_id, list):
-            queryset = queryset.none()
-            return queryset
-        queryset = queryset.filter(id__in=resources_id)
-        return queryset
+class ExtraFilterFieldsMixin:
+    default_added_filters = [CustomFilter, IDSpmFilter]
+    filter_backends = api_settings.DEFAULT_FILTER_BACKENDS
+    extra_filter_fields = []
+    extra_filter_backends = []
 
+    def get_filter_backends(self):
+        if self.filter_backends != self.__class__.filter_backends:
+            return self.filter_backends
+        return list(self.filter_backends) + \
+               self.default_added_filters + \
+               list(self.extra_filter_backends)
 
-class IDExportFilterMixin(object):
     def filter_queryset(self, queryset):
-        # 下载导入模版
-        if self.request.query_params.get('template') == 'import':
-            return []
-        else:
-            return super(IDExportFilterMixin, self).filter_queryset(queryset)
-
-
-class ApiMessageMixin:
-    success_message = _("%(name)s was %(action)s successfully")
-    _action_map = {"create": _("create"), "update": _("update")}
+        for backend in self.get_filter_backends():
+            queryset = backend().filter_queryset(self.request, queryset, self)
+        return queryset
 
-    def get_success_message(self, cleaned_data):
-        if not isinstance(cleaned_data, dict):
-            return ''
-        data = {k: v for k, v in cleaned_data.items()}
-        action = getattr(self, "action", "create")
-        data["action"] = self._action_map.get(action)
-        try:
-            message = self.success_message % data
-        except:
-            message = ''
-        return message
 
-    def dispatch(self, request, *args, **kwargs):
-        resp = super().dispatch(request, *args, **kwargs)
-        if request.method.lower() in ("get", "delete", "patch"):
-            return resp
-        if resp.status_code >= 400:
-            return resp
-        message = self.get_success_message(resp.data)
-        if message:
-            messages.success(request, message)
-        return resp
+class CommonApiMixin(ExtraFilterFieldsMixin):
+    pass

apps/common/utils/common.py

@@ -8,7 +8,6 @@ import datetime
 import uuid
 from functools import wraps
 import time
-import copy
 import ipaddress
 
 
@@ -199,3 +198,18 @@ def timeit(func):
         logger.debug(msg)
         return result
     return wrapper
+
+
+def group_obj_by_count(objs, count=50):
+    objs_grouped = [
+        objs[i:i + count] for i in range(0, len(objs), count)
+    ]
+    return objs_grouped
+
+
+def dict_get_any(d, keys):
+    for key in keys:
+        value = d.get(key)
+        if value:
+            return value
+    return None

apps/jumpserver/swagger.py

@@ -33,6 +33,21 @@ class CustomSwaggerAutoSchema(SwaggerAutoSchema):
         operation.summary = operation.operation_id
         return operation
 
+    def get_filter_parameters(self):
+        if not self.should_filter():
+            return []
+
+        fields = []
+        if hasattr(self.view, 'get_filter_backends'):
+            backends = self.view.get_filter_backends()
+        elif hasattr(self.view, 'filter_backends'):
+            backends = self.view.filter_backends
+        else:
+            backends = []
+        for filter_backend in backends:
+            fields += self.probe_inspectors(self.filter_inspectors, 'get_filter_parameters', filter_backend()) or []
+        return fields
+
 
 def get_swagger_view(version='v1'):
     from .urls import api_v1, api_v2

apps/ops/celery/utils.py

@@ -6,7 +6,9 @@ import os
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
 from django.db.utils import ProgrammingError, OperationalError
-from django_celery_beat.models import PeriodicTask, IntervalSchedule, CrontabSchedule
+from django_celery_beat.models import (
+    PeriodicTask, IntervalSchedule, CrontabSchedule, PeriodicTasks
+)
 
 
 def create_or_update_celery_periodic_tasks(tasks):
@@ -75,17 +77,20 @@ def create_or_update_celery_periodic_tasks(tasks):
         task = PeriodicTask.objects.update_or_create(
             defaults=defaults, name=name,
         )
+        PeriodicTasks.update_changed()
         return task
 
 
 def disable_celery_periodic_task(task_name):
     from django_celery_beat.models import PeriodicTask
     PeriodicTask.objects.filter(name=task_name).update(enabled=False)
+    PeriodicTasks.update_changed()
 
 
 def delete_celery_periodic_task(task_name):
     from django_celery_beat.models import PeriodicTask
     PeriodicTask.objects.filter(name=task_name).delete()
+    PeriodicTasks.update_changed()
 
 
 def get_celery_task_log_path(task_id):

apps/ops/inventory.py

@@ -51,7 +51,7 @@ class JMSBaseInventory(BaseInventory):
     def make_proxy_command(asset):
         gateway = asset.domain.random_gateway()
         proxy_command_list = [
-            "ssh", "-p", str(gateway.port),
+            "ssh", "-o", "Port={}".format(gateway.port),
             "-o", "StrictHostKeyChecking=no",
             "{}@{}".format(gateway.username, gateway.ip),
             "-W", "%h:%p", "-q",

apps/ops/tasks.py

@@ -2,6 +2,7 @@
 import os
 import subprocess
 import datetime
+import time
 
 from django.conf import settings
 from celery import shared_task, subtask
@@ -122,3 +123,22 @@ def hello123():
 def hello_callback(result):
     print(result)
     print("Hello callback")
+
+
+@shared_task
+def add(a, b):
+    time.sleep(5)
+    return max(b)
+
+
+@shared_task
+def add_m(x):
+    from celery import chain
+    a = range(x)
+    b = [a[i:i + 10] for i in range(0, len(a), 10)]
+    s = list()
+    s.append(add.s(b[0], b[1]))
+    for i in b[1:]:
+        s.append(add.s(i))
+    res = chain(*tuple(s))()
+    return res

apps/orgs/mixins/api.py

@@ -3,7 +3,7 @@
 from django.shortcuts import get_object_or_404
 from rest_framework.viewsets import ModelViewSet
 from rest_framework_bulk import BulkModelViewSet
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 
 from ..utils import set_to_root_org
 from ..models import Organization
@@ -20,14 +20,16 @@ class RootOrgViewMixin:
         return super().dispatch(request, *args, **kwargs)
 
 
-class OrgModelViewSet(IDInCacheFilterMixin, ModelViewSet):
+class OrgModelViewSet(CommonApiMixin, ModelViewSet):
     def get_queryset(self):
         return super().get_queryset().all()
 
 
-class OrgBulkModelViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class OrgBulkModelViewSet(CommonApiMixin, BulkModelViewSet):
     def get_queryset(self):
         queryset = super().get_queryset().all()
+        if hasattr(self, 'swagger_fake_view'):
+            return queryset[:1]
         if hasattr(self, 'action') and self.action == 'list' and \
             hasattr(self, 'serializer_class') and \
                 hasattr(self.serializer_class, 'setup_eager_loading'):

apps/orgs/mixins/serializers.py

@@ -11,7 +11,8 @@ from ..utils import get_current_org_id_for_serializer
 
 __all__ = [
     "OrgResourceSerializerMixin", "BulkOrgResourceSerializerMixin",
-    "BulkOrgResourceModelSerializer", "OrgMembershipSerializerMixin"
+    "BulkOrgResourceModelSerializer", "OrgMembershipSerializerMixin",
+    "OrgResourceModelSerializerMixin",
 ]
 
 
@@ -42,6 +43,10 @@ class OrgResourceSerializerMixin(serializers.Serializer):
         return fields
 
 
+class OrgResourceModelSerializerMixin(OrgResourceSerializerMixin, serializers.ModelSerializer):
+    pass
+
+
 class BulkOrgResourceSerializerMixin(BulkSerializerMixin, OrgResourceSerializerMixin):
     pass
 

apps/orgs/utils.py

@@ -18,6 +18,8 @@ def get_org_from_request(request):
 
 
 def set_current_org(org):
+    if isinstance(org, str):
+        org = Organization.get_instance(org)
     setattr(thread_local, 'current_org_id', org.id)
 
 

apps/perms/api/mixin.py

 # -*- coding: utf-8 -*-
 #
-import uuid
-from django.db.models import Q
 
 from rest_framework.generics import get_object_or_404
-from assets.utils import LabelFilterMixin
 from common.permissions import IsValidUser, IsOrgAdminOrAppUser
 from common.utils import get_logger
 from orgs.utils import set_to_root_org
-from ..hands import User, UserGroup, Asset, SystemUser
-from .. import serializers
+from ..hands import User, UserGroup
 
 
 logger = get_logger(__name__)
@@ -54,101 +50,3 @@ class UserGroupPermissionMixin:
         return user_group
 
 
-class GrantAssetsMixin(LabelFilterMixin):
-    serializer_class = serializers.AssetGrantedSerializer
-
-    def get_serializer_queryset(self, queryset):
-        assets_ids = []
-        system_users_ids = set()
-        for asset in queryset:
-            assets_ids.append(asset["id"])
-            system_users_ids.update(set(asset["system_users"]))
-        assets = Asset.objects.filter(id__in=assets_ids).only(
-            *self.serializer_class.Meta.only_fields
-        )
-        assets_map = {asset.id: asset for asset in assets}
-        system_users = SystemUser.objects.filter(id__in=system_users_ids).only(
-            *self.serializer_class.system_users_only_fields
-        )
-        system_users_map = {s.id: s for s in system_users}
-        data = []
-        for item in queryset:
-            i = item["id"]
-            asset = assets_map.get(i)
-            if not asset:
-                continue
-
-            _system_users = item["system_users"]
-            system_users_granted = []
-            for sid, action in _system_users.items():
-                system_user = system_users_map.get(sid)
-                if not system_user:
-                    continue
-                if not asset.has_protocol(system_user.protocol):
-                    continue
-                system_user.actions = action
-                system_users_granted.append(system_user)
-            asset.system_users_granted = system_users_granted
-            data.append(asset)
-        return data
-
-    def get_serializer(self, assets_items=None, many=True):
-        if assets_items is None:
-            assets_items = []
-        assets_items = self.get_serializer_queryset(assets_items)
-        return super().get_serializer(assets_items, many=many)
-
-    def filter_queryset_by_id(self, assets_items):
-        i = self.request.query_params.get("id")
-        if not i:
-            return assets_items
-        try:
-            pk = uuid.UUID(i)
-        except ValueError:
-            return assets_items
-        assets_map = {asset['id']: asset for asset in assets_items}
-        if pk in assets_map:
-            return [assets_map.get(pk)]
-        else:
-            return []
-
-    def search_queryset(self, assets_items):
-        search = self.request.query_params.get("search")
-        if not search:
-            return assets_items
-        assets_map = {asset['id']: asset for asset in assets_items}
-        assets_ids = set(assets_map.keys())
-        assets_ids_search = Asset.objects.filter(id__in=assets_ids).filter(
-            Q(hostname__icontains=search) | Q(ip__icontains=search)
-        ).values_list('id', flat=True)
-        return [assets_map.get(asset_id) for asset_id in assets_ids_search]
-
-    def filter_queryset_by_label(self, assets_items):
-        labels_id = self.get_filter_labels_ids()
-        if not labels_id:
-            return assets_items
-
-        assets_map = {asset['id']: asset for asset in assets_items}
-        assets_matched = Asset.objects.filter(id__in=assets_map.keys())
-        for label_id in labels_id:
-            assets_matched = assets_matched.filter(labels=label_id)
-        assets_ids_matched = assets_matched.values_list('id', flat=True)
-        return [assets_map.get(asset_id) for asset_id in assets_ids_matched]
-
-    def sort_queryset(self, assets_items):
-        order_by = self.request.query_params.get('order', 'hostname')
-
-        if order_by not in ['hostname', '-hostname', 'ip', '-ip']:
-            order_by = 'hostname'
-        assets_map = {asset['id']: asset for asset in assets_items}
-        assets_ids_search = Asset.objects.filter(id__in=assets_map.keys())\
-            .order_by(order_by)\
-            .values_list('id', flat=True)
-        return [assets_map.get(asset_id) for asset_id in assets_ids_search]
-
-    def filter_queryset(self, assets_items):
-        assets_items = self.filter_queryset_by_id(assets_items)
-        assets_items = self.search_queryset(assets_items)
-        assets_items = self.filter_queryset_by_label(assets_items)
-        assets_items = self.sort_queryset(assets_items)
-        return assets_items

apps/perms/api/user_permission/mixin.py

@@ -2,12 +2,12 @@
 #
 from ..mixin import UserPermissionMixin
 from ...utils import AssetPermissionUtilV2, ParserNode
-from ...hands import Node
+from ...hands import Node, Asset
 from common.tree import TreeNodeSerializer
 
 
 class UserAssetPermissionMixin(UserPermissionMixin):
-    util = None
+    util = AssetPermissionUtilV2(None)
     tree = None
 
     def initial(self, *args, **kwargs):
@@ -41,7 +41,9 @@ class UserNodeTreeMixin:
         queryset = self.parse_nodes_to_queryset(queryset)
         return queryset
 
-    def get_serializer(self, queryset, many=True, **kwargs):
+    def get_serializer(self, queryset=None, many=True, **kwargs):
+        if queryset is None:
+            queryset = Node.objects.none()
         queryset = self.get_serializer_queryset(queryset)
         queryset.sort()
         return super().get_serializer(queryset, many=many, **kwargs)
@@ -64,7 +66,9 @@ class UserAssetTreeMixin:
         _queryset = self.parse_assets_to_queryset(queryset, None)
         return _queryset
 
-    def get_serializer(self, queryset, many=True, **kwargs):
+    def get_serializer(self, queryset=None, many=True, **kwargs):
+        if queryset is None:
+            queryset = Asset.objects.none()
         queryset = self.get_serializer_queryset(queryset)
         queryset.sort()
         return super().get_serializer(queryset, many=many, **kwargs)

apps/perms/utils/asset_permission.py

@@ -126,10 +126,10 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
         'comment', 'is_active', 'os', 'org_id'
     )
 
-    def __init__(self, obj, cache_policy='0'):
+    def __init__(self, obj=None, cache_policy='0'):
         self.object = obj
         self.cache_policy = cache_policy
-        self.obj_id = str(obj.id)
+        self.obj_id = str(obj.id) if obj else None
         self._permissions = None
         self._permissions_id = None  # 标记_permission的唯一值
         self._filter_id = 'None'  # 当通过filter更改 permission是标记
@@ -147,6 +147,8 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
     def permissions(self):
         if self._permissions:
             return self._permissions
+        if self.object is None:
+            return AssetPermission.objects.none()
         object_cls = self.object.__class__.__name__
         func = self.get_permissions_map[object_cls]
         permissions = func(self.object)

apps/settings/api.py

@@ -97,6 +97,8 @@ class LDAPUserListApi(generics.ListAPIView):
     serializer_class = LDAPUserSerializer
 
     def get_queryset(self):
+        if hasattr(self, 'swagger_fake_view'):
+            return []
         util = LDAPUtil()
         try:
             users = util.search_user_items()

apps/static/js/jumpserver.js

@@ -153,6 +153,7 @@ function activeNav() {
     } else {
         $("#" + app).addClass('active');
         $('#' + app + ' #' + resource).addClass('active');
+        $('#' + app + ' #' + resource.replace('-', '_')).addClass('active');
     }
 }
 

apps/templates/_nav.html

@@ -171,7 +171,5 @@
 
 <script>
 $(document).ready(function () {
-    var current_org = '{{ CURRENT_ORG.name }}';
-    console.log(current_org);
 })
 </script>
\ No newline at end of file

apps/terminal/serializers_v2/terminal.py

@@ -58,5 +58,5 @@ class TerminalSerializer(serializers.ModelSerializer):
 
 class TerminalRegistrationSerializer(serializers.Serializer):
     name = serializers.CharField(max_length=128)
-    comment = serializers.CharField(max_length=128)
+    comment = serializers.CharField(max_length=128, )
     service_account = ServiceAccountSerializer(read_only=True)

apps/users/api/user.py

@@ -14,7 +14,7 @@ from common.permissions import (
     IsOrgAdmin, IsCurrentUserOrReadOnly, IsOrgAdminOrAppUser,
     CanUpdateDeleteUser,
 )
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from common.utils import get_logger
 from orgs.utils import current_org
 from .. import serializers
@@ -30,7 +30,7 @@ __all__ = [
 ]
 
 
-class UserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class UserViewSet(CommonApiMixin, BulkModelViewSet):
     filter_fields = ('username', 'email', 'name', 'id')
     search_fields = filter_fields
     queryset = User.objects.exclude(role=User.ROLE_APP)