Skip to content

J
jumpserver
Commit 9dd951dd authored by ibuler's avatar ibuler
Browse files
Options

[Update] 优化可连接性

parent 2e6ba2ff
Show whitespace changes
Inline Side-by-side
Showing with 170 additions and 193 deletions
apps/assets/const.py
View file @ 9dd951dd
# -*- coding: utf-8 -*-
#
from django.utils.translation import ugettext_lazy as _
UPDATE_ASSETS_HARDWARE_TASKS = [
{
......@@ -11,7 +11,6 @@ UPDATE_ASSETS_HARDWARE_TASKS = [
}
]
ADMIN_USER_CONN_CACHE_KEY = "ADMIN_USER_CONN_{}"
TEST_ADMIN_USER_CONN_TASKS = [
{
"name": "ping",
......@@ -49,7 +48,6 @@ TEST_WINDOWS_SYSTEM_USER_CONN_TASKS = [
}
]
ASSET_USER_CONN_CACHE_KEY = 'ASSET_USER_CONN_{}'
TEST_ASSET_USER_CONN_TASKS = [
{
"name": "ping",
......@@ -74,5 +72,10 @@ TASK_OPTIONS = {
}
CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX = '_KEY_ASSET_BULK_UPDATE_ID_{}'
CONN_UNREACHABLE, CONN_REACHABLE, CONN_UNKNOWN = range(0, 3)
CONNECTIVITY_CHOICES = (
(CONN_UNREACHABLE, _("Unreachable")),
(CONN_REACHABLE, _('Reachable')),
(CONN_UNKNOWN, _("Unknown")),
)
apps/assets/models/asset.py
View file @ 9dd951dd
......@@ -6,16 +6,14 @@ import uuid
import logging
import random
from functools import reduce
from collections import defaultdict
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from django.core.validators import MinValueValidator, MaxValueValidator
from .user import AdminUser, SystemUser
from orgs.mixins import OrgModelMixin, OrgManager
from ..utils import Connectivity
__all__ = ['Asset', 'Protocol']
logger = logging.getLogger(__name__)
......@@ -230,12 +228,12 @@ class Asset(OrgModelMixin):
@property
def connectivity(self):
if not self.admin_user:
return self.UNKNOWN
return self.admin_user.get_connectivity_of(self)
return Connectivity.unknown()
return self.admin_user.get_asset_connectivity(self)
@connectivity.setter
def connectivity(self, value):
self.admin_user.set_connectivity_of(self, value)
self.admin_user.set_asset_connectivity(self, value)
def get_auth_info(self):
if not self.admin_user:
......
apps/assets/models/authbook.py
View file @ 9dd951dd
......@@ -3,12 +3,10 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from orgs.mixins import OrgManager
from ..utils import Connectivity
from .base import AssetUser
from ..const import ASSET_USER_CONN_CACHE_KEY
__all__ = ['AuthBook']
......@@ -32,6 +30,7 @@ class AuthBook(AssetUser):
backend = "db"
# 用于system user和admin_user的动态设置
_connectivity = None
CONN_CACHE_KEY = "ASSET_USER_CONN_{}"
class Meta:
verbose_name = _('AuthBook')
......@@ -65,20 +64,17 @@ class AuthBook(AssetUser):
self._set_version()
self._set_latest()
@property
def _conn_cache_key(self):
return ASSET_USER_CONN_CACHE_KEY.format(self.id)
def get_related_assets(self):
return [self.asset]
def generate_id_with_asset(self, asset):
return self.id
@property
def connectivity(self):
if self._connectivity:
return self._connectivity
value = cache.get(self._conn_cache_key, self.UNKNOWN)
return value
@connectivity.setter
def connectivity(self, value):
cache.set(self._conn_cache_key, value, 3600)
return self.get_asset_connectivity(self.asset)
@property
def keyword(self):
......
apps/assets/models/base.py
View file @ 9dd951dd
......@@ -14,8 +14,11 @@ from common.utils import (
get_signer, ssh_key_string_to_obj, ssh_key_gen, get_logger
)
from common.validators import alphanumeric
from common import fields
from orgs.mixins import OrgModelMixin
from .utils import private_key_validator
from ..utils import Connectivity
from .. import const
signer = get_signer()
......@@ -26,7 +29,7 @@ class AssetUser(OrgModelMixin):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
name = models.CharField(max_length=128, verbose_name=_('Name'))
username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric])
_password = models.CharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
_password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
_private_key = models.TextField(max_length=4096, blank=True, null=True, verbose_name=_('SSH private key'), validators=[private_key_validator, ])
_public_key = models.TextField(max_length=4096, blank=True, verbose_name=_('SSH public key'))
comment = models.TextField(blank=True, verbose_name=_('Comment'))
......@@ -34,13 +37,8 @@ class AssetUser(OrgModelMixin):
date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by'))
UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3)
CONNECTIVITY_CHOICES = (
(UNREACHABLE, _("Unreachable")),
(REACHABLE, _('Reachable')),
(UNKNOWN, _("Unknown")),
)
CONNECTIVITY_CACHE_KEY = "CONNECTIVITY_{}"
CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_ASSET_CONNECTIVITY_{}"
_prefer = "system_user"
@property
......@@ -109,6 +107,10 @@ class AssetUser(OrgModelMixin):
pass
return None
def get_related_assets(self):
assets = self.assets.all()
return assets
def set_auth(self, password=None, private_key=None, public_key=None):
update_fields = []
if password:
......@@ -124,17 +126,52 @@ class AssetUser(OrgModelMixin):
if update_fields:
self.save(update_fields=update_fields)
def get_auth(self, asset=None):
pass
def set_connectivity(self, summary):
unreachable = summary.get('dark', {}).keys()
reachable = summary.get('contacted', {}).keys()
def get_connectivity_of(self, asset):
for asset in self.get_related_assets():
if asset.hostname in unreachable:
self.set_asset_connectivity(asset, Connectivity.unreachable())
elif asset.hostname in reachable:
self.set_asset_connectivity(asset, Connectivity.reachable())
else:
self.set_asset_connectivity(asset, Connectivity.unknown())
@property
def connectivity(self):
assets = self.get_related_assets()
data = {
'unreachable': [],
'reachable': [],
'unknown': [],
}
for asset in assets:
connectivity = self.get_asset_connectivity(asset)
if connectivity.is_reachable():
data["reachable"].append(asset.hostname)
elif connectivity.is_unreachable():
data["unreachable"].append(asset.hostname)
else:
data["unknown"].append(asset.hostname)
return data
@property
def connectivity_amount(self):
return {k: len(v) for k, v in self.connectivity.items()}
@property
def assets_amount(self):
return self.get_related_assets().count()
def get_asset_connectivity(self, asset):
i = self.generate_id_with_asset(asset)
key = self.CONNECTIVITY_CACHE_KEY.format(i)
return cache.get(key)
key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
return cache.get(key, const.CONN_UNKNOWN)
def set_connectivity_of(self, asset, c):
def set_asset_connectivity(self, asset, c):
i = self.generate_id_with_asset(asset)
key = self.CONNECTIVITY_CACHE_KEY.format(i)
key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
cache.set(key, c, 3600)
def load_specific_asset_auth(self, asset):
......@@ -168,9 +205,10 @@ class AssetUser(OrgModelMixin):
private_key, public_key = ssh_key_gen(
username=self.username
)
self.set_auth(password=password,
private_key=private_key,
public_key=public_key)
self.set_auth(
password=password, private_key=private_key,
public_key=public_key
)
def auto_gen_auth_password(self):
password = str(uuid.uuid4())
......@@ -187,24 +225,18 @@ class AssetUser(OrgModelMixin):
}
def generate_id_with_asset(self, asset):
id_ = '{}_{}'.format(asset.id, self.id)
id_ = uuid.UUID(md5(id_.encode()).hexdigest())
return id_
i = '{}_{}'.format(asset.id, self.id)
i = uuid.UUID(md5(i.encode()).hexdigest())
return i
def construct_to_authbook(self, asset):
from . import AuthBook
fields = [
'name', 'username', 'comment', 'org_id',
'_password', '_private_key', '_public_key',
'date_created', 'date_updated', 'created_by'
]
id_ = self.generate_id_with_asset(asset)
obj = AuthBook(id=id_, asset=asset, version=0, is_latest=True)
obj._connectivity = self.get_connectivity_of(asset)
for field in fields:
value = getattr(self, field)
setattr(obj, field, value)
return obj
i = self.generate_id_with_asset(asset)
self.id = i
self.asset = asset
self.version = 0
self.is_latest = True
return self
class Meta:
abstract = True
apps/assets/models/user.py
View file @ 9dd951dd
......@@ -4,13 +4,11 @@
import logging
from django.core.cache import cache
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.validators import MinValueValidator, MaxValueValidator
from common.utils import get_signer
from ..const import SYSTEM_USER_CONN_CACHE_KEY
from .base import AssetUser
......@@ -31,7 +29,7 @@ class AdminUser(AssetUser):
become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4)
become_user = models.CharField(default='root', max_length=64)
_become_pass = models.CharField(default='', max_length=128)
CONNECTIVE_CACHE_KEY = '_JMS_ADMIN_USER_CONNECTIVE_{}'
CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}'
_prefer = "admin_user"
def __str__(self):
......@@ -61,31 +59,6 @@ class AdminUser(AssetUser):
info = None
return info
def get_related_assets(self):
assets = self.assets.all()
return assets
@property
def assets_amount(self):
return self.get_related_assets().count()
@property
def connectivity(self):
from .asset import Asset
assets = self.get_related_assets().values_list('id', 'hostname', flat=True)
data = {
'unreachable': [],
'reachable': [],
}
for asset_id, hostname in assets:
key = Asset.CONNECTIVITY_CACHE_KEY.format(str(self.id))
value = cache.get(key, Asset.UNKNOWN)
if value == Asset.REACHABLE:
data['reachable'].append(hostname)
elif value == Asset.UNREACHABLE:
data['unreachable'].append(hostname)
return data
class Meta:
ordering = ['name']
unique_together = [('name', 'org_id')]
......@@ -141,9 +114,6 @@ class SystemUser(AssetUser):
login_mode = models.CharField(choices=LOGIN_MODE_CHOICES, default=LOGIN_AUTO, max_length=10, verbose_name=_('Login mode'))
cmd_filters = models.ManyToManyField('CommandFilter', related_name='system_users', verbose_name=_("Command filter"), blank=True)
SYSTEM_USER_CACHE_KEY = "__SYSTEM_USER_CACHED_{}"
CONNECTIVE_CACHE_KEY = '_JMS_SYSTEM_USER_CONNECTIVE_{}'
def __str__(self):
return '{0.name}({0.username})'.format(self)
......@@ -157,49 +127,6 @@ class SystemUser(AssetUser):
'auto_push': self.auto_push,
}
def get_related_assets(self):
assets = set(self.assets.all())
return assets
@property
def connectivity(self):
cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id))
value = cache.get(cache_key, None)
if not value or 'unreachable' not in value:
return {'unreachable': [], 'reachable': []}
else:
return value
@connectivity.setter
def connectivity(self, value):
data = self.connectivity
unreachable = data['unreachable']
reachable = data['reachable']
assets = {asset.hostname: asset for asset in self.assets.all()}
for host in value.get('dark', {}).keys():
if host not in unreachable:
unreachable.append(host)
if host in reachable:
reachable.remove(host)
self.set_connectivity_of(assets.get(host), self.UNREACHABLE)
for host in value.get('contacted'):
if host not in reachable:
reachable.append(host)
if host in unreachable:
unreachable.remove(host)
self.set_connectivity_of(assets.get(host), self.REACHABLE)
cache_key = self.CONNECTIVE_CACHE_KEY.format(str(self.id))
cache.set(cache_key, data, 3600)
@property
def assets_unreachable(self):
return self.connectivity.get('unreachable')
@property
def assets_reachable(self):
return self.connectivity.get('reachable')
@property
def login_mode_display(self):
return self.get_login_mode_display()
......@@ -210,12 +137,6 @@ class SystemUser(AssetUser):
else:
return False
def set_cache(self):
cache.set(self.SYSTEM_USER_CACHE_KEY.format(self.id), self, 3600)
def expire_cache(self):
cache.delete(self.SYSTEM_USER_CACHE_KEY.format(self.id))
@property
def cmd_filter_rules(self):
from .cmd_filter import CommandFilterRule
......@@ -233,18 +154,6 @@ class SystemUser(AssetUser):
return False, matched_cmd
return True, None
@classmethod
def get_system_user_by_id_or_cached(cls, sid):
cached = cache.get(cls.SYSTEM_USER_CACHE_KEY.format(sid))
if cached:
return cached
try:
system_user = cls.objects.get(id=sid)
system_user.set_cache()
return system_user
except cls.DoesNotExist:
return None
class Meta:
ordering = ['name']
unique_together = [('name', 'org_id')]
......
apps/assets/serializers/admin_user.py
View file @ 9dd951dd
# -*- coding: utf-8 -*-
#
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from common.serializers import AdaptedBulkListSerializer
from ..models import Node, AdminUser
from ..const import ADMIN_USER_CONN_CACHE_KEY
from orgs.mixins import BulkOrgResourceModelSerializer
from .base import AuthSerializer
......@@ -20,9 +18,6 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer):
password = serializers.CharField(
required=False, write_only=True, label=_('Password')
)
unreachable_amount = serializers.SerializerMethodField(label=_('Unreachable'))
assets_amount = serializers.SerializerMethodField(label=_('Asset'))
reachable_amount = serializers.SerializerMethodField(label=_('Reachable'))
class Meta:
list_serializer_class = AdaptedBulkListSerializer
......@@ -38,33 +33,14 @@ class AdminUserSerializer(BulkOrgResourceModelSerializer):
'date_created': {'label': _('Date created')},
'date_updated': {'label': _('Date updated')},
'become': {'read_only': True}, 'become_method': {'read_only': True},
'become_user': {'read_only': True}, 'created_by': {'read_only': True}
'become_user': {'read_only': True}, 'created_by': {'read_only': True},
'assets_amount': {'label', _('Asset')}
}
def get_field_names(self, declared_fields, info):
fields = super().get_field_names(declared_fields, info)
return [f for f in fields if not f.startswith('_')]
@staticmethod
def get_unreachable_amount(obj):
data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name))
if data:
return len(data.get('dark'))
else:
return 0
@staticmethod
def get_reachable_amount(obj):
data = cache.get(ADMIN_USER_CONN_CACHE_KEY.format(obj.name))
if data:
return len(data.get('contacted'))
else:
return 0
@staticmethod
def get_assets_amount(obj):
return obj.assets_amount
class AdminUserAuthSerializer(AuthSerializer):
......
apps/assets/tasks.py
View file @ 9dd951dd
......@@ -15,8 +15,9 @@ from ops.celery.decorator import (
register_as_period_task, after_app_shutdown_clean_periodic
)
from .models import SystemUser, AdminUser, Asset
from .models import SystemUser, AdminUser
from . import const
from .utils import Connectivity
FORKS = 10
......@@ -208,7 +209,7 @@ def test_asset_connectivity_util(assets, task_name=None):
created_by=created_by,
)
result = task.run()
summary = result[1]
summary = result.get("summary", {})
success = summary.get('success', False)
contacted = summary.get('contacted', {})
dark = summary.get('dark', {})
......@@ -218,13 +219,12 @@ def test_asset_connectivity_util(assets, task_name=None):
results_summary['dark'].update(dark)
for asset in assets:
if asset.hostname in results_summary.get('dark', {}):
asset.connectivity = asset.UNREACHABLE
elif asset.hostname in results_summary.get('contacted', []):
asset.connectivity = asset.REACHABLE
if asset.hostname in results_summary.get('dark', {}).keys():
asset.connectivity = Connectivity.unreachable()
elif asset.hostname in results_summary.get('contacted', {}).keys():
asset.connectivity = Connectivity.reachable()
else:
asset.connectivity = asset.UNKNOWN
asset.connectivity = Connectivity.unknown()
return results_summary
......@@ -286,10 +286,6 @@ def test_admin_user_connectivity_manual(admin_user):
## System user connective ##
@shared_task
def set_system_user_connectivity_info(system_user, summary):
system_user.connectivity = summary
@shared_task
def test_system_user_connectivity_util(system_user, assets, task_name):
......@@ -346,7 +342,7 @@ def test_system_user_connectivity_util(system_user, assets, task_name):
results_summary['contacted'].update(contacted)
results_summary['dark'].update(dark)
set_system_user_connectivity_info(system_user, results_summary)
system_user.set_connectivity(results_summary)
return results_summary
......@@ -584,6 +580,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False)
"""
:param asset_user: <AuthBook>对象
:param task_name:
:param run_as_admin:
:return:
"""
from ops.utils import update_or_create_ansible_task
......@@ -607,7 +604,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False)
kwargs["run_as"] = asset_user.username
task, created = update_or_create_ansible_task(*args, **kwargs)
result = task.run()
set_asset_user_connectivity_info(asset_user, result)
asset_user.set_connectivity(result.get("summary", {}))
@shared_task
......
apps/assets/utils.py
View file @ 9dd951dd
# ~*~ coding: utf-8 ~*~
#
import os
import paramiko
from paramiko.ssh_exception import SSHException
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from django.utils import timezone
from common.utils import get_object_or_none
from .models import Asset, SystemUser, Label
......@@ -45,3 +45,62 @@ class LabelFilter:
for kwargs in conditions:
queryset = queryset.filter(**kwargs)
return queryset
class Connectivity:
UNREACHABLE, REACHABLE, UNKNOWN = range(0, 3)
CONNECTIVITY_CHOICES = (
(UNREACHABLE, _("Unreachable")),
(REACHABLE, _('Reachable')),
(UNKNOWN, _("Unknown")),
)
value = UNKNOWN
datetime = timezone.now()
def __init__(self, value, datetime):
self.value = value
self.datetime = datetime
def display(self):
return dict(self.__class__.CONNECTIVITY_CHOICES).get(self.value)
def is_reachable(self):
return self.value == self.REACHABLE
def is_unreachable(self):
return self.value == self.UNREACHABLE
def is_unknown(self):
return self.value == self.UNKNOWN
@classmethod
def unreachable(cls):
return cls(cls.UNREACHABLE, timezone.now())
@classmethod
def reachable(cls):
return cls(cls.REACHABLE, timezone.now())
@classmethod
def unknown(cls):
return cls(cls.UNKNOWN, timezone.now())
@classmethod
def set(cls, key, value, ttl=0):
cache.set(key, value, ttl)
@classmethod
def get(cls, key):
return cache.get(key, cls.UNKNOWN)
@classmethod
def set_unreachable(cls, key, ttl=0):
cls.set(key, cls.unreachable(), ttl)
@classmethod
def set_reachable(cls, key, ttl=0):
cls.set(key, cls.reachable(), ttl)
def __eq__(self, other):
return self.value == other.value
apps/common/utils/common.py
View file @ 9dd951dd
......@@ -293,3 +293,11 @@ class LocalProxy(object):
__rdivmod__ = lambda x, o: x._get_current_object().__rdivmod__(o)
__copy__ = lambda x: copy.copy(x._get_current_object())
__deepcopy__ = lambda x, memo: copy.deepcopy(x._get_current_object(), memo)
def random_string(length):
import string
import random
charset = string.ascii_letters + string.digits
s = [random.choice(charset) for i in range(length)]
return ''.join(s)
apps/terminal/api/session.py
View file @ 9dd951dd
......@@ -47,8 +47,7 @@ class SessionViewSet(BulkModelViewSet):
sid = serializer.validated_data["system_user"]
# guacamole提交的是id
if is_uuid(sid):
_system_user = SystemUser.get_system_user_by_id_or_cached(sid)
if _system_user:
_system_user = get_object_or_404(SystemUser, id=sid)
serializer.validated_data["system_user"] = _system_user.name
return super().perform_create(serializer)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment