Commit 55c95c58 authored by 老广's avatar 老广 Committed by BaiJiangJie

Add new model to operate log (#3546)

* [Update] 添加一下model到operate log, [platform,remoteapppermission,changeauthplan,gatherusertask]

* [Bugfix] 修改了返回platform的几个位置,修改了command execution的url

* [Update] 优化ops task表结构,避免列表页查询几十次sql, 优化了基础的encryptjsonfield

* [Update] 修改adhoc 返回的become字段,避免密码泄露

* [Update] 修改变量名称
parent 907703d9
......@@ -40,6 +40,9 @@ class Migration(migrations.Migration):
('internal', models.BooleanField(default=False, verbose_name='Internal')),
('comment', models.TextField(blank=True, null=True, verbose_name='Comment')),
],
options={
'verbose_name': 'Platform'
}
),
migrations.RunPython(create_internal_platform)
]
......@@ -11,14 +11,13 @@ from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from common.utils import (
get_signer, ssh_key_string_to_obj, ssh_key_gen, get_logger
signer, ssh_key_string_to_obj, ssh_key_gen, get_logger
)
from common.validators import alphanumeric
from common import fields
from orgs.mixins.models import OrgModelMixin
from .utils import private_key_validator, Connectivity
signer = get_signer()
logger = get_logger(__file__)
......
......@@ -10,14 +10,13 @@ from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.validators import MinValueValidator, MaxValueValidator
from common.utils import get_signer
from common.utils import signer
from .base import AssetUser
from .asset import Asset
__all__ = ['AdminUser', 'SystemUser']
logger = logging.getLogger(__name__)
signer = get_signer()
class AdminUser(AssetUser):
......
......@@ -27,6 +27,7 @@ MODELS_NEED_RECORD = (
'User', 'UserGroup', 'Asset', 'Node', 'AdminUser', 'SystemUser',
'Domain', 'Gateway', 'Organization', 'AssetPermission', 'CommandFilter',
'CommandFilterRule', 'License', 'Setting', 'Account', 'SyncInstanceTask',
'Platform', 'RemoteAppPermission', 'ChangeAuthPlan', 'GatherUserTask',
)
......
......@@ -6,9 +6,8 @@ from django import forms
from django.utils import six
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext as _
from ..utils import get_signer
from ..utils import signer
signer = get_signer()
__all__ = [
'FormDictField', 'FormEncryptCharField', 'FormEncryptDictField',
......
......@@ -4,7 +4,7 @@ import json
from django.db import models
from django.utils.translation import ugettext_lazy as _
from ..utils import get_signer
from ..utils import signer
__all__ = [
......@@ -12,8 +12,8 @@ __all__ = [
'JsonCharField', 'JsonTextField', 'JsonListCharField', 'JsonListTextField',
'JsonDictCharField', 'JsonDictTextField', 'EncryptCharField',
'EncryptTextField', 'EncryptMixin', 'EncryptJsonDictTextField',
'EncryptJsonDictCharField',
]
signer = get_signer()
class JsonMixin:
......@@ -108,14 +108,24 @@ class JsonTextField(JsonMixin, models.TextField):
class EncryptMixin:
"""
EncryptMixin要放在最前面
"""
def from_db_value(self, value, expression, connection, context):
if value is not None:
return signer.unsign(value)
return None
if value is None:
return value
value = signer.unsign(value)
sp = super()
if hasattr(sp, 'from_db_value'):
return sp.from_db_value(value, expression, connection, context)
return value
def get_prep_value(self, value):
if value is None:
return value
sp = super()
if hasattr(sp, 'get_prep_value'):
value = sp.get_prep_value(value)
return signer.sign(value)
......@@ -150,3 +160,6 @@ class EncryptJsonDictTextField(EncryptMixin, JsonDictTextField):
pass
class EncryptJsonDictCharField(EncryptMixin, JsonDictCharField):
pass
......@@ -2,11 +2,10 @@ from django.test import TestCase
# Create your tests here.
from .utils import random_string, get_signer
from .utils import random_string, signer
def test_signer_len():
signer = get_signer()
results = {}
for i in range(1, 4096):
s = random_string(i)
......
......@@ -184,8 +184,11 @@ def encrypt_password(password, salt=None):
def get_signer():
signer = Signer(settings.SECRET_KEY)
return signer
s = Signer(settings.SECRET_KEY)
return s
signer = get_signer()
def ensure_last_char_is_ascii(data):
......
......@@ -105,7 +105,7 @@ CELERY_TASK_SERIALIZER = 'pickle'
CELERY_RESULT_SERIALIZER = 'pickle'
CELERY_RESULT_BACKEND = CELERY_BROKER_URL
CELERY_ACCEPT_CONTENT = ['json', 'pickle']
CELERY_RESULT_EXPIRES = 3600
CELERY_RESULT_EXPIRES = 600
# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
# CELERY_WORKER_LOG_FORMAT = '%(message)s'
# CELERY_WORKER_TASK_LOG_FORMAT = '%(task_id)s %(task_name)s %(message)s'
......
......@@ -4,6 +4,7 @@
from django.shortcuts import get_object_or_404
from rest_framework import viewsets, generics
from rest_framework.views import Response
from django.db.models import Count, Q
from common.permissions import IsOrgAdmin
from common.serializers import CeleryTaskSerializer
......@@ -31,6 +32,7 @@ class TaskViewSet(viewsets.ModelViewSet):
queryset = queryset.filter(created_by=current_org.id)
else:
queryset = queryset.filter(created_by='')
queryset = queryset.select_related('latest_history')
return queryset
......
......@@ -33,11 +33,14 @@ def get_after_app_ready_tasks():
def register_as_period_task(
crontab=None, interval=None, name=None,
args=(), kwargs=None,
description=''):
"""
Warning: Task must be have not any args and kwargs
:param crontab: "* * * * *"
:param interval: 60*60*60
:param args: ()
:param kwargs: {}
:param description: "
:param name: ""
:return:
......@@ -58,7 +61,8 @@ def register_as_period_task(
'task': task,
'interval': interval,
'crontab': crontab,
'args': (),
'args': args,
'kwargs': kwargs if kwargs else {},
'enabled': True,
'description': description
}
......
......@@ -74,8 +74,6 @@ def create_or_update_celery_periodic_tasks(tasks):
kwargs=json.dumps(detail.get('kwargs', {})),
description=detail.get('description') or ''
)
print(defaults)
task = PeriodicTask.objects.update_or_create(
defaults=defaults, name=name,
)
......@@ -101,4 +99,3 @@ def get_celery_task_log_path(task_id):
path = os.path.join(settings.CELERY_LOG_DIR, rel_path)
os.makedirs(os.path.dirname(path), exist_ok=True)
return path
# Generated by Django 2.2.7 on 2019-12-17 09:13
from django.db import migrations, models
import django.db.models.deletion
from django.core.exceptions import ObjectDoesNotExist
def migrate_task_data(apps, schema_editor):
task_model = apps.get_model("ops", "Task")
db_alias = schema_editor.connection.alias
tasks = task_model.objects.using(db_alias).all()
for task in tasks:
try:
latest_history = task.history.latest()
except ObjectDoesNotExist:
latest_history = None
try:
latest_adhoc = task.adhoc.latest()
except ObjectDoesNotExist:
latest_adhoc = None
if latest_history and latest_history.adhoc:
latest_history.hosts_amount = latest_history.adhoc.hosts.count()
latest_history.save()
total_run_amount = task.history.all().count()
success_run_amount = task.history.filter(is_success=True).count()
task.latest_history = latest_history
task.latest_adhoc = latest_adhoc
task.total_run_amount = total_run_amount
task.success_run_amount = success_run_amount
task.save()
class Migration(migrations.Migration):
dependencies = [
('ops', '0008_auto_20190919_2100'),
]
operations = [
migrations.AddField(
model_name='task',
name='latest_adhoc',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='task_latest', to='ops.AdHoc'),
),
migrations.AddField(
model_name='task',
name='latest_history',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='task_latest', to='ops.AdHocRunHistory'),
),
migrations.AddField(
model_name='task',
name='success_run_amount',
field=models.IntegerField(default=0),
),
migrations.AddField(
model_name='task',
name='total_run_amount',
field=models.IntegerField(default=0),
),
migrations.AddField(
model_name='adhocrunhistory',
name='hosts_amount',
field=models.IntegerField(default=0, verbose_name='Host amount'),
),
migrations.AddField(
model_name='adhocrunhistory',
name='task_display',
field=models.CharField(blank=True, default='', max_length=128,
verbose_name='Task display'),
),
migrations.RunPython(migrate_task_data),
]
# Generated by Django 2.2.7 on 2019-12-17 09:58
import common.fields.model
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('ops', '0009_auto_20191217_1713'),
]
operations = [
migrations.RemoveField(
model_name='adhoc',
name='_hosts',
),
migrations.AlterField(
model_name='adhoc',
name='_become',
field=common.fields.model.EncryptJsonDictCharField(blank=True, default='', max_length=1024, verbose_name='Become'),
),
migrations.AlterField(
model_name='adhoc',
name='_options',
field=common.fields.model.JsonDictCharField(default='', max_length=1024, verbose_name='Options'),
),
migrations.AlterField(
model_name='adhoc',
name='_tasks',
field=common.fields.model.JsonListTextField(verbose_name='Tasks'),
),
migrations.RenameField(
model_name='adhoc',
old_name='_become',
new_name='become',
),
migrations.RenameField(
model_name='adhoc',
old_name='_options',
new_name='options',
),
migrations.RenameField(
model_name='adhoc',
old_name='_tasks',
new_name='tasks',
),
migrations.AlterField(
model_name='adhocrunhistory',
name='_result',
field=common.fields.model.JsonDictTextField(blank=True, null=True, verbose_name='Adhoc raw result'),
),
migrations.AlterField(
model_name='adhocrunhistory',
name='_summary',
field=common.fields.model.JsonDictTextField(blank=True, null=True, verbose_name='Adhoc result summary'),
),
migrations.RenameField(
model_name='adhocrunhistory',
old_name='_result',
new_name='result',
),
migrations.RenameField(
model_name='adhocrunhistory',
old_name='_summary',
new_name='summary',
),
]
This diff is collapsed.
......@@ -6,59 +6,74 @@ from django.shortcuts import reverse
from ..models import Task, AdHoc, AdHocRunHistory, CommandExecution
class TaskSerializer(serializers.ModelSerializer):
class Meta:
model = Task
fields = [
'id', 'name', 'interval', 'crontab', 'is_periodic',
'is_deleted', 'comment', 'created_by', 'date_created',
'versions', 'is_success', 'timedelta', 'assets_amount',
'date_updated', 'history_summary',
]
class AdHocSerializer(serializers.ModelSerializer):
class Meta:
model = AdHoc
exclude = ('_tasks', '_options', '_hosts', '_become')
def get_field_names(self, declared_fields, info):
fields = super().get_field_names(declared_fields, info)
fields.extend(['tasks', 'options', 'hosts', 'become', 'short_id'])
return fields
class AdHocRunHistorySerializer(serializers.ModelSerializer):
task = serializers.SerializerMethodField()
adhoc_short_id = serializers.SerializerMethodField()
stat = serializers.SerializerMethodField()
class Meta:
model = AdHocRunHistory
exclude = ('_result', '_summary')
@staticmethod
def get_adhoc_short_id(obj):
return obj.adhoc.short_id
fields = '__all__'
@staticmethod
def get_task(obj):
return obj.adhoc.task.id
return obj.task.id
@staticmethod
def get_stat(obj):
return {
"total": obj.adhoc.hosts.count(),
"total": obj.hosts_amount,
"success": len(obj.summary.get("contacted", [])),
"failed": len(obj.summary.get("dark", [])),
}
def get_field_names(self, declared_fields, info):
fields = super().get_field_names(declared_fields, info)
fields.extend(['summary', 'short_id'])
fields.extend(['short_id', 'adhoc_short_id'])
return fields
class AdHocRunHistoryExcludeResultSerializer(AdHocRunHistorySerializer):
def get_field_names(self, declared_fields, info):
fields = super().get_field_names(declared_fields, info)
fields = [i for i in fields if i not in ['result', 'summary']]
return fields
class TaskSerializer(serializers.ModelSerializer):
latest_history = AdHocRunHistoryExcludeResultSerializer(read_only=True)
class Meta:
model = Task
fields = [
'id', 'name', 'interval', 'crontab', 'is_periodic',
'is_deleted', 'comment', 'created_by', 'date_created',
'date_updated', 'latest_history',
]
read_only_fields = [
'is_deleted', 'created_by', 'date_created', 'date_updated',
'latest_adhoc', 'latest_history', 'total_run_amount',
'success_run_amount',
]
class AdHocSerializer(serializers.ModelSerializer):
become_display = serializers.ReadOnlyField()
class Meta:
model = AdHoc
fields = [
"id", "task", 'tasks', "pattern", "options",
"hosts", "run_as_admin", "run_as", "become",
"created_by", "date_created", "short_id",
"become_display",
]
read_only_fields = [
'created_by', 'date_created'
]
extra_kwargs = {
"become": {'write_only': True}
}
class CommandExecutionSerializer(serializers.ModelSerializer):
result = serializers.JSONField(read_only=True)
log_url = serializers.SerializerMethodField()
......
......@@ -78,7 +78,7 @@
{% endif %}
<tr>
<td>{% trans 'Become' %}</td>
<td><b>{{ object.become.user }}</b></td>
<td><b>{{ object.become_display }}</b></td>
</tr>
<tr>
<td>{% trans 'Created by' %}</td>
......
......@@ -103,7 +103,7 @@ $(document).ready(function () {
if (!cellData) {
$(td).html("")
} else {
$(td).html(cellData.user)
$(td).html(cellData)
}
}},
{targets: 6, createdCell: function (td, cellData) {
......@@ -118,8 +118,12 @@ $(document).ready(function () {
}}
],
ajax_url: '{% url "api-ops:adhoc-list" %}?task={{ object.pk }}',
columns: [{data: function(){return ""}}, {data: "short_id" }, {data: "hosts", orderable:false}, {data: "pattern", orderable:false},
{data: "run_as"}, {data: "become", orderable:false}, {data: "date_created"}, {data: "id", orderable:false}]
columns: [
{data: function(){return ""}}, {data: "short_id"},
{data: "hosts", orderable:false}, {data: "pattern", orderable:false},
{data: "run_as"}, {data: "become_display", orderable:false},
{data: "date_created"}, {data: "id", orderable:false}
]
};
jumpserver.initDataTable(options);
}).on('click', '.celery-task-log', function () {
......
......@@ -80,11 +80,23 @@
</tr>
<tr>
<td>{% trans 'Is finished' %}:</td>
<td><b>{{ object.latest_history.is_finished|yesno:"Yes,No,Unkown" }}</b></td>
<td><b>
{% if object.latest_history.is_finished %}
{% trans 'Yes' %}
{% else %}
{% trans 'No' %}
{% endif %}
</b></td>
</tr>
<tr>
<td>{% trans 'Is success ' %}:</td>
<td><b>{{ object.latest_history.is_success|yesno:"Yes,No,Unkown" }}</b></td>
<td><b>
{% if object.latest_history.is_success %}
{% trans 'Yes' %}
{% else %}
{% trans 'No' %}
{% endif %}
</b></td>
</tr>
<tr>
<td>{% trans 'Contents' %}:</td>
......
......@@ -10,7 +10,6 @@
</th>
<th class="text-left">{% trans 'Name' %}</th>
<th class="text-center">{% trans 'Run times' %}</th>
<th class="text-center">{% trans 'Versions' %}</th>
<th class="text-center">{% trans 'Hosts' %}</th>
<th class="text-center">{% trans 'Success' %}</th>
<th class="text-center">{% trans 'Date' %}</th>
......@@ -36,34 +35,40 @@ $(document).ready(function () {
$(td).html(innerHtml);
}},
{targets: 2, createdCell: function (td, cellData) {
var summary = cellData ? cellData.stat : {failed: 0, success: 0, total: 0};
var innerHtml = '<span class="text-danger">failed</span>/<span class="text-navy">success</span>/total';
if (cellData) {
innerHtml = innerHtml.replace('failed', cellData.failed)
.replace('success', cellData.success)
.replace('total', cellData.total);
$(td).html(innerHtml);
} else {
$(td).html('')
}
innerHtml = innerHtml.replace('failed', summary.failed)
.replace('success', summary.success)
.replace('total', summary.total);
$(td).html(innerHtml);
}},
{targets: 5, createdCell: function (td, cellData) {
{targets: 3, createdCell: function (td, cellData) {
var hostsAmount = cellData ? cellData.hosts_amount : 0;
$(td).html(hostsAmount)
}},
{targets: 4, createdCell: function (td, cellData) {
var successBtn = '<i class="fa fa-check text-navy"></i>';
var failedBtn = '<i class="fa fa-times text-danger"></i>';
if (cellData) {
if (cellData && cellData.is_success) {
$(td).html(successBtn)
} else {
$(td).html(failedBtn)
}
}},
{targets: 6, createdCell: function (td, cellData) {
$(td).html(toSafeLocalDateStr(cellData));
{targets: 5, createdCell: function (td, cellData) {
if (cellData) {
$(td).html(toSafeLocalDateStr(cellData.date_start));
} else {
$(td).html('');
}
}},
{targets: 7, createdCell: function (td, cellData) {
{targets: 6, createdCell: function (td, cellData) {
cellData = cellData ? cellData.timedelta : 0;
var delta = readableSecond(cellData);
$(td).html(delta);
}},
{
targets: 8,
targets: 7,
createdCell: function (td, cellData, rowData) {
var runBtn = '<a data-uid="ID" class="btn btn-xs btn-primary btn-run">{% trans "Run" %}</a> '.replace('ID', cellData);
var delBtn = '<a data-uid="ID" class="btn btn-xs btn-danger btn-del">{% trans "Delete" %}</a>'.replace('ID', cellData);
......@@ -73,10 +78,11 @@ $(document).ready(function () {
],
ajax_url: '{% url "api-ops:task-list" %}',
columns: [
{data: "id"}, {data: "name", className: "text-left"}, {data: "history_summary", orderable: false},
{data: "versions", orderable: false}, {data: "assets_amount", orderable: false},
{data: "is_success", orderable: false}, {data: "date_updated"},
{data: "timedelta", orderable:false}, {data: "id", orderable: false},
{data: "id"}, {data: "name", className: "text-left"},
{data: "latest_history", orderable: false},
{data: "latest_history", orderable: false},
{data: "latest_history", orderable: false}, {data: "latest_history"},
{data: "latest_history", orderable:false}, {data: "id", orderable: false},
],
order: [],
op_html: $('#actions').html()
......
......@@ -20,5 +20,5 @@ urlpatterns = [
path('celery/task/<uuid:pk>/log/', views.CeleryTaskLogView.as_view(), name='celery-task-log'),
path('command-execution/', views.CommandExecutionListView.as_view(), name='command-execution-list'),
path('command-execution/start/', views.CommandExecutionStartView.as_view(), name='command-execution-start'),
path('command-execution/create/', views.CommandExecutionCreateView.as_view(), name='command-execution-create'),
]
......@@ -15,7 +15,7 @@ from ..forms import CommandExecutionForm
__all__ = [
'CommandExecutionListView', 'CommandExecutionStartView'
'CommandExecutionListView', 'CommandExecutionCreateView'
]
......@@ -55,7 +55,7 @@ class CommandExecutionListView(PermissionsMixin, DatetimeSearchMixin, ListView):
return super().get_context_data(**kwargs)
class CommandExecutionStartView(PermissionsMixin, TemplateView):
class CommandExecutionCreateView(PermissionsMixin, TemplateView):
template_name = 'ops/command_execution_create.html'
form_class = CommandExecutionForm
permission_classes = [IsValidUser]
......
......@@ -46,6 +46,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer):
被授权资产的数据结构
"""
protocols = ProtocolsField(label=_('Protocols'), required=False, read_only=True)
platform = serializers.ReadOnlyField(source='platform_base')
class Meta:
model = Asset
......
......@@ -437,7 +437,7 @@ def sort_assets(assets, order_by='hostname', reverse=False):
class ParserNode:
nodes_only_fields = ("key", "value", "id")
assets_only_fields = ("platform", "hostname", "id", "ip", "protocols")
assets_only_fields = ("hostname", "id", "ip", "protocols", "org_id")
system_users_only_fields = (
"id", "name", "username", "protocol", "priority", "login_mode",
)
......@@ -445,7 +445,6 @@ class ParserNode:
@staticmethod
def parse_node_to_tree_node(node):
name = '{} ({})'.format(node.value, node.assets_amount)
# name = node.value
data = {
'id': node.key,
'name': name,
......@@ -468,7 +467,7 @@ class ParserNode:
@staticmethod
def parse_asset_to_tree_node(node, asset):
icon_skin = 'file'
platform = asset.platform.lower()
platform = asset.platform_base.lower()
if platform == 'windows':
icon_skin = 'windows'
elif platform == 'linux':
......@@ -489,8 +488,8 @@ class ParserNode:
'hostname': asset.hostname,
'ip': asset.ip,
'protocols': asset.protocols_as_list,
'platform': asset.platform,
"org_name": asset.org_name,
'platform': asset.platform_base,
'org_name': asset.org_name,
},
}
}
......
......@@ -5,9 +5,7 @@ from django.db.utils import ProgrammingError, OperationalError
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from common.utils import get_signer
signer = get_signer()
from common.utils import signer
class SettingQuerySet(models.QuerySet):
......
......@@ -44,7 +44,6 @@ function toggleSpliter() {
showTree = 1;
});
} else {
console.log("hide")
$("#split-right").attr("class", "col-sm-9");
$("#toggle-icon").attr("class", "fa fa-angle-left fa-x");
$("#split-left").show(500);
......
......@@ -116,7 +116,7 @@
</a>
<ul class="nav nav-second-level">
<li id="task"><a href="{% url 'ops:task-list' %}">{% trans 'Task list' %}</a></li>
<li id="command-execution"><a href="{% url 'ops:command-execution-start' %}">{% trans 'Batch command' %}</a></li>
<li id="command-execution"><a href="{% url 'ops:command-execution-create' %}">{% trans 'Batch command' %}</a></li>
{% if request.user.is_superuser %}
<li><a href="{% url 'flower-view' path='' %}" target="_blank" >{% trans 'Task monitor' %}</a></li>
{% endif %}
......
......@@ -22,7 +22,7 @@
{% if SECURITY_COMMAND_EXECUTION %}
<li id="ops">
<a href="{% url 'ops:command-execution-start' %}">
<a href="{% url 'ops:command-execution-create' %}">
<i class="fa fa-terminal" style="width: 14px"></i> <span class="nav-label">{% trans 'Command execution' %}</span><span class="label label-info pull-right"></span>
</a>
</li>
......@@ -41,4 +41,4 @@
<a href="{% url 'terminal:web-sftp' %}" target="_blank"><i class="fa fa-file" style="width: 14px"></i>
<span class="nav-label">{% trans 'File manager' %}</span>
</a>
</li>
\ No newline at end of file
</li>
......@@ -4,9 +4,8 @@ from django.db import migrations
def get_storage_data(s):
from common.utils import get_signer
from common.utils import signer
import json
signer = get_signer()
value = s.value
encrypted = s.encrypted
if encrypted:
......
......@@ -17,15 +17,13 @@ from django.utils import timezone
from django.shortcuts import reverse
from orgs.utils import current_org
from common.utils import get_signer, date_expired_default, get_logger, lazyproperty
from common.utils import signer, date_expired_default, get_logger, lazyproperty
from common import fields
from ..signals import post_user_change_password
__all__ = ['User']
signer = get_signer()
logger = get_logger(__file__)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment