[Update] 修改任务执行

apps/jumpserver/celery_flower.py

+# -*- coding: utf-8 -*-
+#
+from django.http import HttpResponse
+from django.conf import settings
+from django.utils.translation import ugettext as _
+from django.views.decorators.csrf import csrf_exempt
+
+from proxy.views import proxy_view
+
+flower_url = settings.FLOWER_URL
+
+
+@csrf_exempt
+def celery_flower_view(request, path):
+    if not request.user.is_superuser:
+        return HttpResponse("Forbidden")
+    remote_url = 'http://{}/{}'.format(flower_url, path)
+    try:
+        response = proxy_view(request, remote_url)
+    except Exception as e:
+        msg = _("<h1>Flow service unavailable, check it</h1>") + \
+              '<br><br> <div>{}</div>'.format(e)
+        response = HttpResponse(msg)
+    return response
+

apps/jumpserver/conf.py

@@ -382,7 +382,8 @@ defaults = {
     'SYSLOG_ADDR': '',  # '192.168.0.1:514'
     'SYSLOG_FACILITY': 'user',
     'PERM_SINGLE_ASSET_TO_UNGROUP_NODE': False,
-    'WINDOWS_SSH_DEFAULT_SHELL': 'cmd'
+    'WINDOWS_SSH_DEFAULT_SHELL': 'cmd',
+    'FLOWER_URL': "127.0.0.1:5555"
 }
 
 

apps/jumpserver/settings.py

@@ -623,3 +623,4 @@ BACKEND_ASSET_USER_AUTH_VAULT = False
 
 PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
 WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
+FLOWER_URL = CONFIG.FLOWER_URL

apps/jumpserver/urls.py

@@ -7,7 +7,9 @@ from django.conf.urls.static import static
 from django.conf.urls.i18n import i18n_patterns
 from django.views.i18n import JavaScriptCatalog
 
-from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
+# from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
+from . import views
+from .celery_flower import celery_flower_view
 from .swagger import get_swagger_view
 
 api_v1 = [
@@ -40,6 +42,7 @@ app_view_patterns = [
     path('orgs/', include('orgs.urls.views_urls', namespace='orgs')),
     path('auth/', include('authentication.urls.view_urls'), name='auth'),
     path('applications/', include('applications.urls.views_urls', namespace='applications')),
+    re_path(r'flower/(?P<path>.*)', celery_flower_view, name='flower-view'),
 ]
 
 
@@ -57,13 +60,13 @@ js_i18n_patterns = i18n_patterns(
 
 
 urlpatterns = [
-    path('', IndexView.as_view(), name='index'),
+    path('', views.IndexView.as_view(), name='index'),
     path('api/v1/', include(api_v1)),
     path('api/v2/', include(api_v2)),
-    re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', redirect_format_api),
-    path('api/health/', HealthCheckView.as_view(), name="health"),
-    path('luna/', LunaView.as_view(), name='luna-view'),
-    path('i18n/<str:lang>/', I18NView.as_view(), name='i18n-switch'),
+    re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', views.redirect_format_api),
+    path('api/health/', views.HealthCheckView.as_view(), name="health"),
+    path('luna/', views.LunaView.as_view(), name='luna-view'),
+    path('i18n/<str:lang>/', views.I18NView.as_view(), name='i18n-switch'),
     path('settings/', include('settings.urls.view_urls', namespace='settings')),
 
     # External apps url

apps/jumpserver/views.py

@@ -224,3 +224,6 @@ class HealthCheckView(APIView):
 
     def get(self, request):
         return JsonResponse({"status": 1, "time": int(time.time())})
+
+
+

apps/ops/migrations/0008_auto_20190919_2100.py

+# Generated by Django 2.1.7 on 2019-09-19 13:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0007_auto_20190724_2002'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True, verbose_name='Date updated'),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='date_created',
+            field=models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Date created'),
+        ),
+        migrations.AlterModelOptions(
+            name='task',
+            options={'get_latest_by': 'date_created',
+                     'ordering': ('-date_updated',)},
+        ),
+    ]

apps/ops/models/adhoc.py

@@ -13,7 +13,7 @@ from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django_celery_beat.models import PeriodicTask
 
-from common.utils import get_signer, get_logger
+from common.utils import get_signer, get_logger, lazyproperty
 from orgs.utils import set_to_root_org
 from ..celery.utils import delete_celery_periodic_task, \
     create_or_update_celery_periodic_tasks, \
@@ -42,7 +42,8 @@ class Task(models.Model):
     is_deleted = models.BooleanField(default=False)
     comment = models.TextField(blank=True, verbose_name=_("Comment"))
     created_by = models.CharField(max_length=128, blank=True, default='')
-    date_created = models.DateTimeField(auto_now_add=True, db_index=True)
+    date_created = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name=_("Date created"))
+    date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
     __latest_adhoc = None
     _ignore_auto_created_by = True
 
@@ -51,16 +52,39 @@ class Task(models.Model):
         return str(self.id).split('-')[-1]
 
     @property
-    def latest_adhoc(self):
-        if not self.__latest_adhoc:
-            self.__latest_adhoc = self.get_latest_adhoc()
-        return self.__latest_adhoc
+    def versions(self):
+        return self.adhoc.all().count()
+
+    @property
+    def is_success(self):
+        if self.latest_history:
+            return self.latest_history.is_success
+        else:
+            return False
 
-    @latest_adhoc.setter
-    def latest_adhoc(self, item):
-        self.__latest_adhoc = item
+    @property
+    def timedelta(self):
+        if self.latest_history:
+            return self.latest_history.timedelta
+        else:
+            return 0
 
     @property
+    def date_start(self):
+        if self.latest_history:
+            return self.latest_history.date_start
+        else:
+            return None
+
+    @property
+    def assets_amount(self):
+        return self.latest_adhoc.hosts.count()
+
+    @lazyproperty
+    def latest_adhoc(self):
+        return self.get_latest_adhoc()
+
+    @lazyproperty
     def latest_history(self):
         try:
             return self.history.all().latest()
@@ -139,6 +163,7 @@ class Task(models.Model):
     class Meta:
         db_table = 'ops_task'
         unique_together = ('name', 'created_by')
+        ordering = ('-date_updated',)
         get_latest_by = 'date_created'
 
 
@@ -246,6 +271,7 @@ class AdHoc(models.Model):
             )
 
     def _run_only(self):
+        Task.objects.filter(id=self.task.id).update(date_updated=timezone.now())
         runner = AdHocRunner(self.inventory, options=self.options)
         try:
             result = runner.run(

apps/ops/serializers.py

@@ -19,7 +19,12 @@ class CeleryTaskSerializer(serializers.Serializer):
 class TaskSerializer(serializers.ModelSerializer):
     class Meta:
         model = Task
-        fields = '__all__'
+        fields = [
+            'id', 'name', 'interval', 'crontab', 'is_periodic',
+            'is_deleted', 'comment', 'created_by', 'date_created',
+            'versions', 'is_success', 'timedelta', 'assets_amount',
+            'date_updated', 'history_summary',
+        ]
 
 
 class AdHocSerializer(serializers.ModelSerializer):

apps/ops/tasks.py

@@ -128,7 +128,7 @@ def hello_callback(result):
 @shared_task
 def add(a, b):
     time.sleep(5)
-    return max(b)
+    return a + b
 
 
 @shared_task

apps/ops/templates/ops/task_list.html

 {% extends '_base_list.html' %}
-{% load i18n %}
-{% load static %}
-
-{% block content_left_head %}
-{#    <div class="uc pull-left m-r-5"><a class="btn btn-sm btn-primary btn-create-asset"> {% trans "Create task" %} </a></div>#}
-{% endblock %}
-
-
-{% block table_search %}
-    <form id="search_form" method="get" action="" class="pull-right form-inline">
-        <div class="input-group">
-            <input type="text" class="form-control input-sm" name="keyword" placeholder="{% trans 'Search'  %}" value="{{ keyword }}">
-        </div>
-        <div class="input-group">
-            <div class="input-group-btn">
-                <button id='search_btn' type="submit" class="btn btn-sm btn-primary">
-                    {% trans "Search"  %}
-                </button>
-            </div>
-        </div>
-    </form>
-{% endblock %}
-
-{% block table_head %}
-    <th class="text-center"></th>
-    <th class="text-center">{% trans 'Name' %}</th>
+{% load i18n static %}
+{% block table_search %}{% endblock %}
+{% block table_container %}
+    <table class="table table-striped table-bordered table-hover " id="task_list_table">
+        <thead>
+        <tr>
+            <th class="text-center">
+                <input id="" type="checkbox" class="ipt_check_all">
+            </th>
+            <th class="text-left">{% trans 'Name' %}</th>
             <th class="text-center">{% trans 'Run times' %}</th>
             <th class="text-center">{% trans 'Versions' %}</th>
             <th class="text-center">{% trans 'Hosts' %}</th>
@@ -32,70 +16,72 @@
             <th class="text-center">{% trans 'Date' %}</th>
             <th class="text-center">{% trans 'Time' %}</th>
             <th class="text-center">{% trans 'Action' %}</th>
-{% endblock %}
-
-{% block table_body %}
-    {% for object in task_list %}
-        <tr class="gradeX">
-            <td class="text-center"><input type="checkbox" class="cbx-term"> </td>
-            <td class="text-left"><a href="{% url 'ops:task-detail' pk=object.id %}">{{ object.name }}</a></td>
-            <td class="text-center">
-                <span class="text-danger">{{ object.history_summary.failed }}</span>/<span class="text-navy">{{ object.history_summary.success}}</span>/{{ object.history_summary.total}}
-            </td>
-            <td class="text-center">{{ object.adhoc.all | length}}</td>
-            <td class="text-center">{{ object.latest_adhoc.hosts | length}}</td>
-            <td class="text-center">
-            {% if object.latest_history %}
-                {% if object.latest_history.is_success %}
-                    <i class="fa fa-check text-navy"></i>
-                {% else %}
-                    <i class="fa fa-times text-danger"></i>
-                {% endif %}
-            {% endif %}
-            </td>
-            <td class="text-center">{{ object.latest_history.date_start }}</td>
-            <td class="text-center">
-                {% if object.latest_history %}
-                {{ object.latest_history.timedelta|floatformat }} s
-                {% endif %}
-            </td>
-            <td class="text-center">
-                <a data-uid="{{ object.id }}" class="btn btn-xs btn-primary btn-run">{% trans "Run" %}</a>
-                <a data-uid="{{ object.id }}" class="btn btn-xs btn-danger btn-del">{% trans "Delete" %}</a>
-            </td>
         </tr>
-    {% endfor %}
+        </thead>
+    </table>
 {% endblock %}
 
+{% block content_bottom_left %}{% endblock %}
 {% block custom_foot_js %}
-<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
 <script>
-$(document).ready(function() {
-    $('table').DataTable({
-        "searching": false,
-        "paging": false,
-        "bInfo" : false,
-        "order": [],
-        "columnDefs": [
-            { "targets": 0, "orderable": false },
-            { "targets": 4, "orderable": false },
-            { "targets": 5, "orderable": false },
-            { "targets": 8, "orderable": false }
-        ]
-    });
-    $('.select2').select2({
-        dropdownAutoWidth : true,
-        width: 'auto'
-    });
-    $('#date .input-daterange').datepicker({
-        format: "yyyy-mm-dd",
-        todayBtn: "linked",
-        keyboardNavigation: false,
-        forceParse: false,
-        calendarWeeks: true,
-        autoclose: true
-    });
-
+$(document).ready(function () {
+    var options = {
+        ele: $('#task_list_table'),
+        buttons: [],
+        columnDefs: [
+            {targets: 1, createdCell: function (td, cellData, rowData) {
+                cellData = htmlEscape(cellData);
+                var innerHtml = '<a href="{% url "ops:task-detail" pk=DEFAULT_PK %}">' + cellData + '</a>'
+                innerHtml = innerHtml.replace('{{ DEFAULT_PK }}', rowData.id);
+                $(td).html(innerHtml);
+            }},
+            {targets: 2, createdCell: function (td, cellData) {
+                var innerHtml = '<span class="text-danger">failed</span>/<span class="text-navy">success</span>/total';
+                if (cellData) {
+                    innerHtml = innerHtml.replace('failed', cellData.failed)
+                        .replace('success', cellData.success)
+                        .replace('total', cellData.total);
+                    $(td).html(innerHtml);
+                } else {
+                    $(td).html('')
+                }
+            }},
+            {targets: 5, createdCell: function (td, cellData) {
+                var successBtn = '<i class="fa fa-check text-navy"></i>';
+                var failedBtn = '<i class="fa fa-times text-danger"></i>';
+                if (cellData) {
+                    $(td).html(successBtn)
+                } else {
+                    $(td).html(failedBtn)
+                }
+            }},
+            {targets: 6, createdCell: function (td, cellData) {
+                $(td).html(toSafeLocalDateStr(cellData));
+            }},
+            {targets: 7, createdCell: function (td, cellData) {
+                var delta = readableSecond(cellData);
+                $(td).html(delta);
+            }},
+            {
+                targets: 8,
+                createdCell: function (td, cellData, rowData) {
+                    var runBtn = '<a data-uid="ID" class="btn btn-xs btn-primary btn-run">{% trans "Run" %}</a> '.replace('ID', cellData);
+                    var delBtn = '<a data-uid="ID" class="btn btn-xs btn-danger btn-del">{% trans "Delete" %}</a>'.replace('ID', cellData);
+                    $(td).html(runBtn + delBtn)
+                }
+            }
+        ],
+        ajax_url: '{% url "api-ops:task-list" %}',
+        columns: [
+            {data: "id"}, {data: "name", className: "text-left"}, {data: "history_summary", orderable: false},
+            {data: "versions", orderable: false}, {data: "assets_amount", orderable: false},
+            {data: "is_success", orderable: false}, {data: "date_updated"},
+            {data: "timedelta", orderable:false}, {data: "id", orderable: false},
+        ],
+        order: [],
+        op_html: $('#actions').html()
+    };
+    jumpserver.initServerSideDataTable(options);
 }).on('click', '.btn-del', function () {
     var $this = $(this);
     var name = $this.closest("tr").find(":nth-child(2)").children('a').html();
@@ -122,7 +108,6 @@ $(document).ready(function() {
         success: success,
         success_message: "{% trans 'Task start: ' %}" + " " + name
     });
-
 })
 </script>
 {% endblock %}

apps/ops/views/adhoc.py

@@ -2,7 +2,7 @@
 
 from django.utils.translation import ugettext as _
 from django.conf import settings
-from django.views.generic import ListView, DetailView
+from django.views.generic import ListView, DetailView, TemplateView
 
 from common.mixins import DatetimeSearchMixin
 from common.permissions import PermissionsMixin, IsOrgAdmin
@@ -17,7 +17,7 @@ __all__ = [
 ]
 
 
-class TaskListView(PermissionsMixin, DatetimeSearchMixin, ListView):
+class TaskListView(PermissionsMixin, TemplateView):
     paginate_by = settings.DISPLAY_PER_PAGE
     model = Task
     ordering = ('-date_created',)
@@ -26,27 +26,10 @@ class TaskListView(PermissionsMixin, DatetimeSearchMixin, ListView):
     keyword = ''
     permission_classes = [IsOrgAdmin]
 
-    def get_queryset(self):
-        queryset = super().get_queryset()
-        if current_org.is_real():
-            queryset = queryset.filter(created_by=current_org.id)
-        else:
-            queryset = queryset.filter(created_by='')
-
-        self.keyword = self.request.GET.get('keyword', '')
-        if self.keyword:
-            queryset = queryset.filter(
-                name__icontains=self.keyword,
-            )
-        return queryset
-
     def get_context_data(self, **kwargs):
         context = {
             'app': _('Ops'),
             'action': _('Task list'),
-            'date_from': self.date_from,
-            'date_to': self.date_to,
-            'keyword': self.keyword,
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/static/js/jumpserver.js

@@ -1139,7 +1139,10 @@ function timeOffset(a, b) {
     var start = safeDate(a);
     var end = safeDate(b);
     var offset = (end - start) / 1000;
+    return readableSecond(offset)
+}
 
+function readableSecond(offset) {
     var days = offset / 3600 / 24;
     var hours = offset / 3600;
     var minutes = offset / 60;

apps/templates/_nav.html

@@ -114,6 +114,9 @@
         <ul class="nav nav-second-level">
             <li id="task"><a href="{% url 'ops:task-list' %}">{% trans 'Task list' %}</a></li>
             <li id="command-execution"><a href="{% url 'ops:command-execution-start' %}">{% trans 'Batch command' %}</a></li>
+            {% if request.user.is_superuser %}
+            <li><a href="{% url 'flower-view' path='' %}" target="_blank" >{% trans 'Task monitor' %}</a></li>
+            {% endif %}
         </ul>
     </li>
 {% endif %}

jms

@@ -200,9 +200,13 @@ def is_running(s, unlink=True):
 
 
 def parse_service(s):
-    all_services = ['gunicorn', 'celery_ansible', 'celery_default', 'beat']
+    all_services = [
+        'gunicorn', 'celery_ansible', 'celery_default', 'beat', 'flower'
+    ]
     if s == 'all':
         return all_services
+    elif s == 'gunicorn':
+        return ['gunicorn', 'flower']
     elif s == "celery":
         return ["celery_ansible", "celery_default"]
     elif "," in s:
@@ -240,17 +244,19 @@ def get_start_gunicorn_kwargs():
 
 
 def get_start_celery_ansible_kwargs():
-    print("\n- Start Celery as Distributed Task Queue")
+    print("\n- Start Celery as Distributed Task Queue: Ansible")
     return get_start_worker_kwargs('ansible', 4)
 
 
 def get_start_celery_default_kwargs():
+    print("\n- Start Celery as Distributed Task Queue: Celery")
     return get_start_worker_kwargs('celery', 2)
 
 
 def get_start_worker_kwargs(queue, num):
     # Todo: Must set this environment, otherwise not no ansible result return
     os.environ.setdefault('PYTHONOPTIMIZE', '1')
+    os.environ.setdefault('ANSIBLE_FORCE_COLOR', 'True')
 
     if os.getuid() == 0:
         os.environ.setdefault('C_FORCE_ROOT', '1')
@@ -261,6 +267,24 @@ def get_start_worker_kwargs(queue, num):
         '-l', 'INFO',
         '-c', str(num),
         '-Q', queue,
+        '-n', '{}@%h'.format(queue)
+    ]
+    return {"cmd": cmd, "cwd": APPS_DIR}
+
+
+def get_start_flower_kwargs():
+    print("\n- Start Flower as Task Monitor")
+    if os.getuid() == 0:
+        os.environ.setdefault('C_FORCE_ROOT', '1')
+
+    cmd = [
+        'celery', 'flower',
+        '-A', 'ops',
+        '-l', 'INFO',
+        '--url_prefix=flower',
+        '--auto_refresh=False',
+        '--max_tasks=1000',
+        '--tasks_columns=uuid,name,args,state,received,started,runtime,worker'
     ]
     return {"cmd": cmd, "cwd": APPS_DIR}
 
@@ -333,6 +357,7 @@ def start_service(s):
         "celery_ansible": get_start_celery_ansible_kwargs,
         "celery_default": get_start_celery_default_kwargs,
         "beat": get_start_beat_kwargs,
+        "flower": get_start_flower_kwargs,
     }
 
     kwargs = services_kwargs.get(s)()
@@ -449,7 +474,7 @@ if __name__ == '__main__':
     )
     parser.add_argument(
         "service", type=str, default="all", nargs="?",
-        choices=("all", "gunicorn", "celery", "beat", "celery,beat"),
+        choices=("all", "gunicorn", "celery", "beat", "celery,beat", "flower"),
         help="The service to start",
     )
     parser.add_argument('-d', '--daemon', nargs="?", const=1)