[Update] 修改celery位置

apps/common/api.py

 # -*- coding: utf-8 -*-
 #
-import os
 import json
-import uuid
 
-from django.core.cache import cache
-from rest_framework.generics import RetrieveAPIView
 from rest_framework.views import Response, APIView
 from ldap3 import Server, Connection
 from django.core.mail import get_connection, send_mail
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 
-from .permissions import IsSuperUser, IsAppUser
+from .permissions import IsSuperUser
 from .serializers import MailTestSerializer, LDAPTestSerializer
-from .celery import FINISHED
-from .const import FILE_END_GUARD, celery_task_pre_key
 
 
 class MailTestingAPI(APIView):
@@ -112,27 +106,4 @@ class DjangoSettingsAPI(APIView):
         return Response(configs)
 
 
-class CeleryTaskLogApi(APIView):
-    permission_classes = (IsSuperUser,)
-    buff_size = 1024 * 10
-    end = False
-
-    def get(self, request, *args, **kwargs):
-        task_id = kwargs.get('pk')
-        info = cache.get(celery_task_pre_key + task_id, {})
-        log_path = info.get("log_path")
-        mark = request.query_params.get("mark") or str(uuid.uuid4())
-
-        if not log_path or not os.path.isfile(log_path):
-            return Response({"data": _("Waiting ...")}, status=203)
-
-        with open(log_path, 'r') as f:
-            offset = cache.get(mark, 0)
-            f.seek(offset)
-            data = f.read(self.buff_size).replace('\n', '\r\n')
-            mark = str(uuid.uuid4())
-            cache.set(mark, f.tell(), 5)
-
-            if data == '' and info["status"] == FINISHED:
-                self.end = True
-            return Response({"data": data, 'end': self.end, 'mark': mark})
+

apps/common/models.py

@@ -80,11 +80,3 @@ class Setting(models.Model):
     class Meta:
         db_table = "settings"
 
-
-class CeleryTask(models.Model):
-    id = models.UUIDField()
-    name = models.CharField(max_length=1024)
-    status = models.CharField(max_length=128)
-    date_published = models.DateTimeField(auto_now_add=True)
-    date_start = models.DateTimeField(null=True)
-    date_finished = models.DateTimeField(null=True)
\ No newline at end of file

apps/common/urls/api_urls.py

@@ -10,5 +10,4 @@ urlpatterns = [
     url(r'^v1/mail/testing/$', api.MailTestingAPI.as_view(), name='mail-testing'),
     url(r'^v1/ldap/testing/$', api.LDAPTestingAPI.as_view(), name='ldap-testing'),
     url(r'^v1/django-settings/$', api.DjangoSettingsAPI.as_view(), name='django-settings'),
-    url(r'^v1/celery/task/(?P<pk>[0-9a-zA-Z\-]{36})/log/$', api.CeleryTaskLogApi.as_view(), name='celery-task-log'),
 ]

apps/common/urls/view_urls.py

@@ -11,6 +11,4 @@ urlpatterns = [
     url(r'^email/$', views.EmailSettingView.as_view(), name='email-setting'),
     url(r'^ldap/$', views.LDAPSettingView.as_view(), name='ldap-setting'),
     url(r'^terminal/$', views.TerminalSettingView.as_view(), name='terminal-setting'),
-
-    url(r'^celery/task/(?P<pk>[0-9a-zA-Z\-]{36})/log/$', views.CeleryTaskLogView.as_view(), name='celery-task-log'),
 ]

apps/common/views.py

@@ -122,18 +122,3 @@ class TerminalSettingView(AdminUserRequiredMixin, TemplateView):
             return render(request, self.template_name, context)
 
 
-class CeleryTaskLogView(AdminUserRequiredMixin, TemplateView):
-    template_name = 'common/celery_task_log.html'
-    task_log_path = None
-
-    def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
-        task_id = self.kwargs.get("pk")
-
-        if cache.get(celery_task_pre_key+task_id) is None:
-            raise Http404()
-
-        context.update({
-            "task_id": self.kwargs.get("pk")
-        })
-        return context

apps/ops/api.py

@@ -6,18 +6,15 @@ from django.core.cache import cache
 from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext as _
 from rest_framework import viewsets, generics
-from rest_framework.views import APIView
 from rest_framework.views import Response
 
 from .hands import IsSuperUser
-from common.const import FILE_END_GUARD
-from .models import Task, AdHoc, AdHocRunHistory
-from .serializers import TaskSerializer, AdHocSerializer, AdHocRunHistorySerializer
+from .models import Task, AdHoc, AdHocRunHistory, CeleryTask
+from .serializers import TaskSerializer, AdHocSerializer, \
+    AdHocRunHistorySerializer
 from .tasks import run_ansible_task
 
 
-
-
 class TaskViewSet(viewsets.ModelViewSet):
     queryset = Task.objects.all()
     serializer_class = TaskSerializer
@@ -67,28 +64,28 @@ class AdHocRunHistorySet(viewsets.ModelViewSet):
         return self.queryset
 
 
-class LogFileViewApi(APIView):
+class CeleryTaskLogApi(generics.RetrieveAPIView):
     permission_classes = (IsSuperUser,)
     buff_size = 1024 * 10
     end = False
+    queryset = CeleryTask.objects.all()
 
     def get(self, request, *args, **kwargs):
-        file_path = request.query_params.get("file")
         mark = request.query_params.get("mark") or str(uuid.uuid4())
+        task = super().get_object()
+        log_path = task.full_log_path
 
-        if not os.path.isfile(file_path):
-            print(file_path)
-            return Response({"error": _("Log file not found")}, status=204)
+        if not log_path or not os.path.isfile(log_path):
+            return Response({"data": _("Waiting ...")}, status=203)
 
-        with open(file_path, 'r') as f:
+        with open(log_path, 'r') as f:
             offset = cache.get(mark, 0)
             f.seek(offset)
             data = f.read(self.buff_size).replace('\n', '\r\n')
             mark = str(uuid.uuid4())
             cache.set(mark, f.tell(), 5)
 
-            if FILE_END_GUARD  in data:
-                data.replace(FILE_END_GUARD, '')
+            if data == '' and task.is_finished():
                 self.end = True
-
             return Response({"data": data, 'end': self.end, 'mark': mark})
+

apps/ops/apps.py

@@ -5,3 +5,7 @@ from django.apps import AppConfig
 
 class OpsConfig(AppConfig):
     name = 'ops'
+
+    def ready(self):
+        super().ready()
+        from .celery import signal_handler

apps/ops/celery/__init__.py

@@ -15,4 +15,3 @@ app = Celery('jumpserver')
 # pickle the object when using Windows.
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])
-

apps/ops/celery/signal_handler.py

@@ -5,27 +5,21 @@ import datetime
 import sys
 
 from django.conf import settings
+from django.utils import timezone
 from django.core.cache import cache
+from django.db import transaction
 from celery import subtask
 from celery.signals import worker_ready, worker_shutdown, task_prerun, \
     task_postrun, after_task_publish
-
 from django_celery_beat.models import PeriodicTask
 
-from common.utils import get_logger, TeeObj
+from common.utils import get_logger, TeeObj, get_object_or_none
 from common.const import celery_task_pre_key
-
 from .utils import get_after_app_ready_tasks, get_after_app_shutdown_clean_tasks
-
+from ..models import CeleryTask
 
 logger = get_logger(__file__)
 
-WAITING = "waiting"
-RUNNING = "running"
-FINISHED = "finished"
-
-EXPIRE_TIME = 3600
-
 
 @worker_ready.connect
 def on_app_ready(sender=None, headers=None, body=None, **kwargs):
@@ -54,18 +48,31 @@ def after_app_shutdown(sender=None, headers=None, body=None, **kwargs):
     PeriodicTask.objects.filter(name__in=tasks).delete()
 
 
+@after_task_publish.connect
+def after_task_publish_signal_handler(sender, headers=None, **kwargs):
+    CeleryTask.objects.create(
+        id=headers["id"], status=CeleryTask.WAITING, name=headers["task"]
+    )
+
+
 @task_prerun.connect
 def pre_run_task_signal_handler(sender, task_id=None, task=None, **kwargs):
-    task_key = celery_task_pre_key + task_id
-    info = cache.get(task_key, {})
+    t = get_object_or_none(CeleryTask, id=task_id)
+    if t is None:
+        logger.warn("Not get the task: {}".format(task_id))
+        return
     now = datetime.datetime.now().strftime("%Y-%m-%d")
-    log_dir = os.path.join(settings.PROJECT_DIR, "data", "celery", now)
-    if not os.path.exists(log_dir):
-        os.makedirs(log_dir)
-    log_path = os.path.join(log_dir, task_id + '.log')
-    info.update({"status": RUNNING, "log_path": log_path})
-    cache.set(task_key, info, EXPIRE_TIME)
-    f = open(log_path, 'w')
+    log_path = os.path.join(now, task_id + '.log')
+    full_path = os.path.join(CeleryTask.LOG_DIR, log_path)
+
+    if not os.path.exists(os.path.dirname(full_path)):
+        os.makedirs(os.path.dirname(full_path))
+    with transaction.atomic():
+        t.date_start = timezone.now()
+        t.status = CeleryTask.RUNNING
+        t.log_path = log_path
+        t.save()
+    f = open(full_path, 'w')
     tee = TeeObj(f)
     sys.stdout = tee
     task.log_f = tee
@@ -73,17 +80,15 @@ def pre_run_task_signal_handler(sender, task_id=None, task=None, **kwargs):
 
 @task_postrun.connect
 def post_run_task_signal_handler(sender, task_id=None, task=None, **kwargs):
-    task_key = celery_task_pre_key + task_id
-    info = cache.get(task_key, {})
-    info.update({"status": FINISHED})
-    cache.set(task_key, info, EXPIRE_TIME)
+    t = get_object_or_none(CeleryTask, id=task_id)
+    if t is None:
+        logger.warn("Not get the task: {}".format(task_id))
+        return
+    with transaction.atomic():
+        t.status = CeleryTask.FINISHED
+        t.date_finished = timezone.now()
+        t.save()
     task.log_f.flush()
     sys.stdout = task.log_f.origin_stdout
     task.log_f.close()
 
-
-@after_task_publish.connect
-def after_task_publish_signal_handler(sender, headers=None, **kwargs):
-    task_id = headers["id"]
-    key = celery_task_pre_key + task_id
-    cache.set(key, {"status": WAITING}, EXPIRE_TIME)
\ No newline at end of file

apps/ops/models/__init__.py

+# -*- coding: utf-8 -*-
+#
+
+from .adhoc import *
+from .celery import *
\ No newline at end of file

apps/ops/models.py → apps/ops/models/adhoc.py

@@ -11,15 +11,14 @@ from django.db import models
 from django.conf import settings
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from django_celery_beat.models import CrontabSchedule, IntervalSchedule, \
-    PeriodicTask
+from django_celery_beat.models import PeriodicTask
 
 from common.utils import get_signer, get_logger
-from .celery.utils import delete_celery_periodic_task, \
+from ..celery.utils import delete_celery_periodic_task, \
     create_or_update_celery_periodic_tasks, \
     disable_celery_periodic_task
-from .ansible import AdHocRunner, AnsibleError
-from .inventory import JMSInventory
+from ..ansible import AdHocRunner, AnsibleError
+from ..inventory import JMSInventory
 
 __all__ = ["Task", "AdHoc", "AdHocRunHistory"]
 
@@ -91,7 +90,7 @@ class Task(models.Model):
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
-        from .tasks import run_ansible_task
+        from ..tasks import run_ansible_task
         super().save(
             force_insert=force_insert,  force_update=force_update,
             using=using, update_fields=update_fields,

apps/ops/models/celery.py

+# -*- coding: utf-8 -*-
+#
+import uuid
+import os
+from django.conf import settings
+from django.db import models
+
+
+class CeleryTask(models.Model):
+    WAITING = "waiting"
+    RUNNING = "running"
+    FINISHED = "finished"
+    LOG_DIR = os.path.join(settings.PROJECT_DIR, 'data', 'celery')
+
+    STATUS_CHOICES = (
+        (WAITING, WAITING),
+        (RUNNING, RUNNING),
+        (FINISHED, FINISHED),
+    )
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4)
+    name = models.CharField(max_length=1024)
+    status = models.CharField(max_length=128, choices=STATUS_CHOICES)
+    log_path = models.CharField(max_length=256, blank=True, null=True)
+    date_published = models.DateTimeField(auto_now_add=True)
+    date_start = models.DateTimeField(null=True)
+    date_finished = models.DateTimeField(null=True)
+
+    def __str__(self):
+        return "{}: {}".format(self.name, self.id)
+
+    def is_finished(self):
+        return self.status == self.FINISHED
+
+    @property
+    def full_log_path(self):
+        return os.path.join(self.LOG_DIR, self.log_path)

apps/ops/tasks.py

 # coding: utf-8
 from celery import shared_task, subtask
 
-
 from common.utils import get_logger, get_object_or_none
 from .models import Task
 

apps/ops/templates/ops/adhoc_history_detail.html

@@ -19,7 +19,7 @@
                                 <a href="{% url 'ops:adhoc-history-detail' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history detail' %} </a>
                             </li>
                             <li>
-                                <a href="{% url 'ops:adhoc-history-output' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'History output' %} </a>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Output' %} </a>
                             </li>
                         </ul>
                     </div>

apps/ops/templates/ops/adhoc_history_output.html

-{% extends 'base.html' %}
-{% load static %}
-{% load i18n %}
-
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-    <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
-</head>
-{% endblock %}
-{% block content %}
-    <div class="wrapper wrapper-content animated fadeInRight">
-        <div class="row">
-            <div class="col-sm-12">
-                <div class="ibox float-e-margins">
-                    <div class="panel-options">
-                        <ul class="nav nav-tabs">
-                           <li>
-                                <a href="{% url 'ops:adhoc-history-detail' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history detail' %} </a>
-                            </li>
-                            <li class="active">
-                                <a href="{% url 'ops:adhoc-history-output' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'History output' %} </a>
-                            </li>
-                        </ul>
-                    </div>
-                    <div class="tab-content" style="height: 800px">
-                        <iframe src="{% url 'ops:adhoc-history-output-alone' pk=object.pk %}" width="100%" height="100%">
-
-                        </iframe>
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-{% endblock %}
-

apps/common/templates/common/celery_task_log.html → apps/ops/templates/ops/celery_task_log.html

@@ -35,7 +35,7 @@
     var rowHeight = 1;
     var colWidth = 1;
     var mark = '';
-    var url = "{% url 'api-common:celery-task-log' pk=task_id %}";
+    var url = "{% url 'api-ops:celery-task-log' pk=object.id %}";
     var term;
     var end = false;
     var error = false;

apps/ops/templates/ops/celery_task_output.html

-{% load static %}
-<head>
-    <title>term.js</title>
-    <script src="{% static 'js/jquery-2.1.1.js' %}"></script>
-    <style>
-      html {
-        background: #000;
-      }
-      h1 {
-        margin-bottom: 20px;
-        font: 20px/1.5 sans-serif;
-      }
-      .terminal {
-        float: left;
-        font-family: 'Monaco', 'Consolas', "DejaVu Sans Mono", "Liberation Mono", monospace;
-        font-size: 12px;
-        color: #f0f0f0;
-        background-color: #555;
-        padding: 20px 20px 20px;
-      }
-      .terminal-cursor {
-        color: #000;
-        background: #f0f0f0;
-      }
-    </style>
-</head>
-<div class="container">
-    <div id="term">
-    </div>
-</div>
-
-
-<script src="{% static 'js/term.js' %}"></script>
-<script>
-    var rowHeight = 1;
-    var colWidth = 1;
-    var mark = '';
-    var url = "{% url 'api-ops:history-output' pk=object.id %}";
-    var term;
-    var end = false;
-    var has_error = false;
-
-    function calWinSize() {
-        var t = $('.terminal');
-        rowHeight = 1.00 * t.height() / 24;
-        colWidth = 1.00 * t.width() / 80;
-    }
-    function resize() {
-       var rows = Math.floor(window.innerHeight / rowHeight) - 2;
-       var cols = Math.floor(window.innerWidth / colWidth) - 5;
-       term.resize(cols, rows);
-    }
-    function requestAndWrite() {
-        if (!end) {
-            $.ajax({
-                url: url + '?mark=' + mark,
-                method: "GET",
-                contentType: "application/json; charset=utf-8"
-            }).done(function(data, textStatue, jqXHR) {
-                term.write(data.data);
-                mark = data.mark;
-                if (data.end){
-                    end = true
-                }
-            }).fail(function(jqXHR, textStatus, errorThrown) {
-                if (!has_error) {
-                    var error = jqXHR.responseJSON.error;
-                    term.write('\x1b[31m' + error + '\x1b[m\r\n');
-                    has_error = true
-                }
-            });
-        }
-    }
-    $(document).ready(function () {
-        term = new Terminal({
-            cols: 80,
-            rows: 24,
-            useStyle: true,
-            screenKeys: false,
-            convertEol: false,
-            cursorBlink: false
-        });
-        term.open();
-        term.on('data', function (data) {
-            term.write(data.replace('\r', '\r\n'))
-        });
-        calWinSize();
-        resize();
-        $('.terminal').detach().appendTo('#term');
-        setInterval(function () {
-            requestAndWrite()
-        }, 200)
-    });
-</script>

apps/ops/templates/ops/task_adhoc.html

@@ -24,6 +24,9 @@
                             <li>
                                 <a href="{% url 'ops:task-history' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history' %} </a>
                             </li>
+                            <li>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.latest_history.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
+                            </li>
                         </ul>
                     </div>
                     <div class="tab-content">

apps/ops/templates/ops/task_detail.html

@@ -25,7 +25,7 @@
                                 <a href="{% url 'ops:task-history' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history' %} </a>
                             </li>
                             <li>
-                                <a href="{% url 'ops:adhoc-history-output' pk=object.latest_history.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.latest_history.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
                             </li>
                         </ul>
                     </div>

apps/ops/templates/ops/task_history.html

@@ -24,6 +24,9 @@
                             <li class="active">
                                 <a href="{% url 'ops:task-history' pk=object.pk %}" class="text-center"><i class="fa fa-laptop"></i> {% trans 'Run history' %} </a>
                             </li>
+                            <li>
+                                <a class="text-center celery-task-log" onclick="window.open('{% url 'ops:celery-task-log' pk=object.latest_history.pk %}','', 'width=800,height=600')"><i class="fa fa-laptop"></i> {% trans 'Last run output' %} </a>
+                            </li>
                         </ul>
                     </div>
                     <div class="tab-content">

apps/ops/templates/ops/task_list.html

@@ -113,8 +113,8 @@ $(document).ready(function() {
     };
     var success = function(data) {
         var task_id = data.task;
-        var url = '{% url "common:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
-        window.open(url, '', 'width=800,height=800')
+        var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id);
+        window.open(url, '', 'width=800,height=600')
     };
     APIUpdateAttr({
         url: the_url,

apps/ops/urls/api_urls.py

@@ -15,7 +15,7 @@ router.register(r'v1/history', api.AdHocRunHistorySet, 'history')
 
 urlpatterns = [
     url(r'^v1/tasks/(?P<pk>[0-9a-zA-Z\-]{36})/run/$', api.TaskRun.as_view(), name='task-run'),
-    # url(r'^v1/history/(?P<pk>[0-9a-zA-Z\-]{36})/output/$', api.CeleryTaskOutputApi.as_view(), name='history-output'),
+    url(r'^v1/celery/task/(?P<pk>[0-9a-zA-Z\-]{36})/log/$', api.CeleryTaskLogApi.as_view(), name='celery-task-log'),
 ]
 
 urlpatterns += router.urls

apps/ops/urls/view_urls.py

@@ -18,6 +18,5 @@ urlpatterns = [
     url(r'^adhoc/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.AdHocDetailView.as_view(), name='adhoc-detail'),
     url(r'^adhoc/(?P<pk>[0-9a-zA-Z\-]{36})/history/$', views.AdHocHistoryView.as_view(), name='adhoc-history'),
     url(r'^adhoc/history/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.AdHocHistoryDetailView.as_view(), name='adhoc-history-detail'),
-    url(r'^adhoc/history/(?P<pk>[0-9a-zA-Z\-]{36})/_output/$', views.CeleryTaskOutputView.as_view(), name='adhoc-history-output-alone'),
-    url(r'^adhoc/history/(?P<pk>[0-9a-zA-Z\-]{36})/output/$', views.AdHocHistoryOutputView.as_view(), name='adhoc-history-output'),
+    url(r'^celery/task/(?P<pk>[0-9a-zA-Z\-]{36})/log/$', views.CeleryTaskLogView.as_view(), name='celery-task-log'),
 ]

apps/ops/views.py

@@ -5,7 +5,7 @@ from django.conf import settings
 from django.views.generic import ListView, DetailView, TemplateView
 
 from common.mixins import DatetimeSearchMixin
-from .models import Task, AdHoc, AdHocRunHistory
+from .models import Task, AdHoc, AdHocRunHistory, CeleryTask
 from .hands import AdminUserRequiredMixin
 
 
@@ -121,19 +121,6 @@ class AdHocHistoryDetailView(AdminUserRequiredMixin, DetailView):
         return super().get_context_data(**kwargs)
 
 
-class CeleryTaskOutputView(AdminUserRequiredMixin, TemplateView):
-    model = AdHocRunHistory
-    template_name = 'ops/celery_task_output.html'
-
-
-class AdHocHistoryOutputView(AdminUserRequiredMixin, DetailView):
-    model = AdHocRunHistory
-    template_name = 'ops/adhoc_history_output.html'
-
-    def get_context_data(self, **kwargs):
-        context = {
-            'app': _('Ops'),
-            'action': _('Run history detail'),
-        }
-        kwargs.update(context)
-        return super().get_context_data(**kwargs)
\ No newline at end of file
+class CeleryTaskLogView(AdminUserRequiredMixin, DetailView):
+    template_name = 'ops/celery_task_log.html'
+    model = CeleryTask