[Feature] 优化Ops ansible api

apps/assets/utils.py

 # ~*~ coding: utf-8 ~*~
 #
-from ops.utils import run_AdHoc
+from .models import Asset
 
 
 def test_admin_user_connective_manual(asset):
+    from ops.utils import run_AdHoc
     if not isinstance(asset, list):
         asset = [asset]
     task_tuple = (
@@ -15,3 +16,6 @@ def test_admin_user_connective_manual(asset):
     else:
         return True
 
+
+def get_assets_by_id_list(id_list):
+    return Asset.objects.filter(id__in=id_list)

apps/locale/zh/LC_MESSAGES/django.po

@@ -1418,23 +1418,23 @@ msgid "Assets id"
 msgstr "资产id"
 
 #: ops/models.py:27
-msgid "Task module and args json format"
+msgid "Playbook module and args json format"
 msgstr ""
 
 #: ops/models.py:28
-msgid "Task run pattern"
+msgid "Playbook run pattern"
 msgstr ""
 
 #: ops/models.py:29
-msgid "Task raw result"
+msgid "Playbook raw result"
 msgstr ""
 
 #: ops/models.py:30
-msgid "Task summary"
+msgid "Playbook summary"
 msgstr ""
 
 #: ops/templates/ops/task_detail.html:19
-msgid "Task replay detail"
+msgid "Playbook replay detail"
 msgstr "任务记录详情"
 
 #: ops/templates/ops/task_detail.html:62
@@ -1669,7 +1669,7 @@ msgid "Job Center"
 msgstr "作业中心"
 
 #: templates/_nav.html:51
-msgid "Task"
+msgid "Playbook"
 msgstr "任务"
 
 #: templates/_nav.html:62

apps/ops/ansible/callback.py

 # ~*~ coding: utf-8 ~*~
 
-from collections import defaultdict
 from ansible.plugins.callback import CallbackBase
 
 
-class CommandResultCallback(CallbackBase):
-    def __init__(self, display=None):
-        self.result_q = dict(contacted={}, dark={})
-        super(CommandResultCallback, self).__init__(display)
-
-    def gather_result(self, n, res):
-        self.result_q[n][res._host.name] = {}
-        self.result_q[n][res._host.name]['cmd'] = res._result.get('cmd')
-        self.result_q[n][res._host.name]['stderr'] = res._result.get('stderr')
-        self.result_q[n][res._host.name]['stdout'] = res._result.get('stdout')
-        self.result_q[n][res._host.name]['rc'] = res._result.get('rc')
-
-    def v2_runner_on_ok(self, result):
-        self.gather_result("contacted", result)
-
-    def v2_runner_on_failed(self, result, ignore_errors=False):
-        self.gather_result("dark", result)
-
-    def v2_runner_on_unreachable(self, result):
-        self.gather_result("dark", result)
-
-    def v2_runner_on_skipped(self, result):
-        self.gather_result("dark", result)
-
-
 class AdHocResultCallback(CallbackBase):
     """
     AdHoc result Callback
     """
     def __init__(self, display=None):
-        self.result_q = dict(contacted={}, dark={})
-        super(AdHocResultCallback, self).__init__(display)
-
-    def gather_result(self, n, res):
-        if res._host.name in self.result_q[n]:
-            self.result_q[n][res._host.name].append(res._result)
+        # result_raw example: {
+        #   "ok": {"hostname": []},
+        #   "failed": {"hostname": []},
+        #   "unreachable: {"hostname": []},
+        #   "skipped": {"hostname": []},
+        # }
+        # results_summary example: {
+        #   "contacted": {"hostname",...},
+        #   "dark": {"hostname": ["error",...],},
+        # }
+        self.results_raw = dict(ok={}, failed={}, unreachable={}, skipped={})
+        self.results_summary = dict(contacted=set(), dark={})
+        super().__init__(display)
+
+    def gather_result(self, t, host, res):
+        if self.results_raw[t].get(host):
+            self.results_raw[t][host].append(res)
+        else:
+            self.results_raw[t][host] = [res]
+        self.clean_result(t, host, res)
+
+    def clean_result(self, t, host, res):
+        contacted = self.results_summary["contacted"]
+        dark = self.results_summary["dark"]
+        if t in ("ok", "skipped") and host not in dark:
+            contacted.add(host)
         else:
-            self.result_q[n][res._host.name] = [res._result]
+            dark[host].append(res)
+            if host in contacted:
+                contacted.remove(dark)
 
-    def v2_runner_on_ok(self, result):
-        self.gather_result("contacted", result)
+    def runner_on_ok(self, host, res):
+        self.gather_result("ok", host, res)
 
-    def v2_runner_on_failed(self, result, ignore_errors=False):
-        self.gather_result("dark", result)
+    def runner_on_failed(self, host, res, ignore_errors=False):
+        self.gather_result("failed", host, res)
 
-    def v2_runner_on_unreachable(self, result):
-        self.gather_result("dark", result)
+    def runner_on_unreachable(self, host, res):
+        self.gather_result("unreachable", host, res)
 
-    def v2_runner_on_skipped(self, result):
-        self.gather_result("dark", result)
+    def runner_on_skipped(self, host, item=None):
+        self.gather_result("skipped", host, item)
 
-    def v2_playbook_on_task_start(self, task, is_conditional):
-        pass
 
-    def v2_playbook_on_play_start(self, play):
-        pass
+class CommandResultCallback(AdHocResultCallback):
+    def __init__(self, display=None):
+        self.results_command = dict()
+        super().__init__(display)
+
+    def gather_result(self, t, host, res):
+        super().gather_result(t, host, res)
+        self.gather_cmd(t, host, res)
+
+    def gather_cmd(self, t, host, res):
+        cmd = {}
+        if t == "ok":
+            cmd['cmd'] = res.get('cmd')
+            cmd['stderr'] = res.get('stderr')
+            cmd['stdout'] = res.get('stdout')
+            cmd['rc'] = res.get('rc')
+        else:
+            cmd['err'] = "Error: {}".format(res)
+        self.results_command[host] = cmd
 
 
 class PlaybookResultCallBack(CallbackBase):

apps/ops/ansible/exceptions.py

+# -*- coding: utf-8 -*-
+#
+
+
+class AnsibleError(Exception):
+    pass

apps/ops/ansible/inventory.py

 # ~*~ coding: utf-8 ~*~
-from ansible.inventory import Inventory, Host, Group
-from ansible.vars import VariableManager
+from ansible.inventory.group import Group
+from ansible.inventory.host import Host
+from ansible.vars.manager import VariableManager
+from ansible.inventory.manager import InventoryManager
 from ansible.parsing.dataloader import DataLoader
 
 
 class JMSHost(Host):
-    def __init__(self, asset):
-        self.asset = asset
-        self.name = name = asset.get('hostname') or asset.get('ip')
-        self.port = port = asset.get('port') or 22
-        super(JMSHost, self).__init__(name, port)
-        self.set_all_variable()
+    def __init__(self, host_data):
+        """
+        初始化
+        :param host_data:  {
+            "hostname": "",
+            "ip": "",
+            "port": "",
+            "username": "",
+            "password": "",
+            "private_key": "",
+            "become": {
+                "method": "",
+                "user": "",
+                "pass": "",
+            }
+            "groups": [],
+            "vars": {},
+        }
+        """
+        self.host_data = host_data
+        hostname = host_data.get('hostname') or host_data.get('ip')
+        port = host_data.get('port') or 22
+        super(JMSHost, self).__init__(hostname, port)
+        self.__set_required_variables()
+        self.__set_extra_variables()
 
-    def set_all_variable(self):
-        asset = self.asset
-        self.set_variable('ansible_host', asset['ip'])
-        self.set_variable('ansible_port', asset['port'])
-        self.set_variable('ansible_user', asset['username'])
+    def __set_required_variables(self):
+        host_data = self.host_data
+        self.set_variable('ansible_host', host_data['ip'])
+        self.set_variable('ansible_port', host_data['port'])
+        self.set_variable('ansible_user', host_data['username'])
 
         # 添加密码和秘钥
-        if asset.get('password'):
-            self.set_variable('ansible_ssh_pass', asset['password'])
-        if asset.get('private_key'):
-            self.set_variable('ansible_ssh_private_key_file', asset['private_key'])
+        if host_data.get('password'):
+            self.set_variable('ansible_ssh_pass', host_data['password'])
+        if host_data.get('private_key'):
+            self.set_variable('ansible_ssh_private_key_file', host_data['private_key'])
 
         # 添加become支持
-        become = asset.get("become", False)
+        become = host_data.get("become", False)
         if become:
             self.set_variable("ansible_become", True)
             self.set_variable("ansible_become_method", become.get('method', 'sudo'))
@@ -34,58 +55,73 @@ class JMSHost(Host):
         else:
             self.set_variable("ansible_become", False)
 
+    def __set_extra_variables(self):
+        for k, v in self.host_data.get('vars', {}).items():
+            self.set_variable(k, v)
+
+    def __repr__(self):
+        return self.name
 
-class JMSInventory(Inventory):
+
+class JMSInventory(InventoryManager):
     """
     提供生成Ansible inventory对象的方法
     """
+    loader_class = DataLoader
+    variable_manager_class = VariableManager
+    host_manager_class = JMSHost
 
     def __init__(self, host_list=None):
         if host_list is None:
             host_list = []
-        assert isinstance(host_list, list)
         self.host_list = host_list
-        self.loader = DataLoader()
-        self.variable_manager = VariableManager()
-        super(JMSInventory, self).__init__(self.loader, self.variable_manager,
-                                           host_list=host_list)
+        assert isinstance(host_list, list)
+        self.loader = self.loader_class()
+        self.variable_manager = self.variable_manager_class()
+        super().__init__(self.loader)
 
-    def parse_inventory(self, host_list):
-        """用于生成动态构建Ansible Inventory.
-        self.host_list: [
-            {"name": "asset_name",
-             "ip": <ip>,
-             "port": <port>,
-             "user": <user>,
-             "pass": <pass>,
-             "key": <sshKey>,
-             "groups": ['group1', 'group2'],
-             "other_host_var": <other>},
-             {...},
-        ]
+    def get_groups(self):
+        return self._inventory.groups
+
+    def get_group(self, name):
+        return self._inventory.groups.get(name, None)
 
-        :return: 返回一个Ansible的inventory对象
+    def parse_sources(self, cache=False):
         """
+        用于生成动态构建Ansible Inventory. super().__init__ 会自动调用
+        host_list: [{
+            "hostname": "",
+            "ip": "",
+            "port": "",
+            "username": "",
+            "password": "",
+            "private_key": "",
+            "become": {
+                "method": "",
+                "user": "",
+                "pass": "",
+            },
+            "groups": [],
+            "vars": {},
+          },
+        ]
 
-        # TODO: 验证输入
-        # 创建Ansible Group,如果没有则创建default组
-        ungrouped = Group('ungrouped')
-        all = Group('all')
-        all.add_child_group(ungrouped)
-        self.groups = dict(all=all, ungrouped=ungrouped)
+        :return: None
+        """
+        group_all = self.get_group('all')
+        ungrouped = self.get_group('ungrouped')
 
-        for asset in host_list:
-            host = JMSHost(asset=asset)
-            asset_groups = asset.get('groups')
-            if asset_groups:
-                for group_name in asset_groups:
-                    if group_name not in self.groups:
+        for host_data in self.host_list:
+            host = self.host_manager_class(host_data=host_data)
+            self.hosts[host_data['hostname']] = host
+            groups_data = host_data.get('groups')
+            if groups_data:
+                for group_name in groups_data:
+                    group = self.get_group(group_name)
+                    if group is None:
                         group = Group(group_name)
-                        self.groups[group_name] = group
-                    else:
-                        group = self.groups[group_name]
+                        self.add_group(group)
                     group.add_host(host)
             else:
                 ungrouped.add_host(host)
-            all.add_host(host)
-
+            group_all.add_host(host)

apps/ops/ansible/test_inventory.py

+# -*- coding: utf-8 -*-
+#
+
+import sys
+import unittest
+
+
+sys.path.insert(0, '../..')
+from ops.ansible.inventory import JMSInventory
+
+
+class TestJMSInventory(unittest.TestCase):
+    def setUp(self):
+        host_list = [{
+            "hostname": "testserver1",
+            "ip": "102.1.1.1",
+            "port": 22,
+            "username": "root",
+            "password": "password",
+            "private_key": "/tmp/private_key",
+            "become": {
+                "method": "sudo",
+                "user": "root",
+                "pass": None,
+            },
+            "groups": ["group1", "group2"],
+            "vars": {"sexy": "yes"},
+        }, {
+            "hostname": "testserver2",
+            "ip": "8.8.8.8",
+            "port": 2222,
+            "username": "root",
+            "password": "password",
+            "private_key": "/tmp/private_key",
+            "become": {
+                "method": "su",
+                "user": "root",
+                "pass": "123",
+            },
+            "groups": ["group3", "group4"],
+            "vars": {"love": "yes"},
+        }]
+
+        self.inventory = JMSInventory(host_list=host_list)
+
+    def test_hosts(self):
+        print("#"*10 + "Hosts" + "#"*10)
+        for host in self.inventory.hosts:
+            print(host)
+
+    def test_groups(self):
+        print("#" * 10 + "Groups" + "#" * 10)
+        for group in self.inventory.groups:
+            print(group)
+
+    def test_group_all(self):
+        print("#" * 10 + "all group hosts" + "#" * 10)
+        group = self.inventory.get_group('all')
+        print(group.hosts)
+
+
+if __name__ == '__main__':
+    unittest.main()

apps/ops/ansible/test_runner.py

+# -*- coding: utf-8 -*-
+#
+
+import unittest
+import sys
+
+sys.path.insert(0, "../..")
+
+from ops.ansible.runner import AdHocRunner, CommandRunner
+
+
+class TestAdHocRunner(unittest.TestCase):
+    def setUp(self):
+        host_data = [
+            {
+                "hostname": "testserver",
+                "ip": "192.168.244.168",
+                "port": 22,
+                "username": "root",
+                "password": "redhat",
+            },
+        ]
+        self.runner = AdHocRunner(hosts=host_data)
+
+    def test_run(self):
+        tasks = [
+            {"action": {"module": "shell", "args": "ls"}},
+            {"action": {"module": "shell", "args": "whoami"}},
+        ]
+        ret = self.runner.run(tasks, "all")
+        print(ret.results_summary)
+        print(ret.results_raw)
+
+
+class TestCommandRunner(unittest.TestCase):
+    def setUp(self):
+        host_data = [
+            {
+                "hostname": "testserver",
+                "ip": "192.168.244.168",
+                "port": 22,
+                "username": "root",
+                "password": "redhat",
+            },
+        ]
+        self.runner = CommandRunner(hosts=host_data)
+
+    def test_execute(self):
+        res = self.runner.execute('ls', 'all')
+        print(res.results_command)
+
+
+if __name__ == "__main__":
+    unittest.main()

apps/ops/api.py

@@ -4,12 +4,12 @@
 from rest_framework import viewsets
 
 from .hands import IsSuperUser
-from .models import Task
+from .models import Playbook
 from .serializers import TaskSerializer
 
 
 class TaskViewSet(viewsets.ModelViewSet):
-    queryset = Task.objects.all()
+    queryset = Playbook.objects.all()
     serializer_class = TaskSerializer
     permission_classes = (IsSuperUser,)
 

apps/ops/models.py

@@ -7,40 +7,32 @@ import uuid
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from assets.models import Asset
 
-__all__ = ["Task"]
+__all__ = ["Playbook"]
 
 
 logger = logging.getLogger(__name__)
 
 
-class Task(models.Model):
+class AdHoc(models.Model):
     uuid = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, blank=True, verbose_name=_('Name'))
-    date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time'))
-    date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time'))
-    timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True)
-    is_finished = models.BooleanField(default=False, verbose_name=_('Is finished'))
-    is_success = models.BooleanField(default=False, verbose_name=_('Is success'))
-    assets = models.TextField(blank=True, null=True, verbose_name=_('Assets id'))  # Asset inventory may be change
-    _modules_args = models.TextField(blank=True, null=True, verbose_name=_('Task module and args json format'))
-    pattern = models.CharField(max_length=64, default='all', verbose_name=_('Task run pattern'))
-    result = models.TextField(blank=True, null=True, verbose_name=_('Task raw result'))
-    summary = models.TextField(blank=True, null=True, verbose_name=_('Task summary'))
+    tasks = models.TextField(verbose_name=_('Tasks'))  # [{'name': 'task_name', 'module': '', 'args': ''}, ]
+    hosts = models.TextField(blank=True, null=True, verbose_name=_('Hosts'))  # Asset inventory may be change
+    pattern = models.CharField(max_length=64, default='all', verbose_name=_('Playbook run pattern'))
 
-    def __unicode__(self):
-        return "%s" % self.uuid
+    def __str__(self):
+        return "%s" % self.name
 
-    @property
-    def total_assets(self):
-        assets_id = [i for i in self.assets.split(',') if i.isdigit()]
-        assets = Asset.objects.filter(id__in=assets_id)
+    def get_hosts_mapped_assets(self):
+        from assets.utils import get_assets_by_id_list
+        assets_id = [i for i in self.hosts.split(',')]
+        assets = get_assets_by_id_list(assets_id)
         return assets
 
     @property
-    def assets_json(self):
-        return [asset._to_secret_json() for asset in self.total_assets]
+    def inventory(self):
+        return [asset._to_secret_json() for asset in self.get_hosts_mapped_assets()]
 
     @property
     def module_args(self):
@@ -57,3 +49,12 @@ class Task(models.Model):
         self._modules_args = json.dumps(module_args_)
 
 
+class History(models.Model):
+    uuid = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time'))
+    date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time'))
+    timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True)
+    is_finished = models.BooleanField(default=False, verbose_name=_('Is finished'))
+    is_success = models.BooleanField(default=False, verbose_name=_('Is success'))
+    result = models.TextField(blank=True, null=True, verbose_name=_('Playbook raw result'))
+    summary = models.TextField(blank=True, null=True, verbose_name=_('Playbook summary'))

apps/ops/serializers.py

@@ -2,12 +2,12 @@
 from __future__ import unicode_literals
 from rest_framework import serializers
 
-from .models import Task
+from .models import Playbook
 
 
 class TaskSerializer(serializers.ModelSerializer):
     class Meta:
-        model = Task
+        model = Playbook
         fields = '__all__'
 
 

apps/ops/tasks.py

@@ -12,8 +12,8 @@ logger = get_logger(__file__)
 
 @shared_task
 def rerun_task(task_id):
-    from .models import Task
-    record = Task.objects.get(uuid=task_id)
+    from .models import Playbook
+    record = Playbook.objects.get(uuid=task_id)
     assets = record.assets_json
     task_tuple = record.module_args
     pattern = record.pattern

apps/ops/utils.py

@@ -3,6 +3,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import json
+import re
 import time
 import uuid
 
@@ -41,16 +42,16 @@ def run_AdHoc(task_tuple, assets,
 
     runner = AdHocRunner(assets)
     if record:
-        from .models import Task
-        if not Task.objects.filter(uuid=task_id):
-            record = Task(uuid=task_id,
-                          name=task_name,
-                          assets=','.join(str(asset['id']) for asset in assets),
-                          module_args=task_tuple,
-                          pattern=pattern)
+        from .models import Playbook
+        if not Playbook.objects.filter(uuid=task_id):
+            record = Playbook(uuid=task_id,
+                              name=task_name,
+                              assets=','.join(str(asset['id']) for asset in assets),
+                              module_args=task_tuple,
+                              pattern=pattern)
             record.save()
         else:
-            record = Task.objects.get(uuid=task_id)
+            record = Playbook.objects.get(uuid=task_id)
             record.date_start = timezone.now()
             record.date_finished = None
             record.timedelta = None
@@ -76,3 +77,14 @@ def run_AdHoc(task_tuple, assets,
             record.is_success = False
         record.save()
     return summary, result
+
+
+UUID_PATTERN = re.compile(r'[0-9a-zA-Z\-]{36}')
+
+
+def is_uuid(s):
+    if UUID_PATTERN.match(s):
+        return True
+    else:
+        return False
+

apps/ops/views.py

@@ -9,13 +9,13 @@ from django.views.generic import ListView, DetailView, View
 from django.utils import timezone
 from django.shortcuts import redirect, reverse
 
-from .models import Task
+from .models import Playbook
 from ops.tasks import rerun_task
 
 
 class TaskListView(ListView):
     paginate_by = settings.CONFIG.DISPLAY_PER_PAGE
-    model = Task
+    model = Playbook
     ordering = ('-date_start',)
     context_object_name = 'task_list'
     template_name = 'ops/task_list.html'
@@ -53,7 +53,7 @@ class TaskListView(ListView):
     def get_context_data(self, **kwargs):
         context = {
             'app': 'Ops',
-            'action': 'Task record list',
+            'action': 'Playbook record list',
             'date_from': self.date_from_s,
             'date_to': self.date_to_s,
             'keyword': self.keyword,
@@ -63,13 +63,13 @@ class TaskListView(ListView):
 
 
 class TaskDetailView(DetailView):
-    model = Task
+    model = Playbook
     template_name = 'ops/task_detail.html'
 
     def get_context_data(self, **kwargs):
         context = {
             'app': 'Ops',
-            'action': 'Task record detail',
+            'action': 'Playbook record detail',
             'results': json.loads(self.object.summary or '{}'),
         }
         kwargs.update(context)

apps/templates/_nav.html

@@ -52,7 +52,7 @@
         <i class="fa fa-coffee"></i> <span class="nav-label">{% trans 'Job Center' %}</span><span class="fa arrow"></span>
     </a>
     <ul class="nav nav-second-level">
-        <li id="task"><a href="{% url 'ops:task-list' %}">{% trans 'Task' %}</a></li>
+        <li id="task"><a href="{% url 'ops:task-list' %}">{% trans 'Playbook' %}</a></li>
     </ul>
 </li>
 

apps/terminal/models.py

@@ -113,7 +113,7 @@ class Task(models.Model):
 
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, choices=NAME_CHOICES, verbose_name=_("Name"))
-    args = models.CharField(max_length=1024, verbose_name=_("Task Args"))
+    args = models.CharField(max_length=1024, verbose_name=_("Playbook Args"))
     terminal = models.ForeignKey(Terminal, null=True, on_delete=models.CASCADE)
     is_finished = models.BooleanField(default=False)
     date_created = models.DateTimeField(auto_now_add=True)