Merge branch 'test' into 'master'

1.4 上线

See merge request alpha/sun!93

_celery.py

+# !/usr/bin/env python
+# encoding=utf-8
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+
+from django.conf import settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sun.settings')
+
+app = Celery('sun_tasks')
+app.conf.task_default_queue = 'sun_tasks'
+
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

api/__init__.py

@@ -2,5 +2,5 @@
 # -*- coding: utf-8 -*-
 # __author__ = "chenwei"
 # Date: 2018/11/15
-
+from _celery import app as celery_app
 

api/channel_build.py

+import hashlib
+
+from django.conf import settings
+from gm_types.ascle import ERROR
+from helios.rpc.exceptions import RPCFaultException
+
+from utils.base import APIView
+from utils.channel_package_tools import ChannelPackage
+from utils.logger import error_logger
+from .tasks import version_release_task, channel_build, channel_build_one
+
+
+def get_md5_value(src):
+    md5 = hashlib.md5()
+    md5.update(src)
+    md5_digest = md5.hexdigest()
+    return md5_digest
+
+
+class Vendor(APIView):
+    def get(self, request):
+        """
+        获取渠道商列表
+        :param request:
+        :return:
+        """
+        query = request.GET.get('q', '')
+        page = int(request.GET.get('page', 1))
+        offset = int(request.GET.get('offset', 30))
+
+        data = request.rpc.origin['endpoint'](
+            query=query, start=(page - 1) * offset, offset=offset
+        ).unwrap()
+        return data
+
+
+class ChannelList(APIView):
+    def get(self, request):
+        """
+        渠道列表
+        :param request:
+        :return:
+        """
+        page = int(request.GET.get('page', 1))
+        limit = int(request.GET.get('limit', 10))
+        filters = self.handle_filter(request.GET.get('filter', ""))
+        try:
+            channel_list = self.rpc['venus/sun/channel_build/channel/list'](
+                offset=(page-1)*limit, limit=limit, filters=filters
+            ).unwrap()
+        except Exception as e:
+            error_logger.error(u'获取渠道列表失败：%s', e)
+            raise e
+        data = {'total': channel_list.get('total', 0)}
+        url = settings.APK_DOMAIN + settings.APK_BUILD_DIR + '/' + settings.APK_RELEASE_DIR + "/{name}/gm_alpha_{name}.apk"
+        channels = [
+            {
+                'id': data.get('id'),
+                'name': data.get('name'),
+                'url_name': data.get('url_name'),
+                'pack': data.get('pack'),
+                'released_version': data.get('released_version'),
+                'download_url': url.format(name=data.get('url_name')),
+            }
+            for data in channel_list.get('data', [])]
+        data.update({'channel': channels})
+        return self.write_success(message='获取渠道列表成功', data=data)
+
+
+class ChannelVersionList(APIView):
+    def get(self, request):
+        """
+        :param request:
+        :return:
+        """
+        channel_id = request.GET.get('channel_id')
+        page = int(request.GET.get('page', 1))
+        limit = int(request.GET.get('limit', 10))
+        try:
+            channel_list = self.rpc['venus/sun/channel_build/channel_version/list'](
+                offset=(page-1)*limit, limit=limit, channel_id=channel_id
+            ).unwrap()
+        except Exception as e:
+            error_logger.error(u'获取渠道列表失败：%s', e)
+            raise e
+        data = {'total': channel_list.get('total', 0)}
+        url = settings.APK_DOMAIN + settings.APK_BUILD_DIR + "/{version}/{name}/gm_alpha_{name}.apk"
+        channels = [
+            {
+                'version_id': data.get('version_id'),
+                'channel_id': data.get('channel_id'),
+                'version': data.get('version'),
+                'name': data.get('name'),
+                'url_name': data.get('url_name'),
+                'packed_time': data.get('packed_time'),
+                'packed': data.get('packed'),
+                'release_time': data.get('release_time'),
+                'is_released': data.get('is_released'),
+                'download_url': url.format(version=data.get('version'), name=data.get('url_name')),
+            }
+            for data in channel_list.get('data', [])]
+        data.update({'channel': channels})
+        return self.write_success(message='获取渠道列表成功', data=data)
+
+
+class Channel(APIView):
+    def get(self, request):
+        """
+        渠道详情
+        :param request:
+        :return:
+        """
+        channel_id = request.GET.get('channel_id')
+        if not channel_id:
+            error_logger.error(u'参数不完整')
+            return self.write_fail(code=ERROR.ARG_MISS, message='参数不完整')
+        try:
+            channel_data = self.rpc['venus/sun/channel_build/channel/get'](
+                id=channel_id
+            ).unwrap()
+        except Exception as e:
+            error_logger.error(u'获取渠道详情失败：%s', e)
+            raise e
+        channel = {
+            'id': channel_data.get('id'),
+            'name': channel_data.get('name'),
+            'url_name': channel_data.get('url_name'),
+            'pack': channel_data.get('pack'),
+            'type': channel_data.get('type'),
+            'released_version': channel_data.get('released_version'),
+        }
+        return self.write_success(message='获取渠道详情成功', data=channel)
+
+    def post(self, request):
+        """
+        渠道创建 / 编辑
+        :param request:
+        :return:
+        """
+        channel_id = request.POST.get('id', None)
+        channel_name = request.POST.get('name', '')
+        url_name = request.POST.get('url_name', '')
+        auto_pack = request.POST.get('pack', True)
+        channel_type = request.POST.get('type', 2)
+        if not all([channel_name, url_name, auto_pack, channel_type]):
+            error_logger.error(u'参数不完整')
+            return self.write_fail(code=ERROR.ARG_MISS, message='参数不完整')
+        if channel_id:
+            channel = self.rpc['venus/sun/channel_build/channel/get'](
+                id=channel_id
+            ).unwrap()
+        else:
+            channel = {}
+        data = dict()
+        if channel_name:
+            data.update({'name': channel_name})
+        if url_name:
+            data.update({'url_name': url_name})
+        if auto_pack:
+            data.update({'pack': auto_pack})
+        if channel_type:
+            data.update({'type': channel_type})
+        try:
+            resp = self.rpc['venus/sun/channel_build/channel/edit'](
+                id=channel_id, data=data
+            ).unwrap()
+        except Exception as e:
+            error_logger.error('操作失败%s', e)
+            return self.write_fail(message='路径名不能重复', code=-1)
+        version = resp.get('version', '')
+        if version and channel.get('url_name') != url_name:
+            channel_build_one.delay(version, url_name)
+        return self.write_success(message='操作成功', data={})
+
+
+class ChannelPacked(APIView):
+    def post(self, request):
+        """
+        当前版本的渠道包发布
+        :param request:
+        :return:
+        """
+        version_id = request.POST.get('version_id')
+        channel_id = request.POST.get('channel_id')
+        if not all([channel_id, version_id]):
+            error_logger.error(u'参数不完整')
+            return self.write_fail(code=ERROR.ARG_MISS, message='参数不完整')
+
+        channel = self.rpc['venus/sun/channel_build/channel/get'](
+            id=channel_id
+        ).unwrap()
+        version = self.rpc['venus/sun/channel_build/version/get'](
+            id=version_id
+        ).unwrap()
+        ChannelPackage.publish_apk(channel.get('url_name'), version.get('version'))
+        try:
+            self.rpc['venus/sun/channel_build/version/release'](
+                version_id=version_id, channel_ids=[channel_id]
+            ).unwrap()  #发布
+        except RPCFaultException as e:
+            error_logger.error(u'发布失败：%s', e)
+            return self.write_fail(code=-1, message='发布失败！')
+        return self.write_success(data={}, message='发布成功！')
+
+
+class VersionList(APIView):
+    def get(self, request):
+        """
+        所有版本列表
+        :param request:
+        :return:
+        """
+        page = int(request.GET.get('page', 1))
+        offset = int(request.GET.get('limit', 10))
+        try:
+            version_list = self.rpc['venus/sun/channel_build/version/list'](
+                offset=(page-1)*offset, limit=offset
+            ).unwrap()
+        except Exception as e:
+            error_logger.error(u'获取版本列表失败：%s', e)
+            raise e
+        data = {'total': version_list.get('total', 0)}
+        versions = [
+            {
+                'version_id': item.get('id'),
+                'version': item.get('version'),
+                'md5': item.get('md5'),
+                'release_time': item.get('release_time'),
+            } for item in version_list.get('data', [])
+        ]
+        data.update({'data': versions})
+        return self.write_success(data=data)
+
+
+class VersionChannelList(APIView):
+    def get(self, request):
+        page = int(request.GET.get('page', 1))
+        offset = int(request.GET.get('limit', 10))
+        version_id = request.GET.get('version_id', 10)
+        try:
+            version_list = self.rpc['venus/sun/channel_build/version_channel/list'](
+                offset=(page-1)*offset, limit=offset, version_id=version_id
+            ).unwrap()
+        except Exception as e:
+            error_logger.error(u'获取版本列表失败：%s', e)
+            raise e
+        data = {'total': version_list.get('total', 0)}
+        url = settings.APK_DOMAIN + settings.APK_BUILD_DIR + "/{version}/{name}/gm_alpha_{name}.apk"
+        versions = {
+            'version': [{
+                'channel_id': item.get('channel_id'),
+                'version_id': item.get('version_id'),
+                'version': item.get('version'),
+                'name': item.get('name'),
+                'packed_time': item.get('packed_time'),
+                'packed': item.get('packed'),
+                'download_url': url.format(version=item.get('version'), name=item.get('url_name')),
+                'release_time': item.get('release_time'),
+                'is_released': item.get('is_released'),
+            } for item in version_list.get('data', [])],
+            'base_apk_url': ChannelPackage.apk_url(channel='', version=version_list.get('version')),
+        }
+        data.update({'data': versions})
+        return self.write_success(data=data)
+
+
+class VersionDetailUpdate(APIView):
+    def get(self, request):
+        """
+        版本详情
+        :param request:
+        :return:
+        """
+        version_id = request.GET.get('version_id')
+        if not version_id:
+            error_logger.error(u'参数不完整')
+            return self.write_fail(code=ERROR.ARG_MISS, message='参数不完整')
+        version_info = self.rpc['venus/sun/channel_build/version/get'](
+            id=version_id
+        ).unwrap()
+        data = {
+            'id': version_info.get('id'),
+            'version': version_info.get('version'),
+            'release_time': version_info.get('release_time'),
+            'md5': version_info.get('md5'),
+            'description': version_info.get('description'),
+        }
+        return self.write_success(data=data, message='操作成功')
+
+    def post(self, request):
+        """
+        版本信息编辑 只允许修改描述
+        :param request:
+        :return:
+        """
+        version_id = request.POST.get('id')
+        version = request.POST.get('version')
+        md5 = request.POST.get('md5')
+        release_time = request.POST.get('release_time')
+        desc = request.POST.get('description')
+        if not version_id:
+            error_logger.error(u'参数不完整')
+            return self.write_fail(code=ERROR.ARG_MISS, message='参数不完整')
+        data = dict()
+        if version:
+            data.update({'version': version})
+        if release_time:
+            data.update({'release_time': release_time})
+        if desc:
+            data.update({'description': desc})
+        if md5:
+            data.update({'md5': md5})
+        try:
+            self.rpc['venus/sun/channel_build/version/edit'](
+                id=version_id, data=data
+            ).unwrap()
+        except Exception as e:
+            error_logger.error(e)
+            return self.write_fail(code=-1, message='操作失败')
+        return self.write_success(data={}, message='操作成功')
+
+
+class BatchPublishVersion(APIView):
+    def post(self, request):
+        """
+        批量发布 并邮件通知
+        :param request:
+        :return:
+        """
+        version_id = request.POST.get('version_id')
+        if not version_id:
+            error_logger.error(u'参数不完整')
+            return self.write_fail(code=ERROR.ARG_MISS, message='参数不完整')
+        try:
+            # 获取当前版本所有已打包渠道
+            channel_info = self.rpc['venus/sun/channel_build/version/get_channel_info'](
+                version_id=version_id
+            ).unwrap()
+        except RPCFaultException as e:
+            error_logger.error(u'获取渠道失败%s', e)
+            return self.write_fail(code=-1, message='获取渠道失败！')
+        packed_channels = list()
+        for item in channel_info.get('data', []):
+            if item.get('pack', False):
+                packed_channels.append(item)
+        version_release_task.delay(
+            packed_channels, version_id=version_id, version=channel_info.get('version')
+        )
+        return self.write_success(data={}, message='操作成功')
+
+
+class VersionCreate(APIView):
+    def post(self, request):
+        """
+        版本创建  供脚本使用
+        :param request:
+        :return:
+        """
+        apk = request.FILES.get('file').read()
+        file_md5 = get_md5_value(apk)
+        version = request.POST.get('version')
+        md5 = request.POST.get('md5')
+        if md5 != file_md5:
+            raise Exception('md5校验失败, 文件md5{}'.format(file_md5))
+        data = dict()
+        if version:
+            data.update({'version': version})
+        if md5:
+            data.update({'md5': md5})
+        try:
+            version_id = self.rpc['venus/sun/channel_build/version/edit'](
+                data=data
+            ).unwrap()
+        except RPCFaultException as e:
+            error_logger.error('创建版本失败：%s', e)
+            raise Exception('创建版本失败！')
+        channel_info = self.rpc['venus/sun/channel_build/version/get_channel_info'](
+            version_id=version_id
+        ).unwrap()  # 所有渠道的url_name: list()
+        url_names = []
+        for item in channel_info.get('data', []):
+            if item.get('pack', False):
+                url_names.append(item.get('url_name'))
+        channel_build.delay(version, url_names, apk)
+        return {'success': True}

api/tasks/__init__.py

+from .tasks import *

api/tasks/tasks.py

+import os
+from celery import shared_task
+from django.conf import settings
+import requests
+
+from gm_upload.utils.qiniu_tool import QiniuTool
+
+from middleware.rpc import rpc_invoker
+from utils.channel_package_tools import ChannelPackage, QiniuFileTool
+from utils.logger import log_error, info_logger
+
+
+@shared_task
+def version_release_task(packed_channels, version_id=None, version=None):  # 一键发布，发布完成邮件通知
+    rpc = rpc_invoker
+    version = version
+    channel_ids = list()
+    for version2channel in packed_channels:
+        channel = version2channel['url_name']
+        channel_ids.append(version2channel.get('id'))
+        try:
+            ChannelPackage.publish_apk(channel, version)
+        except:
+            log_error()
+            raise Exception('发布失败')
+    # 发布
+    rpc['venus/sun/channel_build/version/release'](
+        version_id=version_id, channel_ids=channel_ids
+    ).unwrap()
+    #  email notify
+    # rpc.origin['endpoint'](version=version, channels=channels).unwrap()
+
+
+@shared_task
+def channel_build_one(version=None, channel=None, data=None):
+    url = ChannelPackage.apk_url('', version)
+    for _ in range(5):
+        try:
+            r = requests.get(url, stream=True)
+            info_logger.debug(url)
+            if r.content:
+                break
+        except:
+            info_logger.debug('第%s下载失败%s：', _, url)
+    chn_pkg = ChannelPackage(r.content, version)
+    build_one_channel(chn_pkg, channel)
+
+
+@shared_task
+def channel_build(version=None, channels=None, data=None, ):
+    url = ChannelPackage.apk_key('', version)
+    QiniuTool.delete(url, settings.APK_SCOPE)
+    target_path = ChannelPackage.get_parent_path(data)
+    QiniuFileTool.upload_file(target_path, url, settings.APK_SCOPE)  # 上传母包
+    if not channels:
+        # channels = [settings.DEFAULT_CHANNEL]
+        return
+
+    info_logger.debug(channels)
+    chn_pkg = ChannelPackage(data, version)
+    # 遍历渠道号并创建对应渠道号的apk文件
+    for channel in channels:
+        build_one_channel(chn_pkg, channel)
+
+
+def build_one_channel(pkg_obj, channel):
+    info_logger.debug(channel)
+    if not channel:
+        return
+    channel = channel.strip()
+    ret = pkg_obj.upload_apk(channel)
+    info_logger.debug(ret)
+    if 'file' in ret:
+        # 打包
+        rpc_invoker['venus/sun/channel_build/channel/pack'](
+            version=pkg_obj.version, url_name=channel
+        ).unwrap()

api/urls.py

@@ -19,6 +19,7 @@ from .upload import *
 from .token import *
 from .face_star import *
 from .advertise import *
+from .channel_build import *
 from .commons import *
 
 urlpatterns = [
@@ -113,6 +114,19 @@ urlpatterns = [
     url(r'^advertise/list$', AdvertiseListView.as_view()),
     url(r'^advertise/create$', AdvertiseCreateView.as_view()),
 
+    # 渠道打包相关API
+    url(r'^channel_build/channel/get', ChannelList.as_view()),
+    url(r'^channel_build/channel_version/get', ChannelVersionList.as_view()),
+    url(r'^channel_build/channel/detail', Channel.as_view()),
+    url(r'^channel_build/channel/edit', Channel.as_view()),
+    url(r'^channel_build/build', ChannelPacked.as_view()),
+    url(r'^channel_build/version/get', VersionList.as_view()),
+    url(r'^channel_build/version_channel/get', VersionChannelList.as_view()),
+    url(r'^channel_build/version/detail', VersionDetailUpdate.as_view()),
+    url(r'^channel_build/version/update', VersionDetailUpdate.as_view()),
+    url(r'^channel_build/batch/release', BatchPublishVersion.as_view()),
+    url(r'^channel_build/version/create', VersionCreate.as_view()),
+
     url(r'^suggestion/list$', SuggestionListView.as_view())
 ]
 

requirements.txt

@@ -15,3 +15,7 @@ git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
 git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
 filetype==1.0.2
 Pillow==5.4.1
+celery==4.2.1
+kombu==4.2.2.post1
+requests==2.21.0
+redis==2.10.6
\ No newline at end of file

sun/settings.py

@@ -157,3 +157,10 @@ OPERATOR_PASSWORD = 123456
 
 # 图片下载的存储路径
 DOWNLOAD_IMAGE_PATH = u'/data/header-images/'
+
+# apk 七牛上传空间
+APK_SCOPE = 'download'
+APK_DOMAIN = 'http://dl.igengmei.com/'
+DEFAULT_CHANNEL = 'benzhan'
+APK_RELEASE_DIR = 'test'
+APK_BUILD_DIR = 'testbuild'

utils/channel_package_tools.py

+# -*- coding: UTF-8 -*-
+import os
+import re
+
+from django.conf import settings
+from gm_upload.utils.qiniu_tool import QiniuTool
+from qiniu import put_file
+
+from utils.logger import log_error, info_logger
+
+
+class QiniuFileTool(QiniuTool):
+    @classmethod
+    def upload_file(cls, file_path, save_name, bucket_name):
+        """
+        上传文件到七牛
+        :param file_path: 文本路径
+        :param save_name: 上传的名字
+        :param bucket_name: 空间名
+        :return:
+        """
+        token = cls.q.upload_token(bucket_name)
+
+        for _ in range(5):  # 多次尝试
+            ret, response_info = put_file(token, save_name, file_path)
+
+            if ret and "key" in ret and response_info.status_code == 200:
+                # assert ret["key"] == save_name
+                # assert ret["hash"] == etag(file_path)
+
+                return {
+                    "file": ret["key"]
+                }
+
+        raise Exception('upload filed')
+
+
+class ChannelPackage(object):
+    # 渠道识别前缀
+    channel_prefix = 'gm_alpha'
+
+    def __init__(self, api_data, version):
+        self.apk_data = api_data
+        self.version = version
+        self.apk = None
+
+    @classmethod
+    def get_parent_path(cls, data):  # 母包地址
+        path = os.getcwd() + '/' + cls.channel_prefix + '.apk'
+        with open(path, 'wb')as fw:
+            fw.write(data)
+        return path
+
+    def build_channel_apk(self, channel_name):
+        cp = re.compile(r'[^{}]+')
+        self.apk = self.channel_prefix + '.apk'
+        walle_cli_path = os.getcwd() + '/walle-cli-all.jar'
+        with open(self.apk, 'wb') as fw:
+            fw.write(self.apk_data)
+        os.system('java -jar {0} batch -c {1} {2}'.format(walle_cli_path, channel_name, self.apk))
+        targe_apk = self.channel_prefix + '_' + channel_name + '.apk'
+        cmd_res = os.popen('java -jar walle-cli-all.jar show %s' % targe_apk)
+        ret_res = cmd_res.readlines()[0]
+        target_channel_info = cp.findall(ret_res)[1]
+        if target_channel_info and channel_name == target_channel_info.split('=')[1]:
+            target_path = self.channel_prefix + '_' + channel_name + '.apk'
+            os.remove(self.apk)
+            return target_path
+        else:
+            log_error()
+        os.remove(self.apk)
+        os.remove(self.channel_prefix + '_' + channel_name + '.apk')
+
+    def upload_apk(self, channel_name):
+        chn_apk_path = self.build_channel_apk(channel_name)
+        apk_uri = self.apk_key(channel_name, self.version)
+        QiniuTool.delete(apk_uri, settings.APK_SCOPE)
+        ret = QiniuFileTool.upload_file(chn_apk_path, apk_uri, settings.APK_SCOPE)
+        os.remove(chn_apk_path)
+        return ret
+
+    @classmethod
+    def apk_key(cls, channel, version):
+        build_dir = getattr(settings, 'APK_BUILD_DIR', '')
+        if channel:
+            uri = '{version}/{channel}/gm_alpha_{channel}.apk'.format(version=version, channel=channel)
+        else:
+            uri = '{version}/gm_alpha.apk'.format(version=version)
+        if build_dir:
+            uri = '{build_dir}/{uri}'.format(build_dir=build_dir, uri=uri)
+        return uri
+
+    @classmethod
+    def apk_url(cls, channel, version):
+        key = cls.apk_key(channel, version)
+        url = settings.APK_DOMAIN + key
+        return url
+
+    @classmethod
+    def publish_apk(cls, channel, version, need_refresh=True):
+        apk_scope = settings.APK_SCOPE
+        version_key = cls.apk_key(channel, version)
+        current_key = cls.apk_key(channel, settings.APK_RELEASE_DIR)
+        ret, info = QiniuTool.bucket.stat(apk_scope, current_key)
+        info_logger.debug(
+            {
+                'channel': channel,
+                'version': version,
+                'version_key': version_key,
+                'current_key': current_key,
+                'ret': ret
+            }
+        )
+        if ret:
+            QiniuTool.delete(current_key, apk_scope)
+        QiniuTool.copy(version_key, current_key, apk_scope, apk_scope)
+        if need_refresh:
+            QiniuTool.refresh(['http://qiniu.dl.igengmei.com/' + current_key])
+        return True