Merge branch 'test' into 'master'

1.4 上线

See merge request !93

_celery.py

+# !/usr/bin/env python
+# encoding=utf-8
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+
+from django.conf import settings
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sun.settings')
+
+app = Celery('sun_tasks')
+app.conf.task_default_queue = 'sun_tasks'
+
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

api/__init__.py

@@ -2,5 +2,5 @@
 # -*- coding: utf-8 -*-
 # __author__ = "chenwei"
 # Date: 2018/11/15
-
+from _celery import app as celery_app
 

api/tasks/__init__.py

+from .tasks import *

api/tasks/tasks.py

+import os
+from celery import shared_task
+from django.conf import settings
+import requests
+
+from gm_upload.utils.qiniu_tool import QiniuTool
+
+from middleware.rpc import rpc_invoker
+from utils.channel_package_tools import ChannelPackage, QiniuFileTool
+from utils.logger import log_error, info_logger
+
+
+@shared_task
+def version_release_task(packed_channels, version_id=None, version=None):  # 一键发布，发布完成邮件通知
+    rpc = rpc_invoker
+    version = version
+    channel_ids = list()
+    for version2channel in packed_channels:
+        channel = version2channel['url_name']
+        channel_ids.append(version2channel.get('id'))
+        try:
+            ChannelPackage.publish_apk(channel, version)
+        except:
+            log_error()
+            raise Exception('发布失败')
+    # 发布
+    rpc['venus/sun/channel_build/version/release'](
+        version_id=version_id, channel_ids=channel_ids
+    ).unwrap()
+    #  email notify
+    # rpc.origin['endpoint'](version=version, channels=channels).unwrap()
+
+
+@shared_task
+def channel_build_one(version=None, channel=None, data=None):
+    url = ChannelPackage.apk_url('', version)
+    for _ in range(5):
+        try:
+            r = requests.get(url, stream=True)
+            info_logger.debug(url)
+            if r.content:
+                break
+        except:
+            info_logger.debug('第%s下载失败%s：', _, url)
+    chn_pkg = ChannelPackage(r.content, version)
+    build_one_channel(chn_pkg, channel)
+
+
+@shared_task
+def channel_build(version=None, channels=None, data=None, ):
+    url = ChannelPackage.apk_key('', version)
+    QiniuTool.delete(url, settings.APK_SCOPE)
+    target_path = ChannelPackage.get_parent_path(data)
+    QiniuFileTool.upload_file(target_path, url, settings.APK_SCOPE)  # 上传母包
+    if not channels:
+        # channels = [settings.DEFAULT_CHANNEL]
+        return
+
+    info_logger.debug(channels)
+    chn_pkg = ChannelPackage(data, version)
+    # 遍历渠道号并创建对应渠道号的apk文件
+    for channel in channels:
+        build_one_channel(chn_pkg, channel)
+
+
+def build_one_channel(pkg_obj, channel):
+    info_logger.debug(channel)
+    if not channel:
+        return
+    channel = channel.strip()
+    ret = pkg_obj.upload_apk(channel)
+    info_logger.debug(ret)
+    if 'file' in ret:
+        # 打包
+        rpc_invoker['venus/sun/channel_build/channel/pack'](
+            version=pkg_obj.version, url_name=channel
+        ).unwrap()

api/urls.py

@@ -19,6 +19,7 @@ from .upload import *
 from .token import *
 from .face_star import *
 from .advertise import *
+from .channel_build import *
 from .commons import *
 
 urlpatterns = [
@@ -113,6 +114,19 @@ urlpatterns = [
     url(r'^advertise/list$', AdvertiseListView.as_view()),
     url(r'^advertise/create$', AdvertiseCreateView.as_view()),
 
+    # 渠道打包相关API
+    url(r'^channel_build/channel/get', ChannelList.as_view()),
+    url(r'^channel_build/channel_version/get', ChannelVersionList.as_view()),
+    url(r'^channel_build/channel/detail', Channel.as_view()),
+    url(r'^channel_build/channel/edit', Channel.as_view()),
+    url(r'^channel_build/build', ChannelPacked.as_view()),
+    url(r'^channel_build/version/get', VersionList.as_view()),
+    url(r'^channel_build/version_channel/get', VersionChannelList.as_view()),
+    url(r'^channel_build/version/detail', VersionDetailUpdate.as_view()),
+    url(r'^channel_build/version/update', VersionDetailUpdate.as_view()),
+    url(r'^channel_build/batch/release', BatchPublishVersion.as_view()),
+    url(r'^channel_build/version/create', VersionCreate.as_view()),
+
     url(r'^suggestion/list$', SuggestionListView.as_view())
 ]
 

requirements.txt

@@ -14,4 +14,8 @@ git+ssh://git@git.wanmeizhensuo.com/backend/gm-dataquery.git@v0.2.10
 git+ssh://git@git.wanmeizhensuo.com/system/gm-tracer.git@v0.1.2
 git+ssh://git@git.wanmeizhensuo.com/alpha/alpha-types.git@master
 filetype==1.0.2
-Pillow==5.4.1
\ No newline at end of file
+Pillow==5.4.1
+celery==4.2.1
+kombu==4.2.2.post1
+requests==2.21.0
+redis==2.10.6
\ No newline at end of file

sun/settings.py

@@ -157,3 +157,10 @@ OPERATOR_PASSWORD = 123456
 
 # 图片下载的存储路径
 DOWNLOAD_IMAGE_PATH = u'/data/header-images/'
+
+# apk 七牛上传空间
+APK_SCOPE = 'download'
+APK_DOMAIN = 'http://dl.igengmei.com/'
+DEFAULT_CHANNEL = 'benzhan'
+APK_RELEASE_DIR = 'test'
+APK_BUILD_DIR = 'testbuild'

utils/channel_package_tools.py

+# -*- coding: UTF-8 -*-
+import os
+import re
+
+from django.conf import settings
+from gm_upload.utils.qiniu_tool import QiniuTool
+from qiniu import put_file
+
+from utils.logger import log_error, info_logger
+
+
+class QiniuFileTool(QiniuTool):
+    @classmethod
+    def upload_file(cls, file_path, save_name, bucket_name):
+        """
+        上传文件到七牛
+        :param file_path: 文本路径
+        :param save_name: 上传的名字
+        :param bucket_name: 空间名
+        :return:
+        """
+        token = cls.q.upload_token(bucket_name)
+
+        for _ in range(5):  # 多次尝试
+            ret, response_info = put_file(token, save_name, file_path)
+
+            if ret and "key" in ret and response_info.status_code == 200:
+                # assert ret["key"] == save_name
+                # assert ret["hash"] == etag(file_path)
+
+                return {
+                    "file": ret["key"]
+                }
+
+        raise Exception('upload filed')
+
+
+class ChannelPackage(object):
+    # 渠道识别前缀
+    channel_prefix = 'gm_alpha'
+
+    def __init__(self, api_data, version):
+        self.apk_data = api_data
+        self.version = version
+        self.apk = None
+
+    @classmethod
+    def get_parent_path(cls, data):  # 母包地址
+        path = os.getcwd() + '/' + cls.channel_prefix + '.apk'
+        with open(path, 'wb')as fw:
+            fw.write(data)
+        return path
+
+    def build_channel_apk(self, channel_name):
+        cp = re.compile(r'[^{}]+')
+        self.apk = self.channel_prefix + '.apk'
+        walle_cli_path = os.getcwd() + '/walle-cli-all.jar'
+        with open(self.apk, 'wb') as fw:
+            fw.write(self.apk_data)
+        os.system('java -jar {0} batch -c {1} {2}'.format(walle_cli_path, channel_name, self.apk))
+        targe_apk = self.channel_prefix + '_' + channel_name + '.apk'
+        cmd_res = os.popen('java -jar walle-cli-all.jar show %s' % targe_apk)
+        ret_res = cmd_res.readlines()[0]
+        target_channel_info = cp.findall(ret_res)[1]
+        if target_channel_info and channel_name == target_channel_info.split('=')[1]:
+            target_path = self.channel_prefix + '_' + channel_name + '.apk'
+            os.remove(self.apk)
+            return target_path
+        else:
+            log_error()
+        os.remove(self.apk)
+        os.remove(self.channel_prefix + '_' + channel_name + '.apk')
+
+    def upload_apk(self, channel_name):
+        chn_apk_path = self.build_channel_apk(channel_name)
+        apk_uri = self.apk_key(channel_name, self.version)
+        QiniuTool.delete(apk_uri, settings.APK_SCOPE)
+        ret = QiniuFileTool.upload_file(chn_apk_path, apk_uri, settings.APK_SCOPE)
+        os.remove(chn_apk_path)
+        return ret
+
+    @classmethod
+    def apk_key(cls, channel, version):
+        build_dir = getattr(settings, 'APK_BUILD_DIR', '')
+        if channel:
+            uri = '{version}/{channel}/gm_alpha_{channel}.apk'.format(version=version, channel=channel)
+        else:
+            uri = '{version}/gm_alpha.apk'.format(version=version)
+        if build_dir:
+            uri = '{build_dir}/{uri}'.format(build_dir=build_dir, uri=uri)
+        return uri
+
+    @classmethod
+    def apk_url(cls, channel, version):
+        key = cls.apk_key(channel, version)
+        url = settings.APK_DOMAIN + key
+        return url
+
+    @classmethod
+    def publish_apk(cls, channel, version, need_refresh=True):
+        apk_scope = settings.APK_SCOPE
+        version_key = cls.apk_key(channel, version)
+        current_key = cls.apk_key(channel, settings.APK_RELEASE_DIR)
+        ret, info = QiniuTool.bucket.stat(apk_scope, current_key)
+        info_logger.debug(
+            {
+                'channel': channel,
+                'version': version,
+                'version_key': version_key,
+                'current_key': current_key,
+                'ret': ret
+            }
+        )
+        if ret:
+            QiniuTool.delete(current_key, apk_scope)
+        QiniuTool.copy(version_key, current_key, apk_scope, apk_scope)
+        if need_refresh:
+            QiniuTool.refresh(['http://qiniu.dl.igengmei.com/' + current_key])
+        return True