[Update] 支持上传大文件

.run_dev_flask.py

+#!/usr/bin/python
+#
+from coco.httpd import app
+from coco.logger import create_logger
+
+create_logger()
+
+if __name__ == '__main__':
+    app.run()

coco/app.py

@@ -4,8 +4,8 @@
 
 import eventlet
 from eventlet.debug import hub_prevent_multiple_readers
-eventlet.monkey_patch()
-hub_prevent_multiple_readers(False)
+# eventlet.monkey_patch()
+# hub_prevent_multiple_readers(False)
 
 import datetime
 import os
@@ -19,8 +19,7 @@ from .sshd import SSHServer
 from .httpd import HttpServer
 from .logger import create_logger
 from .tasks import TaskHandler
-from .utils import get_logger, ugettext as _, \
-    ignore_error
+from .utils import get_logger, ugettext as _, ignore_error
 from .ctx import app_service
 from .recorder import get_replay_recorder
 from .session import Session

coco/httpd/app.py

@@ -14,6 +14,7 @@ app = Flask(__name__, template_folder='templates', static_folder='static')
 app.config.update(config)
 socket_io = SocketIO()
 socket_io.on_namespace(ProxyNamespace('/ssh'))
+socket_io.on_namespace(ProxyNamespace('/coco/ssh'))
 
 # init_kwargs = {'async_mode': 'threading'}
 init_kwargs = {'async_mode': 'eventlet'}

coco/httpd/finder/connector.py → coco/httpd/elfinder/connector.py

@@ -33,12 +33,12 @@ class ElFinderConnector:
         'cmd', 'target', 'targets[]', 'current', 'tree',
         'name', 'content', 'src', 'dst', 'cut', 'init',
         'type', 'width', 'height', 'upload[]', 'dirs[]',
-        'targets'
+        'targets', "chunk", "range", "cid",
     ]
 
     _options = {
         'api': _version,
-        'uplMaxSize': '128M',
+        'uplMaxSize': '10M',
         'options': {
             'separator': '/',
             'disabled': [],
@@ -113,13 +113,6 @@ class ElFinderConnector:
         except Exception as e:
             self.response['error'] = '%s' % e
             logger.exception(e)
-        finally:
-            target = self.data['target']
-            if not target:
-                return
-            volume = self.get_volume(target)
-            if volume:
-                volume.close()
 
     def get_request_data(self):
         data_source = {}
@@ -282,7 +275,19 @@ class ElFinderConnector:
     def __upload(self):
         parent = self.data['target']
         volume = self.get_volume(parent)
-        self.response.update(volume.upload(self.request.files, parent))
+        if self.data.get('chunk') and self.data.get('cid'):
+            self.response.update(
+                volume.upload_as_chunk(
+                    self.request.files, self.data.get('chunk'), parent
+                )
+            )
+        elif self.data.get('chunk'):
+            self.response.update(
+                volume.upload_chunk_merge(parent, self.data.get('chunk'))
+            )
+        else:
+            print("__UPLOAD {}".format(self.data))
+            self.response.update(volume.upload(self.request.files, parent))
 
     def __size(self):
         target = self.data['target']

coco/httpd/finder/volumes/base.py → coco/httpd/elfinder/volumes/base.py

@@ -223,3 +223,13 @@ class BaseVolume:
             :returns: TODO
         """
         raise NotImplementedError
+
+    def upload_as_chunk(self, files, chunk_name, parent):
+        """
+        Upload a large file as chunk
+        :param files:
+        :param chunk_name:
+        :param cid:
+        :param parent:
+        :return:
+        """

coco/httpd/finder/volumes/sftp.py → coco/httpd/elfinder/volumes/sftp.py

 import logging
 import stat
+import re
 
 from flask import send_file
 
@@ -198,6 +199,36 @@ class SFTPVolume(BaseVolume):
         added.append(self._info(path))
         return {'added': added}
 
+    def upload_as_chunk(self, files, chunk_name, parent):
+        added = []
+        print("Upload chunk: {}".format(chunk_name))
+        parent_path = self._path(parent)
+        item = files.get('upload[]')
+        __tmp = chunk_name.split('.')
+        filename = '.'.join(__tmp[:-2])
+        num, total = __tmp[-2].split('_')
+        num, total = int(num), int(total)
+
+        path = self._join(parent_path, filename)
+        remote_path = self._remote_path(path)
+        if num == 0:
+            infos = self._list(parent_path)
+            files_exist = [d['name'] for d in infos]
+            if item.filename in files_exist:
+                raise OSError("File {} exits".format(remote_path))
+        with self.sftp.open(remote_path, 'a') as rf:
+            for data in item:
+                rf.write(data)
+        if num != total:
+            return {'added': added}
+        else:
+            return {'added': added, '_chunkmerged': filename, '_name': filename}
+
+    def upload_chunk_merge(self, parent, chunk):
+        parent_path = self._path(parent)
+        path = self._join(parent_path, chunk)
+        return {"added": [self._info(path)]}
+
     def size(self, target):
         info = self.info(target)
         return info.get('size') or 'Unknown'

coco/httpd/templates/finder/file_manager.html

@@ -21,7 +21,7 @@
                     ]
 	    		},
                 height: $(window).height(),
-	    		url: '{{ url_for("sftp_connector_view") }}',
+	    		url: '{{ url_for("sftp_host_connector_view", host=host) }}',
                 resizable: false,
                 lang: 'pl',
                 requestType: 'get',
@@ -39,10 +39,7 @@
                 },
                 rememberLastDir: false,
                 placesFirst: false,
-                reloadClearHistory: true,
-                handlers: {
-	    		    'open': function(event) { console.log(event.data); }
-                }
+                reloadClearHistory: true
 	    	};
 	    	var start = function(lng) {
 	    		$(function() {

coco/httpd/templates/finder/host_file_manager.html

-<html>
-<head>
-    <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-2.1.1.js') }}"></script>
-    <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-ui-1.10.4.min.js') }}"></script>
-    <link rel="stylesheet" type="text/css" media="screen" href="{{ url_for('static', filename='elfinder/css/elfinder.min.css') }}">
-    <link rel="stylesheet" type="text/css" media="screen" href="{{ url_for('static', filename='elfinder/css/theme-gray.css') }}">
-    <script type="text/javascript" src="{{ url_for('static', filename='elfinder/elfinder.min.js') }}"></script>
-    <script type="text/javascript" src="{{ url_for('static', filename='elfinder/i18n/elfinder.pl.js') }}"></script>
-</head>
-<body style="margin: 0">
-<div id="elfinder"></div>
-<script type="text/javascript" charset="utf-8">
-    $(document).ready(function () {
-        var elf;
-        var opts = {
-            uiOptions : {
-                toolbar: [
-                    ['back', 'forward'],
-                    ['mkdir', 'mkfile'],
-                    ['copy', 'cut', 'paste'],
-                    ['rm'],
-                    ['rename'],
-                    ['view']
-                ]
-            },
-            // ui: [],
-            height: $(window).height(),
-            url: '{{ url_for("sftp_host_connector_view", host=host) }}',
-            resizable: false,
-            lang: 'pl',
-            rememberLastDir: false,
-            reloadClearHistory: true,
-            contextmenu: {
-                navbar: [
-                    'rm'
-                ],
-                cwd: [
-                    'reload', 'back', '|', 'mkdir', 'mkfile', '|',
-                    'upload'
-                ],
-                files: [
-                    'rm', 'rename', 'download'
-                ]
-            },
-            placesFirst: false
-        };
-        var start = function(lng) {
-            $(function() {
-                // Make elFinder (REQUIRED)
-                opts.lang = lng;
-                elf = $('#elfinder').elfinder(opts).elfinder('instance');
-            });
-        };
-        var load = function () {
-            var loct = window.location.search;
-            var full_lng, locm, lng;
-            if (loct && (locm = loct.match(/lang=([a-zA-Z_-]+)/))) {
-                full_lng = locm[1];
-            } else {
-                full_lng = (navigator.browserLanguage || navigator.language || navigator.userLanguage);
-            }
-            lng = full_lng.substr(0,2);
-            if (lng === 'ja') {
-                lng = 'jp';
-            }
-            else if (lng === 'pt') {
-                lng = 'pt_BR';
-            }
-            else if (lng === 'zh') {
-                lng = (full_lng.substr(0,5) == 'zh-tw')? 'zh_TW' : 'zh_CN';
-            }
-            if (lng !== 'en') {
-                $.ajax({
-                    url : "{{ url_for('static', filename='elfinder/i18n/') }}"+'/elfinder.'+lng+'.js',
-                    cache : true,
-                    dataType : 'script'
-                })
-                .done(function() {
-                    start(lng);
-                })
-                .fail(function() {
-                    start('en');
-                });
-           } else {
-                start(lng);
-           }
-        };
-        load();
-        var resizeTimer;
-        $(window).resize(function () {
-            resizeTimer && clearTimeout(resizeTimer);
-            if (!$('#elfinder').hasClass('elfinder-fullscreen')) {
-                resizeTimer = setTimeout(function () {
-                    var h = parseInt($(window).height());
-                    if (h != parseInt($('#elfinder').height())) {
-                        elf.resize('100%', h);
-                    }
-                }, 200);
-            }
-        });
-    });
-</script>
-</body>
-</html>

coco/httpd/view.py

@@ -4,14 +4,14 @@
 from flask import render_template, request, jsonify
 
 from .app import app
-from .finder import connector, volumes
+from .elfinder import connector, volumes
 from ..models import Connection
 from ..sftp import InternalSFTPClient
 from .auth import login_required
 from ..service import app_service
 
 
-@app.route('/elfinder/sftp/connector/<host>/', methods=['GET', 'POST'])
+@app.route('/coco/elfinder/sftp/connector/<host>/', methods=['GET', 'POST'])
 @login_required
 def sftp_host_connector_view(host):
     user = request.current_user
@@ -19,7 +19,7 @@ def sftp_host_connector_view(host):
     connection.user = user
     sftp = InternalSFTPClient(connection)
     volume = volumes.SFTPVolume(sftp)
-    if host:
+    if host != '_':
         asset = app_service.get_asset(host)
         if not asset:
             return jsonify({'error': 'Not found this host'})
@@ -31,28 +31,20 @@ def sftp_host_connector_view(host):
     handler = connector.ElFinderConnector([volume])
     handler.run(request)
 
-    # Some commands (e.g. read file) will return a Django View - if it
-    # is set, return it directly instead of building a response
+    # If download file, return a view response
     if handler.return_view:
         return handler.return_view
-
     if handler.headers['Content-type'] == 'application/json':
         return jsonify(handler.response)
 
 
-@app.route('/elfinder/sftp/connector/', methods=['GET', 'POST'])
-@login_required
-def sftp_connector_view():
-    return sftp_host_connector_view('')
-
-
-@app.route('/elfinder/sftp/<host>/')
+@app.route('/coco/elfinder/sftp/<host>/')
 def sftp_host_finder(host):
-    return render_template('finder/host_file_manager.html', host=host)
+    return render_template('finder/file_manager.html', host=host)
 
 
-@app.route('/elfinder/sftp/')
+@app.route('/coco/elfinder/sftp/')
 def sftp_finder():
-    return render_template('finder/file_manager.html')
+    return render_template('finder/file_manager.html', host='_')
 
 

coco/sftp.py

@@ -27,11 +27,13 @@ class SFTPServer(paramiko.SFTPServerInterface):
             trans = chan.get_transport()
             sftp.close()
 
-            if [c for c in trans._channels.values() if not c.closed]:
+            active_channels = [c for c in trans._channels.values() if not c.closed]
+            if not active_channels:
+                print("CLose transport")
                 trans.close()
-            if sock:
-                sock.close()
-                sock.transport.close()
+                if sock:
+                    sock.close()
+                    sock.transport.close()
         self._sftp = {}
 
     def get_host_sftp(self, host, su):
@@ -100,7 +102,6 @@ class SFTPServer(paramiko.SFTPServerInterface):
             return False
 
     def create_ftp_log(self, path, operate, is_success=True, filename=None):
-        print("Create ftp log: {} {} {}".format(path, operate, filename))
         host, su, rpath = self.parse_path(path)
         asset = self.hosts.get(host)
         date_start = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + " +0000",
@@ -361,6 +362,7 @@ class InternalSFTPClient(SFTPServer):
             except OSError:
                 raise OSError("Open file {} failed".format(rpath))
             finally:
+                print("Open success")
                 self.create_ftp_log(path, operate, success)
             return f
 
@@ -371,4 +373,5 @@ class InternalSFTPClient(SFTPServer):
         return self.remove(path)
 
     def close(self):
+        print("End ")
         return self.session_ended()