Merged in dev (pull request #105)

Dev

.run_dev_flask.py

+#!/usr/bin/python
+#
+from coco.httpd import app, socket_io
+from coco.logger import create_logger
+
+create_logger()
+
+if __name__ == '__main__':
+    socket_io.run(app, debug=False)

coco/app.py

@@ -19,8 +19,7 @@ from .sshd import SSHServer
 from .httpd import HttpServer
 from .logger import create_logger
 from .tasks import TaskHandler
-from .utils import get_logger, ugettext as _, \
-    ignore_error
+from .utils import get_logger, ugettext as _, ignore_error
 from .ctx import app_service
 from .recorder import get_replay_recorder
 from .session import Session

coco/httpd/__init__.py

 # -*- coding: utf-8 -*-
 #
 
-from .app import HttpServer, app
+from .app import HttpServer, app, socket_io
 from . import view

coco/httpd/app.py

@@ -6,7 +6,7 @@ from flask import Flask
 
 from coco.utils import get_logger
 from coco.config import config
-from coco.httpd.ws import ProxyNamespace
+from coco.httpd.ws import ProxyNamespace, ElfinderNamespace
 
 logger = get_logger(__file__)
 
@@ -14,9 +14,10 @@ app = Flask(__name__, template_folder='templates', static_folder='static')
 app.config.update(config)
 socket_io = SocketIO()
 socket_io.on_namespace(ProxyNamespace('/ssh'))
+socket_io.on_namespace(ElfinderNamespace('/elfinder'))
 
 # init_kwargs = {'async_mode': 'threading'}
-init_kwargs = {'async_mode': 'eventlet'}
+init_kwargs = {'async_mode': 'eventlet',}
 socket_io.init_app(app, **init_kwargs)
 socket_io.on_error_default(lambda x: logger.exception(x))
 

coco/httpd/finder/connector.py → coco/httpd/elfinder/connector.py

 import logging
+import json
 
 logger = logging.getLogger(__name__)
 
@@ -33,12 +34,12 @@ class ElFinderConnector:
         'cmd', 'target', 'targets[]', 'current', 'tree',
         'name', 'content', 'src', 'dst', 'cut', 'init',
         'type', 'width', 'height', 'upload[]', 'dirs[]',
-        'targets'
+        'targets', "chunk", "range", "cid", 'reload',
     ]
 
     _options = {
         'api': _version,
-        'uplMaxSize': '128M',
+        'uplMaxSize': '10M',
         'options': {
             'separator': '/',
             'disabled': [],
@@ -113,13 +114,6 @@ class ElFinderConnector:
         except Exception as e:
             self.response['error'] = '%s' % e
             logger.exception(e)
-        finally:
-            target = self.data['target']
-            if not target:
-                return
-            volume = self.get_volume(target)
-            if volume:
-                volume.close()
 
     def get_request_data(self):
         data_source = {}
@@ -221,10 +215,11 @@ class ElFinderConnector:
             volume = list(self.volumes.values())[0]
         else:
             volume = self.get_volume(target)
+
         self.response['cwd'] = volume.info(target)
 
         files = volume.list(target)
-        if 'tree' in self.data:
+        if 'tree' in self.data or 'reload' in self.data:
             parents = volume.parents(target, depth=0)
             parents = filter(lambda x: x not in files, parents)
             files += parents
@@ -282,7 +277,18 @@ class ElFinderConnector:
     def __upload(self):
         parent = self.data['target']
         volume = self.get_volume(parent)
-        self.response.update(volume.upload(self.request.files, parent))
+        if self.data.get('chunk') and self.data.get('cid'):
+            self.response.update(
+                volume.upload_as_chunk(
+                    self.request.files, self.data.get('chunk'), parent
+                )
+            )
+        elif self.data.get('chunk'):
+            self.response.update(
+                volume.upload_chunk_merge(parent, self.data.get('chunk'))
+            )
+        else:
+            self.response.update(volume.upload(self.request.files, parent))
 
     def __size(self):
         target = self.data['target']

coco/httpd/finder/volumes/base.py → coco/httpd/elfinder/volumes/base.py

@@ -223,3 +223,13 @@ class BaseVolume:
             :returns: TODO
         """
         raise NotImplementedError
+
+    def upload_as_chunk(self, files, chunk_name, parent):
+        """
+        Upload a large file as chunk
+        :param files:
+        :param chunk_name:
+        :param cid:
+        :param parent:
+        :return:
+        """

coco/httpd/finder/volumes/sftp.py → coco/httpd/elfinder/volumes/sftp.py

-import logging
 import stat
+import threading
 
 from flask import send_file
 
+from coco.utils import get_logger
 from .base import BaseVolume
 
-
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
 
 
 class SFTPVolume(BaseVolume):
@@ -15,6 +15,7 @@ class SFTPVolume(BaseVolume):
         self.root_name = 'Home'
         super().__init__()
         self._stat_cache = {}
+        self.lock = threading.Lock()
 
     def close(self):
         self.sftp.close()
@@ -94,7 +95,7 @@ class SFTPVolume(BaseVolume):
         """ Get the sub directory of directory
         """
         path = self._path(target)
-        # print("Tree {} {}".format(target, path))
+        print("Tree {} {}".format(target, path))
         infos = self.list(target)
         tree = list(filter(lambda x: x['mime'] == 'directory', infos))
         return tree
@@ -198,6 +199,36 @@ class SFTPVolume(BaseVolume):
         added.append(self._info(path))
         return {'added': added}
 
+    def upload_as_chunk(self, files, chunk_name, parent):
+        added = []
+        print("Upload chunk: {}".format(chunk_name))
+        parent_path = self._path(parent)
+        item = files.get('upload[]')
+        __tmp = chunk_name.split('.')
+        filename = '.'.join(__tmp[:-2])
+        num, total = __tmp[-2].split('_')
+        num, total = int(num), int(total)
+
+        path = self._join(parent_path, filename)
+        remote_path = self._remote_path(path)
+        if num == 0:
+            infos = self._list(parent_path)
+            files_exist = [d['name'] for d in infos]
+            if item.filename in files_exist:
+                raise OSError("File {} exits".format(remote_path))
+        with self.sftp.open(remote_path, 'a') as rf:
+            for data in item:
+                rf.write(data)
+        if num != total:
+            return {'added': added}
+        else:
+            return {'added': added, '_chunkmerged': filename, '_name': filename}
+
+    def upload_chunk_merge(self, parent, chunk):
+        parent_path = self._path(parent)
+        path = self._join(parent_path, chunk)
+        return {"added": [self._info(path)]}
+
     def size(self, target):
         info = self.info(target)
         return info.get('size') or 'Unknown'

coco/httpd/templates/finder/file_manager.html → coco/httpd/templates/elfinder/file_manager.html

@@ -6,8 +6,17 @@
     <link rel="stylesheet" type="text/css" media="screen" href="{{ url_for('static', filename='elfinder/css/theme-gray.css') }}">
     <script type="text/javascript" src="{{ url_for('static', filename='elfinder/elfinder.full.js') }}"></script>
     <script type="text/javascript" src="{{ url_for('static', filename='elfinder/i18n/elfinder.pl.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/socket.io.js') }}"></script>
     <script type="text/javascript" charset="utf-8">
-        $().ready(function () {
+        var socket = io.connect('/elfinder');
+        socket.on('connect', function () {
+            console.log("Connect websocket done")
+        });
+        socket.on('data', function (msg) {
+            var sid = msg.sid;
+            init_elfinder(sid);
+        });
+        function init_elfinder(sid) {
             var elf;
             var opts = {
 	    		uiOptions : {
@@ -20,8 +29,10 @@
                         ['view']
                     ]
 	    		},
-                height: $(window).height(),
-	    		url: '{{ url_for("sftp_connector_view") }}',
+                customData: {'sid': sid},
+                height: '100%',
+                width: '100%',
+	    		url: '{{ url_for("sftp_host_connector_view", host=host) }}',
                 resizable: false,
                 lang: 'pl',
                 requestType: 'get',
@@ -39,16 +50,14 @@
                 },
                 rememberLastDir: false,
                 placesFirst: false,
-                reloadClearHistory: true,
-                handlers: {
-	    		    'open': function(event) { console.log(event.data); }
-                }
+                reloadClearHistory: true
 	    	};
 	    	var start = function(lng) {
 	    		$(function() {
 	    			// Make elFinder (REQUIRED)
                     opts.lang = lng;
 	    			elf = $('#elfinder').elfinder(opts).elfinder('instance');
+                    $(window).trigger('resize');
 	    		});
 	    	};
 	    	var load = function () {
@@ -84,8 +93,6 @@
 	           } else {
 	           	    start(lng);
 	           }
-
-
             };
 	    	load();
             var resizeTimer;
@@ -93,14 +100,26 @@
                 resizeTimer && clearTimeout(resizeTimer);
                 if (!$('#elfinder').hasClass('elfinder-fullscreen')) {
                     resizeTimer = setTimeout(function () {
-                        var h = parseInt($(window).height());
-                        if (h != parseInt($('#elfinder').height())) {
-                            elf.resize('100%', h);
+                        var h, w;
+                        var isTrue = window == parent;
+                        console.log("Window == parent" + isTrue);
+                        if (window != parent) {
+                            h = parseInt(parent.$('.window.active').height());
+                            w = parseInt(parent.$('.window.active').width());
+                        } else {
+                            h = parseInt($(window).height());
+                            w = parseInt($(window).width());
+                        }
+                        var ori_h = parseInt($('#elfinder').height());
+                        var ori_w = parseInt($('#elfinder').width());
+                        console.log("Height: " + h + " Wid: " + w);
+                        if (h !== ori_h || w != ori_w){
+                            elf.resize(w, h);
                         }
                     }, 200);
                 }
             });
-        });
+        }
     </script>
 <div id="elfinder"></div>
 </body>

coco/httpd/templates/finder/host_file_manager.html

-<html>
-<head>
-    <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-2.1.1.js') }}"></script>
-    <script type="text/javascript" src="{{ url_for('static', filename='js/jquery-ui-1.10.4.min.js') }}"></script>
-    <link rel="stylesheet" type="text/css" media="screen" href="{{ url_for('static', filename='elfinder/css/elfinder.min.css') }}">
-    <link rel="stylesheet" type="text/css" media="screen" href="{{ url_for('static', filename='elfinder/css/theme-gray.css') }}">
-    <script type="text/javascript" src="{{ url_for('static', filename='elfinder/elfinder.min.js') }}"></script>
-    <script type="text/javascript" src="{{ url_for('static', filename='elfinder/i18n/elfinder.pl.js') }}"></script>
-</head>
-<body style="margin: 0">
-<div id="elfinder"></div>
-<script type="text/javascript" charset="utf-8">
-    $(document).ready(function () {
-        var elf;
-        var opts = {
-            uiOptions : {
-                toolbar: [
-                    ['back', 'forward'],
-                    ['mkdir', 'mkfile'],
-                    ['copy', 'cut', 'paste'],
-                    ['rm'],
-                    ['rename'],
-                    ['view']
-                ]
-            },
-            // ui: [],
-            height: $(window).height(),
-            url: '{{ url_for("sftp_host_connector_view", host=host) }}',
-            resizable: false,
-            lang: 'pl',
-            rememberLastDir: false,
-            reloadClearHistory: true,
-            contextmenu: {
-                navbar: [
-                    'rm'
-                ],
-                cwd: [
-                    'reload', 'back', '|', 'mkdir', 'mkfile', '|',
-                    'upload'
-                ],
-                files: [
-                    'rm', 'rename', 'download'
-                ]
-            },
-            placesFirst: false
-        };
-        var start = function(lng) {
-            $(function() {
-                // Make elFinder (REQUIRED)
-                opts.lang = lng;
-                elf = $('#elfinder').elfinder(opts).elfinder('instance');
-            });
-        };
-        var load = function () {
-            var loct = window.location.search;
-            var full_lng, locm, lng;
-            if (loct && (locm = loct.match(/lang=([a-zA-Z_-]+)/))) {
-                full_lng = locm[1];
-            } else {
-                full_lng = (navigator.browserLanguage || navigator.language || navigator.userLanguage);
-            }
-            lng = full_lng.substr(0,2);
-            if (lng === 'ja') {
-                lng = 'jp';
-            }
-            else if (lng === 'pt') {
-                lng = 'pt_BR';
-            }
-            else if (lng === 'zh') {
-                lng = (full_lng.substr(0,5) == 'zh-tw')? 'zh_TW' : 'zh_CN';
-            }
-            if (lng !== 'en') {
-                $.ajax({
-                    url : "{{ url_for('static', filename='elfinder/i18n/') }}"+'/elfinder.'+lng+'.js',
-                    cache : true,
-                    dataType : 'script'
-                })
-                .done(function() {
-                    start(lng);
-                })
-                .fail(function() {
-                    start('en');
-                });
-           } else {
-                start(lng);
-           }
-        };
-        load();
-        var resizeTimer;
-        $(window).resize(function () {
-            resizeTimer && clearTimeout(resizeTimer);
-            if (!$('#elfinder').hasClass('elfinder-fullscreen')) {
-                resizeTimer = setTimeout(function () {
-                    var h = parseInt($(window).height());
-                    if (h != parseInt($('#elfinder').height())) {
-                        elf.resize('100%', h);
-                    }
-                }, 200);
-            }
-        });
-    });
-</script>
-</body>
-</html>

coco/httpd/utils.py

+# -*- coding: utf-8 -*-
+#
+
+__sftp_cached = {}
+
+
+def get_cached_sftp(sid):
+    return __sftp_cached.get(sid)
+
+
+def set_cache_sftp(sid, volume):
+    __sftp_cached[sid] = volume

coco/httpd/view.py

@@ -3,23 +3,32 @@
 
 from flask import render_template, request, jsonify
 
+from coco.utils import get_logger
 from .app import app
-from .finder import connector, volumes
+from .elfinder import connector, volumes
 from ..models import Connection
 from ..sftp import InternalSFTPClient
 from .auth import login_required
+from .utils import get_cached_sftp, set_cache_sftp
 from ..service import app_service
 
+logger = get_logger(__file__)
 
-@app.route('/elfinder/sftp/connector/<host>/', methods=['GET', 'POST'])
+
+@app.route('/coco/elfinder/sftp/connector/<host>/', methods=['GET', 'POST'])
 @login_required
 def sftp_host_connector_view(host):
-    user = request.current_user
-    connection = Connection(addr=(request.real_ip, 0))
-    connection.user = user
-    sftp = InternalSFTPClient(connection)
+    sid = request.args.get("sid") or request.values.get('sid')
+    sftp = get_cached_sftp(sid) if sid else None
+    if not sftp:
+        logger.debug("New sftp, sid: {} host: {}".format(sid, host))
+        user = request.current_user
+        connection = Connection(addr=(request.real_ip, 0))
+        connection.user = user
+        sftp = InternalSFTPClient(connection)
+        set_cache_sftp(sid, sftp)
     volume = volumes.SFTPVolume(sftp)
-    if host:
+    if host != '_':
         asset = app_service.get_asset(host)
         if not asset:
             return jsonify({'error': 'Not found this host'})
@@ -28,31 +37,25 @@ def sftp_host_connector_view(host):
             hostname = "{}.{}".format(asset.hostname, asset.org_name)
         volume.root_name = hostname
         volume.base_path = '/' + hostname
+
     handler = connector.ElFinderConnector([volume])
     handler.run(request)
 
-    # Some commands (e.g. read file) will return a Django View - if it
-    # is set, return it directly instead of building a response
+    # If download file, return a view response
     if handler.return_view:
         return handler.return_view
-
     if handler.headers['Content-type'] == 'application/json':
         return jsonify(handler.response)
 
 
-@app.route('/elfinder/sftp/connector/', methods=['GET', 'POST'])
-@login_required
-def sftp_connector_view():
-    return sftp_host_connector_view('')
-
-
-@app.route('/elfinder/sftp/<host>/')
+@app.route('/coco/elfinder/sftp/<host>/')
 def sftp_host_finder(host):
-    return render_template('finder/host_file_manager.html', host=host)
+    return render_template('elfinder/file_manager.html', host=host)
 
 
-@app.route('/elfinder/sftp/')
+@app.route('/coco/elfinder/sftp/')
 def sftp_finder():
-    return render_template('finder/file_manager.html')
+    return render_template('elfinder/file_manager.html', host='_')
+
 
 

coco/httpd/ws.py

@@ -3,14 +3,15 @@
 
 import os
 import uuid
-from flask_socketio import SocketIO, Namespace, join_room
-from flask import Flask, request
+from flask_socketio import join_room
+from flask import request
 
 from ..models import Connection, WSProxy
 from ..proxy import ProxyServer
 from ..utils import get_logger
 from ..ctx import app_service
 from .base import BaseNamespace
+from .utils import get_cached_sftp
 
 BASE_DIR = os.path.dirname(os.path.dirname(__file__))
 logger = get_logger(__file__)
@@ -192,4 +193,14 @@ class ProxyNamespace(BaseNamespace):
         pass
 
     def on_ping(self):
-        self.emit('pong')
+        self.emit('pong')
\ No newline at end of file
+
+
+class ElfinderNamespace(BaseNamespace):
+    def on_connect(self):
+        self.emit('data', {"sid": str(request.sid)})
+
+    def on_disconnect(self):
+        sftp = get_cached_sftp(request.sid)
+        if sftp:
+            sftp.close()

coco/sftp.py

@@ -27,11 +27,13 @@ class SFTPServer(paramiko.SFTPServerInterface):
             trans = chan.get_transport()
             sftp.close()
 
-            if [c for c in trans._channels.values() if not c.closed]:
+            active_channels = [c for c in trans._channels.values() if not c.closed]
+            if not active_channels:
+                print("CLose transport")
                 trans.close()
-            if sock:
-                sock.close()
-                sock.transport.close()
+                if sock:
+                    sock.close()
+                    sock.transport.close()
         self._sftp = {}
 
     def get_host_sftp(self, host, su):
@@ -100,7 +102,6 @@ class SFTPServer(paramiko.SFTPServerInterface):
             return False
 
     def create_ftp_log(self, path, operate, is_success=True, filename=None):
-        print("Create ftp log: {} {} {}".format(path, operate, filename))
         host, su, rpath = self.parse_path(path)
         asset = self.hosts.get(host)
         date_start = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + " +0000",
@@ -361,6 +362,7 @@ class InternalSFTPClient(SFTPServer):
             except OSError:
                 raise OSError("Open file {} failed".format(rpath))
             finally:
+                print("Open success")
                 self.create_ftp_log(path, operate, success)
             return f
 
@@ -371,4 +373,5 @@ class InternalSFTPClient(SFTPServer):
         return self.remove(path)
 
     def close(self):
+        print("End ")
         return self.session_ended()