[Update] Debug gc

coco/app.py

@@ -8,7 +8,9 @@ import time
 import threading
 import socket
 import json
+import tracemalloc
 
+import gc
 from jms.service import AppService
 
 from .config import Config
@@ -66,6 +68,8 @@ class Coco:
         self.replay_recorder_class = None
         self.command_recorder_class = None
         self._task_handler = None
+        tracemalloc.start()
+        self.snapshot = tracemalloc.take_snapshot()
 
     @property
     def name(self):
@@ -130,6 +134,16 @@ class Coco:
     def heartbeat(self):
         _sessions = [s.to_json() for s in self.sessions]
         tasks = self.service.terminal_heartbeat(_sessions)
+        gc.collect()
+        snapshort2 = tracemalloc.take_snapshot()
+        top_stats = snapshort2.compare_to(self.snapshot, 'traceback')
+        print("[ Top 10 differences ]")
+        for stat in top_stats[:10]:
+            print(stat.count_diff)
+        # gc.collect()
+        # print(objgraph.show_most_common_types())
+        # for i in ("User", "Client", "Session", "ProxyServer", "Transport", "Channel", "InteractiveServer", "SSHInterface", "Server", "Asset"):
+        #     print("{} objects: {}".format(i, len(objgraph.by_type(i))))
         if tasks:
             self.handle_task(tasks)
         if tasks is False:
@@ -222,7 +236,6 @@ class Coco:
                 self.clients.remove(client)
                 logger.info("Client {} leave, total {} now".format(client, len(self.clients)))
                 client.close()
-                del client
             except:
                 pass
 

coco/interactive.py

@@ -83,7 +83,6 @@ class InteractiveServer:
 
         while True:
             data = self.client.recv(10)
-            logger.debug(data)
             if len(data) == 0:
                 self.app.remove_client(self.client)
                 break
@@ -300,9 +299,6 @@ class InteractiveServer:
         forwarder = ProxyServer(self.app, self.client)
         forwarder.proxy(asset, system_user)
 
-    def replay_session(self, session_id):
-        pass
-
     def interact(self):
         self.display_banner()
         while True:
@@ -322,4 +318,6 @@ class InteractiveServer:
 
     def close(self):
         self.app.remove_client(self.client)
-        logger.info("Exit interactive server")
+
+    # def __del__(self):
+    #     print("GC: Interactive class been gc")

coco/interface.py

@@ -21,7 +21,7 @@ class SSHInterface(paramiko.ServerInterface):
 
     def __init__(self, app, request):
         self._app = weakref.ref(app)
-        self.request = request
+        self._request = weakref.ref(request)
         self.event = threading.Event()
         self.auth_valid = False
 
@@ -29,6 +29,10 @@ class SSHInterface(paramiko.ServerInterface):
     def app(self):
         return self._app()
 
+    @property
+    def request(self):
+        return self._request()
+
     def check_auth_interactive(self, username, submethods):
         logger.info("Check auth interactive: %s %s" % (username, submethods))
         return paramiko.AUTH_FAILED
@@ -177,6 +181,9 @@ class SSHInterface(paramiko.ServerInterface):
     def get_banner(self):
         return None, None
 
+    # def __del__(self):
+    #     print("GC: SSH interface gc")
+
 
 
 

coco/models.py

@@ -21,6 +21,9 @@ class Request:
         self.change_size_event = threading.Event()
         self.date_start = datetime.datetime.now()
 
+    # def __del__(self):
+    #     print("GC: Request object gc")
+
 
 class SizedList(list):
     def __init__(self, maxsize=0):
@@ -74,8 +77,8 @@ class Client:
     def __str__(self):
         return "<%s from %s:%s>" % (self.user, self.addr[0], self.addr[1])
 
-    def __del__(self):
-        print("GC: client object has been gc")
+    # def __del__(self):
+    #     print("GC: Client object has been gc")
 
 
 class Server:
@@ -157,9 +160,9 @@ class Server:
     def close(self):
         logger.info("Closed server {}".format(self))
         self.parse(b'')
-        self.chan.close()
         self.stop_evt.set()
         self.chan.close()
+        self.chan.transport.close()
 
     @staticmethod
     def _have_enter_char(s):
@@ -186,8 +189,8 @@ class Server:
     def __str__(self):
         return "<To: {}>".format(str(self.asset))
 
-    def __del__(self):
-        print("GC: Server object has been gc")
+    # def __del__(self):
+    #     print("GC: Server object has been gc")
 
 
 class WSProxy:

coco/proxy.py

@@ -24,9 +24,9 @@ class ProxyServer:
     def __init__(self, app, client):
         self._app = weakref.ref(app)
         self.client = client
-        self.request = client.request
         self.server = None
         self.connecting = True
+        self.stop_event = threading.Event()
 
     @property
     def app(self):
@@ -47,6 +47,8 @@ class ProxyServer:
         self.app.add_session(session)
         self.watch_win_size_change_async()
         session.bridge()
+        self.stop_event.set()
+        self.end_watch_win_size_change()
         self.app.remove_session(session)
 
     def validate_permission(self, asset, system_user):
@@ -117,17 +119,20 @@ class ProxyServer:
             self.connecting = False
             self.client.send(b'\r\n')
 
-        term = self.request.meta.get('term', 'xterm')
-        width = self.request.meta.get('width', 80)
-        height = self.request.meta.get('height', 24)
+        request = self.client.request
+        term = request.meta.get('term', 'xterm')
+        width = request.meta.get('width', 80)
+        height = request.meta.get('height', 24)
         chan = ssh.invoke_shell(term, width=width, height=height)
         return Server(chan, asset, system_user)
 
     def watch_win_size_change(self):
-        while self.request.change_size_event.wait():
-            self.request.change_size_event.clear()
-            width = self.request.meta.get('width', 80)
-            height = self.request.meta.get('height', 24)
+        while self.client.request.change_size_event.wait():
+            if self.stop_event.is_set():
+                break
+            self.client.request.change_size_event.clear()
+            width = self.client.request.meta.get('width', 80)
+            height = self.client.request.meta.get('height', 24)
             logger.debug("Change win size: %s - %s" % (width, height))
             try:
                 self.server.chan.resize_pty(width=width, height=height)
@@ -139,6 +144,9 @@ class ProxyServer:
         thread.daemon = True
         thread.start()
 
+    def end_watch_win_size_change(self):
+        self.client.request.change_size_event.set()
+
     def send_connecting_message(self, asset, system_user):
         def func():
             delay = 0.0
@@ -149,5 +157,3 @@ class ProxyServer:
                 delay += 0.1
         thread = threading.Thread(target=func)
         thread.start()
-
-

coco/recorder.py

@@ -178,9 +178,9 @@ class ServerReplayRecorder(ReplayRecorder):
             logger.error("failed report session {}'s replay log".format(session_id))
             return False
 
-    def __del__(self):
-        print("Server replay recorder has been gc")
-        del self.file
+    # def __del__(self):
+    #     print("GC: Server replay recorder has been gc")
+    #     del self.file
 
 
 class ServerCommandRecorder(CommandRecorder, metaclass=Singleton):
@@ -226,8 +226,8 @@ class ServerCommandRecorder(CommandRecorder, metaclass=Singleton):
     def session_end(self, session_id):
         pass
 
-    def __del__(self):
-        print("GC: Session command storage has been gc")
+    # def __del__(self):
+    #     print("GC: Session command storage has been gc")
 
 
 class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
@@ -279,8 +279,8 @@ class ESCommandRecorder(CommandRecorder, metaclass=Singleton):
     def session_end(self, session_id):
         pass
 
-    def __del__(self):
-        print("ES command storage has been gc".format(self))
+    # def __del__(self):
+    #     print("GC: ES command storage has been gc".format(self))
 
 
 def get_command_recorder_class(config):

coco/session.py

@@ -107,7 +107,10 @@ class Session:
 
     def terminate(self):
         msg = b"Terminate by administrator\r\n"
+        try:
             self.client.send(msg)
+        except OSError:
+            pass
         self.close()
 
     def bridge(self):
@@ -185,5 +188,5 @@ class Session:
     def __repr__(self):
         return self.id
 
-    def __del__(self):
-        print("GC: Session object has been GC")
+    # def __del__(self):
+    #     print("GC: Session object has been GC")

coco/sshd.py

@@ -72,10 +72,16 @@ class SSHServer:
             return
 
         while True:
+            if not transport.is_active():
+                transport.close()
+                sock.close()
+                break
             chan = transport.accept()
+            server.event.wait(5)
+
             if chan is None:
                 continue
-            server.event.wait(5)
+
             if not server.event.is_set():
                 logger.warning("Client not request a valid request, exiting")
                 return