Save significant amount of memory by remove unused msgpack unpackers, Log unhandled exceptions, Connectionserver checker error bugfix
This commit is contained in:
parent
b9cf5d97f6
commit
bce0f56d45
4 changed files with 19 additions and 9 deletions
|
@ -21,7 +21,7 @@ class Connection:
|
|||
|
||||
self.server = server
|
||||
self.log = logging.getLogger(str(self))
|
||||
self.unpacker = msgpack.Unpacker() # Stream incoming socket messages here
|
||||
self.unpacker = None # Stream incoming socket messages here
|
||||
self.req_id = 0 # Last request id
|
||||
self.handshake = {} # Handshake info got from peer
|
||||
self.connected = False
|
||||
|
@ -133,8 +133,8 @@ class Connection:
|
|||
self.connected = True
|
||||
self.event_connected.set(self.protocol) # Mark handshake as done
|
||||
|
||||
unpacker = self.unpacker
|
||||
unpacker.feed(firstchar) # Feed the first char we already requested
|
||||
self.unpacker = msgpack.Unpacker()
|
||||
self.unpacker.feed(firstchar) # Feed the first char we already requested
|
||||
try:
|
||||
while True:
|
||||
buff = sock.recv(16*1024)
|
||||
|
@ -142,8 +142,11 @@ class Connection:
|
|||
self.last_recv_time = time.time()
|
||||
self.incomplete_buff_recv += 1
|
||||
self.bytes_recv += len(buff)
|
||||
unpacker.feed(buff)
|
||||
for message in unpacker:
|
||||
if not self.unpacker:
|
||||
self.log.debug("Unpacker created")
|
||||
self.unpacker = msgpack.Unpacker()
|
||||
self.unpacker.feed(buff)
|
||||
for message in self.unpacker:
|
||||
self.incomplete_buff_recv = 0
|
||||
self.handleMessage(message)
|
||||
message = None
|
||||
|
|
|
@ -112,8 +112,13 @@ class ConnectionServer:
|
|||
for connection in self.connections[:]: # Make a copy
|
||||
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
||||
|
||||
if connection.unpacker and idle > 30: # Delete the unpacker if not needed
|
||||
del connection.unpacker
|
||||
connection.unpacker = None
|
||||
connection.log.debug("Unpacker deleted")
|
||||
|
||||
if idle > 60*60: # Wake up after 1h
|
||||
connection.log.debug("[Cleanup] After wakeup: %s" % connection.read_bytes(1024))
|
||||
connection.log.debug("[Cleanup] After wakeup, idle: %s" % idle)
|
||||
connection.close()
|
||||
|
||||
elif idle > 20*60 and connection.last_send_time < time.time()-10: # Idle more than 20 min and we not send request in last 10 sec
|
||||
|
@ -125,7 +130,7 @@ class ConnectionServer:
|
|||
connection.close()
|
||||
|
||||
elif idle > 10 and connection.incomplete_buff_recv > 0: # Incompelte data with more than 10 sec idle
|
||||
connection.log.debug("[Cleanup] Connection buff stalled, content: %s" % connection.read_bytes(1024))
|
||||
connection.log.debug("[Cleanup] Connection buff stalled")
|
||||
connection.close()
|
||||
|
||||
elif idle > 10 and connection.waiting_requests and time.time() - connection.last_send_time > 10: # Sent command and no response in 10 sec
|
||||
|
|
|
@ -19,7 +19,9 @@ def handleError(*args):
|
|||
|
||||
# Ignore notify errors
|
||||
def handleErrorNotify(*args):
|
||||
if args[0].__name__ != "Notify": sys.__excepthook__(*args)
|
||||
if args[0].__name__ != "Notify":
|
||||
logging.exception("Unhandled exception")
|
||||
sys.__excepthook__(*args)
|
||||
|
||||
|
||||
OriginalGreenlet = gevent.Greenlet
|
||||
|
|
|
@ -55,7 +55,7 @@ class FileRequest:
|
|||
buff = StringIO(params["body"])
|
||||
valid = site.content_manager.verifyFile(params["inner_path"], buff)
|
||||
if valid == True: # Valid and changed
|
||||
self.log.debug("Update for %s looks valid, saving..." % params["inner_path"])
|
||||
self.log.info("Update for %s looks valid, saving..." % params["inner_path"])
|
||||
buff.seek(0)
|
||||
file = open(site.getPath(params["inner_path"]), "wb")
|
||||
shutil.copyfileobj(buff, file) # Write buff to disk
|
||||
|
|
Loading…
Reference in a new issue