Redesign the Internet outage detection. Improvements in FileServer threads.
This commit is contained in:
parent
829fd46781
commit
d1b9cc8261
4 changed files with 173 additions and 30 deletions
|
@ -442,6 +442,7 @@ class Connection(object):
|
|||
def handleMessage(self, message):
|
||||
cmd = message["cmd"]
|
||||
|
||||
self.updateOnlineStatus(successful_activity=True)
|
||||
self.last_message_time = time.time()
|
||||
self.last_cmd_recv = cmd
|
||||
if cmd == "response": # New style response
|
||||
|
@ -504,6 +505,7 @@ class Connection(object):
|
|||
|
||||
# Send data to connection
|
||||
def send(self, message, streaming=False):
|
||||
self.updateOnlineStatus(outgoing_activity=True)
|
||||
self.last_send_time = time.time()
|
||||
if config.debug_socket:
|
||||
self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (
|
||||
|
@ -543,6 +545,11 @@ class Connection(object):
|
|||
message = None
|
||||
with self.send_lock:
|
||||
self.sock.sendall(data)
|
||||
# XXX: Should not be used here:
|
||||
# self.updateOnlineStatus(successful_activity=True)
|
||||
# Looks like self.sock.sendall() returns normally, instead of
|
||||
# raising an Exception (at least, some times).
|
||||
# So the only way of detecting the network activity is self.handleMessage()
|
||||
except Exception as err:
|
||||
self.close("Send error: %s (cmd: %s)" % (err, stat_key))
|
||||
return False
|
||||
|
@ -633,3 +640,8 @@ class Connection(object):
|
|||
self.sock = None
|
||||
self.unpacker = None
|
||||
self.event_connected = None
|
||||
|
||||
def updateOnlineStatus(self, outgoing_activity=False, successful_activity=False):
|
||||
self.server.updateOnlineStatus(self,
|
||||
outgoing_activity=outgoing_activity,
|
||||
successful_activity=successful_activity)
|
||||
|
|
|
@ -41,7 +41,11 @@ class ConnectionServer(object):
|
|||
self.ip_incoming = {} # Incoming connections from ip in the last minute to avoid connection flood
|
||||
self.broken_ssl_ips = {} # Peerids of broken ssl connections
|
||||
self.ips = {} # Connection by ip
|
||||
|
||||
self.has_internet = True # Internet outage detection
|
||||
self.last_outgoing_internet_activity_time = 0 # Last time the application tried to send any data
|
||||
self.last_successful_internet_activity_time = 0 # Last time the application successfully sent or received any data
|
||||
self.internet_outage_threshold = 60 * 2
|
||||
|
||||
self.stream_server = None
|
||||
self.stream_server_proxy = None
|
||||
|
@ -60,6 +64,8 @@ class ConnectionServer(object):
|
|||
self.num_outgoing = 0
|
||||
self.had_external_incoming = False
|
||||
|
||||
|
||||
|
||||
self.timecorrection = 0.0
|
||||
self.pool = Pool(500) # do not accept more than 500 connections
|
||||
|
||||
|
@ -252,8 +258,8 @@ class ConnectionServer(object):
|
|||
while self.running:
|
||||
run_i += 1
|
||||
self.ip_incoming = {} # Reset connected ips counter
|
||||
last_message_time = 0
|
||||
s = time.time()
|
||||
self.updateOnlineStatus(None)
|
||||
for connection in self.connections[:]: # Make a copy
|
||||
if connection.ip.endswith(".onion") or config.tor == "always":
|
||||
timeout_multipler = 2
|
||||
|
@ -261,9 +267,6 @@ class ConnectionServer(object):
|
|||
timeout_multipler = 1
|
||||
|
||||
idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
|
||||
if connection.last_message_time > last_message_time and not connection.is_private_ip:
|
||||
# Message from local IPs does not means internet connection
|
||||
last_message_time = connection.last_message_time
|
||||
|
||||
if connection.unpacker and idle > 30:
|
||||
# Delete the unpacker if not needed
|
||||
|
@ -311,18 +314,6 @@ class ConnectionServer(object):
|
|||
# Reset bad action counter every 30 min
|
||||
connection.bad_actions = 0
|
||||
|
||||
# Internet outage detection
|
||||
if time.time() - last_message_time > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)):
|
||||
# Offline: Last message more than 60-600sec depending on connection number
|
||||
if self.has_internet and last_message_time:
|
||||
self.has_internet = False
|
||||
self.onInternetOffline()
|
||||
else:
|
||||
# Online
|
||||
if not self.has_internet:
|
||||
self.has_internet = True
|
||||
self.onInternetOnline()
|
||||
|
||||
self.timecorrection = self.getTimecorrection()
|
||||
|
||||
if time.time() - s > 0.01:
|
||||
|
@ -353,6 +344,48 @@ class ConnectionServer(object):
|
|||
))
|
||||
return num_closed
|
||||
|
||||
# Internet outage detection
|
||||
def updateOnlineStatus(self, connection, outgoing_activity=False, successful_activity=False):
|
||||
|
||||
now = time.time()
|
||||
|
||||
if connection and not connection.is_private_ip:
|
||||
if outgoing_activity:
|
||||
self.last_outgoing_internet_activity_time = now
|
||||
if successful_activity:
|
||||
self.last_successful_internet_activity_time = now
|
||||
self.setInternetStatus(True)
|
||||
return
|
||||
|
||||
if not self.last_outgoing_internet_activity_time:
|
||||
return
|
||||
|
||||
if (
|
||||
(self.last_successful_internet_activity_time < now - self.internet_outage_threshold)
|
||||
and
|
||||
(self.last_successful_internet_activity_time < self.last_outgoing_internet_activity_time)
|
||||
):
|
||||
self.setInternetStatus(False)
|
||||
return
|
||||
|
||||
# This is the old algorithm just in case we missed something
|
||||
idle = now - self.last_successful_internet_activity_time
|
||||
if idle > max(60, 60 * 10 / max(1, float(len(self.connections)) / 50)):
|
||||
# Offline: Last successful activity more than 60-600sec depending on connection number
|
||||
self.setInternetStatus(False)
|
||||
return
|
||||
|
||||
def setInternetStatus(self, status):
|
||||
if self.has_internet == status:
|
||||
return
|
||||
|
||||
self.has_internet = status
|
||||
|
||||
if self.has_internet:
|
||||
gevent.spawn(self.onInternetOnline)
|
||||
else:
|
||||
gevent.spawn(self.onInternetOffline)
|
||||
|
||||
def onInternetOnline(self):
|
||||
self.log.info("Internet online")
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue