Better internet outage detection based on last message from connections

src/Connection/ConnectionServer.py

@@ -33,6 +33,7 @@ class ConnectionServer:
         self.ip_incoming = {}  # Incoming connections from ip in the last minute to avoid connection flood
         self.broken_ssl_peer_ids = {}  # Peerids of broken ssl connections
         self.ips = {}  # Connection by ip
+        self.has_internet = True  # Internet outage detection
 
         self.running = True
         self.thread_checker = gevent.spawn(self.checkConnections)
@@ -170,8 +171,10 @@ class ConnectionServer:
             time.sleep(60)  # Check every minute
             self.ip_incoming = {}  # Reset connected ips counter
             self.broken_ssl_peer_ids = {}  # Reset broken ssl peerids count
+            last_message_time = 0
             for connection in self.connections[:]:  # Make a copy
                 idle = time.time() - max(connection.last_recv_time, connection.start_time, connection.last_message_time)
+                last_message_time = max(last_message_time, connection.last_message_time)
 
                 if connection.unpacker and idle > 30:
                     # Delete the unpacker if not needed
@@ -215,3 +218,21 @@ class ConnectionServer:
                 elif run_i % 30 == 0:
                     # Reset bad action counter every 30 min
                     connection.bad_actions = 0
+
+            # Internet outage detection
+            if time.time() - last_message_time > max(30, 60*5/max(1,len(self.connections)/30)):
+                # Offline: Last message more than 30-300sec depending on connection number
+                if self.has_internet:
+                    self.has_internet = False
+                    self.onInternetOffline()
+            else:
+                # Online
+                if not self.has_internet:
+                    self.has_internet = True
+                    self.onInternetOnline()
+
+    def onInternetOnline(self):
+        self.log.info("Internet online")
+
+    def onInternetOffline(self):
+        self.log.info("Internet offline")

src/File/FileServer.py

@@ -36,18 +36,15 @@ class FileServer(ConnectionServer):
                 )
             else:
                 self.log.debug("FileRequest: %s %s" % (str(connection), message["cmd"]))
-
-        # Internet connection outage detection
-        if len(self.connections) > 5:
-            if time.time() - self.last_request > 60*5:
-                self.log.info("Internet outage detected, no requests received for %.0fs" % (time.time() - self.last_request))
-                self.last_request = time.time()
-                gevent.spawn(self.checkSites, check_files=False, force_port_check=True)
-            else:
-                self.last_request = time.time()
-
         req = FileRequest(self, connection)
         req.route(message["cmd"], message.get("req_id"), message.get("params"))
+        if not self.has_internet:
+            self.has_internet = True
+            self.onInternetOnline()
+
+    def onInternetOnline(self):
+        self.log.info("Internet online")
+        gevent.spawn(self.checkSites, check_files=False, force_port_check=True)
 
     # Reload the FileRequest class to prevent restarts in debug mode
     def reload(self):