Redesign cleanupSites() and all the related stuff and rename it to periodic maintenance.
This commit is contained in:
parent
511a90a5c5
commit
8fd88c50f9
6 changed files with 175 additions and 81 deletions
125
src/Site/Site.py
125
src/Site/Site.py
|
@ -40,6 +40,9 @@ class Site(object):
|
|||
self.log = logging.getLogger("Site:%s" % self.address_short)
|
||||
self.addEventListeners()
|
||||
|
||||
self.periodic_maintenance_interval = 60 * 20
|
||||
self.periodic_maintenance_timestamp = 0
|
||||
|
||||
self.content = None # Load content.json
|
||||
self.peers = {} # Key: ip:port, Value: Peer.Peer
|
||||
self.peers_recent = collections.deque(maxlen=150)
|
||||
|
@ -853,6 +856,11 @@ class Site(object):
|
|||
if self.isServing():
|
||||
self.announcer.announce(*args, **kwargs)
|
||||
|
||||
# The engine tries to maintain the number of active connections:
|
||||
# >= getPreferableActiveConnectionCount()
|
||||
# and
|
||||
# <= getActiveConnectionCountLimit()
|
||||
|
||||
def getPreferableActiveConnectionCount(self):
|
||||
if not self.isServing():
|
||||
return 0
|
||||
|
@ -874,8 +882,16 @@ class Site(object):
|
|||
if len(self.peers) < 50:
|
||||
count = max(count, 5)
|
||||
|
||||
count = min(count, config.connected_limit)
|
||||
|
||||
return count
|
||||
|
||||
def getActiveConnectionCountLimit(self):
|
||||
count_above_preferable = 2
|
||||
limit = self.getPreferableActiveConnectionCount() + count_above_preferable
|
||||
limit = min(limit, config.connected_limit)
|
||||
return limit
|
||||
|
||||
def tryConnectingToMorePeers(self, more=1, pex=True, try_harder=False):
|
||||
max_peers = more * 2 + 10
|
||||
if try_harder:
|
||||
|
@ -920,7 +936,7 @@ class Site(object):
|
|||
if num is None:
|
||||
num = self.getPreferableActiveConnectionCount()
|
||||
|
||||
need = min(len(self.peers), num, config.connected_limit)
|
||||
need = min(len(self.peers), num)
|
||||
|
||||
connected = self.bringConnections(
|
||||
need=need,
|
||||
|
@ -935,8 +951,15 @@ class Site(object):
|
|||
pex=pex,
|
||||
try_harder=True)
|
||||
|
||||
if connected < num:
|
||||
self.markConnectedPeersProtected()
|
||||
|
||||
return connected
|
||||
|
||||
def markConnectedPeersProtected(self):
|
||||
for peer in site.getConnectedPeers():
|
||||
peer.markProtected()
|
||||
|
||||
# Return: Probably peers verified to be connectable recently
|
||||
def getConnectablePeers(self, need_num=5, ignore=[], allow_private=True):
|
||||
peers = list(self.peers.values())
|
||||
|
@ -1022,53 +1045,87 @@ class Site(object):
|
|||
back.append(peer)
|
||||
return back
|
||||
|
||||
# Cleanup probably dead peers and close connection if too much
|
||||
def cleanupPeers(self, peers_protected=[]):
|
||||
def removeDeadPeers(self):
|
||||
peers = list(self.peers.values())
|
||||
if len(peers) > 20:
|
||||
# Cleanup old peers
|
||||
removed = 0
|
||||
if len(peers) > 1000:
|
||||
ttl = 60 * 60 * 1
|
||||
else:
|
||||
ttl = 60 * 60 * 4
|
||||
if len(peers) <= 20:
|
||||
return
|
||||
|
||||
for peer in peers:
|
||||
if peer.connection and peer.connection.connected:
|
||||
continue
|
||||
if peer.connection and not peer.connection.connected:
|
||||
peer.connection = None # Dead connection
|
||||
if time.time() - peer.time_found > ttl: # Not found on tracker or via pex in last 4 hour
|
||||
peer.remove("Time found expired")
|
||||
removed += 1
|
||||
if removed > len(peers) * 0.1: # Don't remove too much at once
|
||||
break
|
||||
removed = 0
|
||||
if len(peers) > 1000:
|
||||
ttl = 60 * 60 * 1
|
||||
elif len(peers) > 100:
|
||||
ttl = 60 * 60 * 4
|
||||
else:
|
||||
ttl = 60 * 60 * 8
|
||||
|
||||
if removed:
|
||||
self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers)))
|
||||
for peer in peers:
|
||||
if peer.isConnected() or peer.isProtected():
|
||||
continue
|
||||
if peer.isTtlExpired(ttl):
|
||||
peer.remove("TTL expired")
|
||||
removed += 1
|
||||
if removed > len(peers) * 0.1: # Don't remove too much at once
|
||||
break
|
||||
|
||||
# Close peers over the limit
|
||||
closed = 0
|
||||
connected_peers = [peer for peer in self.getConnectedPeers() if peer.connection.connected] # Only fully connected peers
|
||||
need_to_close = len(connected_peers) - config.connected_limit
|
||||
if removed:
|
||||
self.log.debug("Cleanup peers result: Removed %s, left: %s" % (removed, len(self.peers)))
|
||||
|
||||
if closed < need_to_close:
|
||||
# Try to keep connections with more sites
|
||||
# Cleanup probably dead peers and close connection if too much
|
||||
def cleanupPeers(self):
|
||||
self.removeDeadPeers()
|
||||
|
||||
limit = self.getActiveConnectionCountLimit()
|
||||
connected_peers = [peer for peer in self.getConnectedPeers() if peer.isConnected()] # Only fully connected peers
|
||||
need_to_close = len(connected_peers) - limit
|
||||
|
||||
if need_to_close > 0:
|
||||
closed = 0
|
||||
for peer in sorted(connected_peers, key=lambda peer: min(peer.connection.sites, 5)):
|
||||
if not peer.connection:
|
||||
if not peer.isConnected():
|
||||
continue
|
||||
if peer.key in peers_protected:
|
||||
if peer.isProtected():
|
||||
continue
|
||||
if peer.connection.sites > 5:
|
||||
break
|
||||
peer.connection.close("Cleanup peers")
|
||||
peer.connection = None
|
||||
peer.disconnect("Cleanup peers")
|
||||
closed += 1
|
||||
if closed >= need_to_close:
|
||||
break
|
||||
|
||||
if need_to_close > 0:
|
||||
self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (len(connected_peers), need_to_close, closed))
|
||||
self.log.debug("Connected: %s, Need to close: %s, Closed: %s" % (
|
||||
len(connected_peers), need_to_close, closed))
|
||||
|
||||
def runPeriodicMaintenance(self, startup=False, force=False):
|
||||
if not self.isServing():
|
||||
return False
|
||||
|
||||
scheduled_time = self.periodic_maintenance_timestamp + self.periodic_maintenance_interval
|
||||
|
||||
if time.time() < scheduled_time and not force:
|
||||
return False
|
||||
|
||||
self.periodic_maintenance_timestamp = time.time()
|
||||
|
||||
self.log.debug("runPeriodicMaintenance: startup=%s" % startup)
|
||||
|
||||
if not startup:
|
||||
self.cleanupPeers()
|
||||
|
||||
if self.peers:
|
||||
with gevent.Timeout(10, exception=False):
|
||||
self.announcer.announcePex()
|
||||
|
||||
# Last check modification failed
|
||||
if self.content_updated is False:
|
||||
self.update()
|
||||
elif self.bad_files:
|
||||
self.retryBadFiles()
|
||||
|
||||
self.needConnections(check_site_on_reconnect=True)
|
||||
|
||||
self.periodic_maintenance_timestamp = time.time()
|
||||
|
||||
return True
|
||||
|
||||
# Send hashfield to peers
|
||||
def sendMyHashfield(self, limit=5):
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue