From d32d9f781b81555d64e4b29fdafd497fbf2e31c9 Mon Sep 17 00:00:00 2001 From: Vadim Ushakov Date: Wed, 3 Nov 2021 11:48:02 +0700 Subject: [PATCH] Move getIpType() from helper to ConnectionServer --- .../AnnounceBitTorrentPlugin.py | 4 +- plugins/TrackerZero/TrackerZeroPlugin.py | 3 +- .../BootstrapperPlugin.py | 2 +- .../Test/TestBootstrapper.py | 6 +- src/Connection/Connection.py | 6 +- src/Connection/ConnectionServer.py | 36 ++++++++++- src/File/FileRequest.py | 4 +- src/File/FileServer.py | 8 +-- src/Peer/Peer.py | 63 ++++++++++++------- src/Site/SiteAnnouncer.py | 20 +++--- src/Site/SiteManager.py | 1 + src/util/helper.py | 3 +- 12 files changed, 109 insertions(+), 47 deletions(-) diff --git a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py index fab7bb1f..734070dd 100644 --- a/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py +++ b/plugins/AnnounceBitTorrent/AnnounceBitTorrentPlugin.py @@ -52,7 +52,7 @@ class SiteAnnouncerPlugin(object): ip, port = tracker_address.split("/")[0].split(":") tracker = UdpTrackerClient(ip, int(port)) - if helper.getIpType(ip) in self.getOpenedServiceTypes(): + if self.connection_server.getIpType(ip) in self.getOpenedServiceTypes(): tracker.peer_port = self.fileserver_port else: tracker.peer_port = 0 @@ -101,7 +101,7 @@ class SiteAnnouncerPlugin(object): def announceTrackerHttp(self, tracker_address, mode="start", num_want=10, protocol="http"): tracker_ip, tracker_port = tracker_address.rsplit(":", 1) - if helper.getIpType(tracker_ip) in self.getOpenedServiceTypes(): + if self.connection_server.getIpType(tracker_ip) in self.getOpenedServiceTypes(): port = self.fileserver_port else: port = 1 diff --git a/plugins/TrackerZero/TrackerZeroPlugin.py b/plugins/TrackerZero/TrackerZeroPlugin.py index e90f085d..a59bc309 100644 --- a/plugins/TrackerZero/TrackerZeroPlugin.py +++ b/plugins/TrackerZero/TrackerZeroPlugin.py @@ -122,7 +122,8 @@ class TrackerZero(object): time_onion_check = time.time() - s - ip_type = helper.getIpType(file_request.connection.ip) + connection_server = file_request.server + ip_type = connection_server.getIpType(file_request.connection.ip) if ip_type == "onion" or file_request.connection.ip in config.ip_local: is_port_open = False diff --git a/plugins/disabled-Bootstrapper/BootstrapperPlugin.py b/plugins/disabled-Bootstrapper/BootstrapperPlugin.py index 59e7af7b..5ddc36b6 100644 --- a/plugins/disabled-Bootstrapper/BootstrapperPlugin.py +++ b/plugins/disabled-Bootstrapper/BootstrapperPlugin.py @@ -49,7 +49,7 @@ class FileRequestPlugin(object): time_onion_check = time.time() - s - ip_type = helper.getIpType(self.connection.ip) + ip_type = self.server.getIpType(self.connection.ip) if ip_type == "onion" or self.connection.ip in config.ip_local: is_port_open = False diff --git a/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py b/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py index 69bdc54c..198cd022 100644 --- a/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py +++ b/plugins/disabled-Bootstrapper/Test/TestBootstrapper.py @@ -28,7 +28,7 @@ def bootstrapper_db(request): @pytest.mark.usefixtures("resetSettings") class TestBootstrapper: def testHashCache(self, file_server, bootstrapper_db): - ip_type = helper.getIpType(file_server.ip) + ip_type = file_server.getIpType(file_server.ip) peer = Peer(file_server.ip, 1544, connection_server=file_server) hash1 = hashlib.sha256(b"site1").digest() hash2 = hashlib.sha256(b"site2").digest() @@ -50,7 +50,7 @@ class TestBootstrapper: def testBootstrapperDb(self, file_server, bootstrapper_db): - ip_type = helper.getIpType(file_server.ip) + ip_type = file_server.getIpType(file_server.ip) peer = Peer(file_server.ip, 1544, connection_server=file_server) hash1 = hashlib.sha256(b"site1").digest() hash2 = hashlib.sha256(b"site2").digest() @@ -111,7 +111,7 @@ class TestBootstrapper: def testPassive(self, file_server, bootstrapper_db): peer = Peer(file_server.ip, 1544, connection_server=file_server) - ip_type = helper.getIpType(file_server.ip) + ip_type = file_server.getIpType(file_server.ip) hash1 = hashlib.sha256(b"hash1").digest() bootstrapper_db.peerAnnounce(ip_type, address=None, port=15441, hashes=[hash1]) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index 40519b7f..c147ee35 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -23,6 +23,7 @@ class Connection(object): ) def __init__(self, server, ip, port, sock=None, target_onion=None, is_tracker_connection=False): + self.server = server self.sock = sock self.cert_pin = None if "#" in ip: @@ -42,7 +43,6 @@ class Connection(object): self.is_private_ip = False self.is_tracker_connection = is_tracker_connection - self.server = server self.unpacker = None # Stream incoming socket messages here self.unpacker_bytes = 0 # How many bytes the unpacker received self.req_id = 0 # Last request id @@ -81,11 +81,11 @@ class Connection(object): def setIp(self, ip): self.ip = ip - self.ip_type = helper.getIpType(ip) + self.ip_type = self.server.getIpType(ip) self.updateName() def createSocket(self): - if helper.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"): + if self.server.getIpType(self.ip) == "ipv6" and not hasattr(socket, "socket_noproxy"): # Create IPv6 connection as IPv4 when using proxy return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) else: diff --git a/src/Connection/ConnectionServer.py b/src/Connection/ConnectionServer.py index 16834ff5..9f24e377 100644 --- a/src/Connection/ConnectionServer.py +++ b/src/Connection/ConnectionServer.py @@ -1,4 +1,5 @@ import logging +import re import time import sys import socket @@ -253,7 +254,7 @@ class ConnectionServer(object): pass def getConnection(self, ip=None, port=None, peer_id=None, create=True, site=None, is_tracker_connection=False): - ip_type = helper.getIpType(ip) + ip_type = self.getIpType(ip) has_per_site_onion = (ip.endswith(".onion") or self.port_opened.get(ip_type, None) == False) and self.tor_manager.start_onions and site if has_per_site_onion: # Site-unique connection for Tor if ip.endswith(".onion"): @@ -520,16 +521,47 @@ class ConnectionServer(object): median = (corrections[mid - 1] + corrections[mid] + corrections[mid + 1]) / 3 return median + + ############################################################################ + + # Methods for handling network address types + # (ipv4, ipv6, onion etc... more to be implemented by plugins) + # + # All the functions handling network address types have "Ip" in the name. + # So it was in the initial codebase, and I keep the naming, since I couldn't + # think of a better option. + # "IP" is short and quite clear and lets you understand that a variable + # contains a peer address or other transport-level address and not + # an address of ZeroNet site. + # + + # Returns type of the given network address. + # Since: 0.8.0 + # Replaces helper.getIpType() in order to be extensible by plugins. + def getIpType(self, ip): + if ip.endswith(".onion"): + return "onion" + elif ":" in ip: + return "ipv6" + elif re.match(r"[0-9\.]+$", ip): + return "ipv4" + else: + return "unknown" + # Checks if a network address can be reachable in the current configuration # and returs a string describing why it cannot. # If the network address can be reachable, returns False. + # Since: 0.8.0 def getIpUnreachability(self, ip): - ip_type = helper.getIpType(ip) + ip_type = self.getIpType(ip) if ip_type == 'onion' and not self.tor_manager.enabled: return "Can't connect to onion addresses, no Tor controller present" if config.tor == "always" and helper.isPrivateIp(ip) and ip not in config.ip_local: return "Can't connect to local IPs in Tor: always mode" return False + # Returns True if ConnctionServer has means for establishing outgoing + # connections to the given address. + # Since: 0.8.0 def isIpReachable(self, ip): return self.getIpUnreachability(ip) == False diff --git a/src/File/FileRequest.py b/src/File/FileRequest.py index f7249d81..8a16e591 100644 --- a/src/File/FileRequest.py +++ b/src/File/FileRequest.py @@ -376,7 +376,7 @@ class FileRequest(object): for hash_id, peers in found.items(): for peer in peers: - ip_type = helper.getIpType(peer.ip) + ip_type = self.server.getIpType(peer.ip) if len(back[ip_type][hash_id]) < 20: back[ip_type][hash_id].append(peer.packMyAddress()) return back @@ -430,7 +430,7 @@ class FileRequest(object): # Check requested port of the other peer def actionCheckport(self, params): - if helper.getIpType(self.connection.ip) == "ipv6": + if self.server.getIpType(self.connection.ip) == "ipv6": sock_address = (self.connection.ip, params["port"], 0, 0) else: sock_address = (self.connection.ip, params["port"]) diff --git a/src/File/FileServer.py b/src/File/FileServer.py index f9f31163..ac4b8c55 100644 --- a/src/File/FileServer.py +++ b/src/File/FileServer.py @@ -57,7 +57,7 @@ class FileServer(ConnectionServer): self.supported_ip_types = ["ipv4"] # Outgoing ip_type support - if helper.getIpType(ip) == "ipv6" or self.isIpv6Supported(): + if self.getIpType(ip) == "ipv6" or self.isIpv6Supported(): self.supported_ip_types.append("ipv6") if ip_type == "ipv6" or (ip_type == "dual" and "ipv6" in self.supported_ip_types): @@ -217,7 +217,7 @@ class FileServer(ConnectionServer): for ip_external in config.ip_external: SiteManager.peer_blacklist.append((ip_external, self.port)) # Add myself to peer blacklist - ip_external_types = set([helper.getIpType(ip) for ip in config.ip_external]) + ip_external_types = set([self.getIpType(ip) for ip in config.ip_external]) res = { "ipv4": "ipv4" in ip_external_types, "ipv6": "ipv6" in ip_external_types @@ -245,7 +245,7 @@ class FileServer(ConnectionServer): res_ipv6 = {"ip": None, "opened": None} else: res_ipv6 = res_ipv6_thread.get() - if res_ipv6["opened"] and not helper.getIpType(res_ipv6["ip"]) == "ipv6": + if res_ipv6["opened"] and not self.getIpType(res_ipv6["ip"]) == "ipv6": log.info("Invalid IPv6 address from port check: %s" % res_ipv6["ip"]) res_ipv6["opened"] = False @@ -266,7 +266,7 @@ class FileServer(ConnectionServer): for ip in interface_ips: if not helper.isPrivateIp(ip) and ip not in self.ip_external_list: self.ip_external_list.append(ip) - res[helper.getIpType(ip)] = True # We have opened port if we have external ip + res[self.getIpType(ip)] = True # We have opened port if we have external ip SiteManager.peer_blacklist.append((ip, self.port)) log.debug("External ip found on interfaces: %s" % ip) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index aad25110..6e4863e0 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -32,7 +32,7 @@ class Peer(object): self.site = site self.key = "%s:%s" % (ip, port) - self.ip_type = helper.getIpType(ip) + self.ip_type = None self.removed = False @@ -53,7 +53,7 @@ class Peer(object): self.reputation = 0 # More likely to connect if larger self.last_content_json_update = 0.0 # Modify date of last received content.json self.protected = 0 - self.reachable = False + self.reachable = None self.connection_error = 0 # Series of connection error self.hash_failed = 0 # Number of bad files from peer @@ -62,15 +62,14 @@ class Peer(object): self.protectedRequests = ["getFile", "streamFile", "update", "listModified"] - self.updateReachable() - def __getattr__(self, key): if key == "hashfield": self.has_hashfield = True self.hashfield = PeerHashfield() return self.hashfield else: - return getattr(self, key) + # Raise appropriately formatted attribute error + return object.__getattribute__(self, key) def log(self, text, log_level = None): if log_level is None: @@ -98,26 +97,18 @@ class Peer(object): self.protected = 0 return self.protected > 0 + def isTtlExpired(self, ttl): + last_activity = max(self.time_found, self.time_response) + return (time.time() - last_activity) > ttl + + # Since 0.8.0 def isConnected(self): if self.connection and not self.connection.connected: self.connection = None return self.connection and self.connection.connected - def isTtlExpired(self, ttl): - last_activity = max(self.time_found, self.time_response) - return (time.time() - last_activity) > ttl - - def isReachable(self): - return self.reachable - - def updateReachable(self): - connection_server = self.getConnectionServer() - if not self.port: - self.reachable = False - else: - self.reachable = connection_server.isIpReachable(self.ip) - # Peer proved to to be connectable recently + # Since 0.8.0 def isConnectable(self): if self.connection_error >= 1: # The last connection attempt failed return False @@ -125,6 +116,36 @@ class Peer(object): return False return self.isReachable() + # Since 0.8.0 + def isReachable(self): + if self.reachable is None: + self.updateCachedState() + return self.reachable + + # Since 0.8.0 + def getIpType(self): + if not self.ip_type: + self.updateCachedState() + return self.ip_type + + # We cache some ConnectionServer-related state for better performance. + # This kind of state currently doesn't change during a program session, + # and it's safe to read and cache it just once. But future versions + # may bring more pieces of dynamic configuration. So we update the state + # on each peer.found(). + def updateCachedState(self): + connection_server = self.getConnectionServer() + if not self.port or self.port == 1: # Port 1 considered as "no open port" + self.reachable = False + else: + self.reachable = connection_server.isIpReachable(self.ip) + self.ip_type = connection_server.getIpType(self.ip) + + + # FIXME: + # This should probably be changed. + # When creating a peer object, the caller must provide either `connection_server`, + # or `site`, so Peer object is able to use `site.connection_server`. def getConnectionServer(self): if self.connection_server: connection_server = self.connection_server @@ -179,7 +200,7 @@ class Peer(object): if self.connection and self.connection.connected: # We have connection to peer return self.connection else: # Try to find from other sites connections - self.connection = self.site.connection_server.getConnection(self.ip, self.port, create=False, site=self.site) + self.connection = self.getConnectionServer().getConnection(self.ip, self.port, create=False, site=self.site) if self.connection: self.connection.sites += 1 return self.connection @@ -213,7 +234,7 @@ class Peer(object): if source in ("tracker", "local"): self.site.peers_recent.appendleft(self) self.time_found = time.time() - self.updateReachable() + self.updateCachedState() # Send a command to peer and return response value def request(self, cmd, params={}, stream_to=None): diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py index 1cb0a445..1baf39af 100644 --- a/src/Site/SiteAnnouncer.py +++ b/src/Site/SiteAnnouncer.py @@ -35,19 +35,25 @@ class SiteAnnouncer(object): self.time_last_announce = 0 self.supported_tracker_count = 0 + # Returns connection_server rela + # Since 0.8.0 + @property + def connection_server(self): + return self.site.connection_server + def getTrackers(self): return config.trackers def getSupportedTrackers(self): trackers = self.getTrackers() - if not self.site.connection_server.tor_manager.enabled: + if not self.connection_server.tor_manager.enabled: trackers = [tracker for tracker in trackers if ".onion" not in tracker] trackers = [tracker for tracker in trackers if self.getAddressParts(tracker)] # Remove trackers with unknown address - if "ipv6" not in self.site.connection_server.supported_ip_types: - trackers = [tracker for tracker in trackers if helper.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"] + if "ipv6" not in self.connection_server.supported_ip_types: + trackers = [tracker for tracker in trackers if self.connection_server.getIpType(self.getAddressParts(tracker)["ip"]) != "ipv6"] return trackers @@ -118,10 +124,10 @@ class SiteAnnouncer(object): back = [] # Type of addresses they can reach me if config.trackers_proxy == "disable" and config.tor != "always": - for ip_type, opened in list(self.site.connection_server.port_opened.items()): + for ip_type, opened in list(self.connection_server.port_opened.items()): if opened: back.append(ip_type) - if self.site.connection_server.tor_manager.start_onions: + if self.connection_server.tor_manager.start_onions: back.append("onion") return back @@ -204,11 +210,11 @@ class SiteAnnouncer(object): self.stats[tracker]["time_status"] = time.time() self.stats[tracker]["last_error"] = str(error) self.stats[tracker]["time_last_error"] = time.time() - if self.site.connection_server.has_internet: + if self.connection_server.has_internet: self.stats[tracker]["num_error"] += 1 self.stats[tracker]["num_request"] += 1 global_stats[tracker]["num_request"] += 1 - if self.site.connection_server.has_internet: + if self.connection_server.has_internet: global_stats[tracker]["num_error"] += 1 self.updateWebsocket(tracker="error") return False diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py index 035e9279..8175a1f5 100644 --- a/src/Site/SiteManager.py +++ b/src/Site/SiteManager.py @@ -172,6 +172,7 @@ class SiteManager(object): return self.resolveDomain(domain) # Checks if the address is blocked. To be implemented in content filter plugins. + # Since 0.8.0 def isAddressBlocked(self, address): return False diff --git a/src/util/helper.py b/src/util/helper.py index 61455b08..f44bcfce 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -290,7 +290,8 @@ local_ip_pattern = re.compile(r"^127\.|192\.168\.|10\.|172\.1[6-9]\.|172\.2[0-9] def isPrivateIp(ip): return local_ip_pattern.match(ip) - +# XXX: Deprecated. Use ConnectionServer.getIpType() instead. +# To be removed in 0.9.0 def getIpType(ip): if ip.endswith(".onion"): return "onion"